diff --git a/examples/cpp/llama/llama_config.ini b/examples/cpp/llama/llama_config.ini index ef789d35d..5a2ab8c3d 100644 --- a/examples/cpp/llama/llama_config.ini +++ b/examples/cpp/llama/llama_config.ini @@ -6,7 +6,7 @@ tensor_para_size=1 pipeline_para_size=1 model_name=llama_7b -model_dir=/notebooks/llama-2-70b-hf-ft-tp-1_llama_decoder/1/1-gpu/ +model_dir=/notebooks/code-llama-34b_llama_decoder/1/1-gpu/ [request] beam_width=1 # beam width for beam search @@ -17,18 +17,19 @@ repetition_penalty=1.0 ; Use for sampling presence_penalty=0.0 ; Only one of repetition_penalty and presence_penalty are allowed. len_penalty=0.0 beam_search_diversity_rate=0.0 -request_batch_size=8 # determine by the request +request_batch_size=1 # determine by the request request_output_len=32 # determine by the request [llama_7b] head_num = 64 kv_head_num = 8 size_per_head = 128 -inter_size = 28672 -num_layer = 3 +inter_size = 22016 +num_layer = 1 rotary_embedding = 128 layernorm_eps = 1e-05 vocab_size = 32000 +rope_theta = 1000000 start_id = 1 end_id = 2 weight_data_type = fp16 diff --git a/examples/cpp/llama/llama_example.cc b/examples/cpp/llama/llama_example.cc index ce761f75f..6d360e73d 100644 --- a/examples/cpp/llama/llama_example.cc +++ b/examples/cpp/llama/llama_example.cc @@ -44,7 +44,7 @@ int main(int argc, char* argv[]) ini_name = std::string(argv[1]); } else { - ini_name = "/notebooks/FasterTransformer/examples/cpp/llama/llama_config.ini"; + ini_name = "/notebooks/tmp/FasterTransformer/examples/cpp/llama/llama_config.ini"; } INIReader reader = INIReader(ini_name); @@ -90,6 +90,7 @@ void llama_example(const INIReader reader) const size_t decoder_layers = reader.GetInteger(model_name, "num_layer"); const size_t rotary_embedding_dim = reader.GetInteger(model_name, "rotary_embedding"); const float rope_theta = reader.GetFloat(model_name, "rope_theta", 10000.f); + printf("rope_theta: %f\n", rope_theta); const float layernorm_eps = reader.GetFloat(model_name, "layernorm_eps"); const int start_id = reader.GetInteger(model_name, "start_id"); const int end_id = reader.GetInteger(model_name, "end_id"); @@ -196,7 +197,8 @@ void llama_example(const INIReader reader) max_input_len, end_id, 1, - "/notebooks/FasterTransformer/examples/cpp/llama/start_ids.csv"); + "/notebooks/tmp/FasterTransformer/examples/cpp/llama/start_ids_real.csv"); + // "/notebooks/tmp/FasterTransformer/examples/cpp/llama/start_ids_real.csv"); int* d_input_ids; @@ -475,7 +477,9 @@ void llama_example(const INIReader reader) if ((i + 1) % (total_output_len) == 0) { outFile << std::endl; } - printf("%5d ", hBuf[i]); + if (i+32 >= outCount) { + printf("%d, ", hBuf[i]); + } // if (i < 10) { // printf("%5d ", hBuf[i]); // } diff --git a/examples/cpp/llama/start_ids.csv b/examples/cpp/llama/start_ids.csv index 612c85964..c3670a1b4 100644 --- a/examples/cpp/llama/start_ids.csv +++ b/examples/cpp/llama/start_ids.csv @@ -1 +1 @@ -1, 18637, 29892, 526, 366, 1136, 455, 2470, 29973, 1815, 366, 5193, 304, 592, 29973 +1, 518, 25580, 29962, 2266, 338, 263, 2566, 10938, 3583, 29876, 29989 diff --git a/examples/cpp/llama/start_ids_real.csv b/examples/cpp/llama/start_ids_real.csv new file mode 100644 index 000000000..a8335ccd7 --- /dev/null +++ b/examples/cpp/llama/start_ids_real.csv @@ -0,0 +1 @@ +1, 518, 25580, 29962, 2266, 338, 263, 2566, 10938, 3583, 29876, 29989, 379, 5005, 29918, 1525, 1254, 3960, 9838, 29903, 29918, 3217, 3904, 5659, 29979, 584, 4810, 3904, 5659, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 15233, 29918, 6248, 518, 20231, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 365, 1299, 518, 383, 3927, 1299, 4514, 1919, 365, 20614, 518, 383, 3927, 1299, 4514, 1919, 349, 7466, 29931, 3235, 29950, 3352, 518, 20231, 4514, 1919, 660, 29965, 1718, 13566, 8895, 29918, 16975, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 16759, 3960, 9838, 29918, 16975, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 7791, 4574, 27266, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 399, 8187, 29918, 29903, 1806, 29965, 8098, 29918, 1525, 15082, 29903, 584, 4810, 3904, 5659, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 29918, 28577, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 21330, 21554, 29918, 29903, 1177, 4741, 29918, 4375, 1254, 29918, 1525, 15082, 3352, 29918, 23487, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29903, 29918, 28577, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 317, 1806, 29965, 8098, 29918, 1525, 15082, 29918, 5813, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 317, 1806, 29965, 8098, 29918, 1525, 15082, 29918, 4219, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 1964, 29918, 23487, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 10014, 2190, 29903, 10403, 13507, 29918, 13875, 1799, 6545, 28541, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 382, 6530, 29907, 29918, 29954, 28902, 1964, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 518, 383, 3927, 1299, 4514, 1919, 29134, 29903, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 383, 3927, 1299, 4514, 1919, 8707, 29911, 1177, 3919, 5746, 29925, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 7534, 29903, 518, 383, 3927, 1299, 4514, 1919, 5012, 7534, 29903, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 383, 3927, 1299, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 349, 4590, 13309, 8098, 518, 383, 3927, 1299, 4514, 891, 382, 6530, 29907, 29918, 29954, 28902, 1964, 29918, 8851, 29923, 29968, 16786, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 8851, 29923, 29968, 518, 383, 3927, 1299, 4514, 1919, 29134, 29903, 29918, 8851, 29923, 29968, 16786, 518, 383, 3927, 1299, 4514, 1919, 8707, 29911, 1177, 3919, 5746, 29925, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 7534, 29903, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 8851, 29923, 29968, 518, 383, 3927, 1299, 4514, 1919, 5012, 7534, 29903, 29918, 8851, 29923, 29968, 16786, 518, 383, 3927, 1299, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 349, 4590, 13309, 8098, 518, 383, 3927, 1299, 4514, 891, 435, 29950, 29965, 29918, 29928, 24943, 8456, 17011, 29918, 3217, 13044, 29918, 29896, 29929, 29918, 29954, 28902, 1964, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 319, 1783, 18474, 518, 383, 3927, 1299, 4514, 1919, 8707, 3738, 29934, 2303, 29928, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 21122, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 7534, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 383, 5690, 29903, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 8098, 29918, 29934, 3040, 518, 383, 3927, 1299, 4514, 1919, 2672, 29907, 1367, 3919, 29918, 29934, 3040, 518, 383, 3927, 1299, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 365, 1299, 518, 383, 3927, 1299, 4514, 1919, 365, 20614, 518, 383, 3927, 1299, 4514, 1919, 341, 8476, 1964, 11937, 29918, 29934, 3040, 518, 383, 3927, 1299, 4514, 1919, 349, 29923, 4590, 1307, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 29923, 4590, 1307, 29918, 18267, 3352, 518, 383, 3927, 1299, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 5195, 3217, 5348, 3352, 518, 383, 3927, 1299, 4514, 1919, 17067, 1254, 4214, 29918, 29934, 3040, 518, 383, 3927, 1299, 4514, 891, 399, 8187, 29918, 7698, 6227, 29979, 29918, 1525, 15082, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29134, 29903, 29918, 29911, 2891, 1964, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29134, 29903, 29918, 29911, 2891, 1964, 29918, 13171, 29918, 29896, 29900, 29900, 29900, 29900, 29900, 518, 383, 3927, 1299, 4514, 1919, 20231, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 5012, 7534, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29903, 29918, 29911, 2891, 1964, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29903, 29918, 29911, 2891, 1964, 29918, 13171, 29918, 29896, 29900, 29900, 29900, 29900, 29900, 518, 383, 3927, 1299, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 10014, 2190, 29903, 10403, 13507, 29918, 13875, 1799, 6545, 28541, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 435, 29950, 29965, 29918, 3217, 13044, 29918, 29896, 29929, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29134, 29918, 11116, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 21122, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 22471, 28483, 1430, 4741, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 383, 5690, 29903, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 365, 1299, 518, 383, 3927, 1299, 4514, 1919, 365, 20614, 518, 383, 3927, 1299, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 306, 29950, 2303, 29918, 3217, 13044, 29918, 29896, 29929, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 11033, 29924, 3235, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 11033, 29924, 3235, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 11033, 29924, 3235, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 15149, 29933, 3352, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 15149, 29933, 3352, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 15149, 29933, 3352, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 350, 3352, 29949, 5348, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 350, 3352, 29949, 5348, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 350, 3352, 29949, 5348, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 7534, 29903, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 5012, 7534, 29903, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 5012, 7534, 29903, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 18340, 7466, 3352, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 18340, 7466, 3352, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 18340, 7466, 3352, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 18340, 29965, 29949, 5348, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 18340, 29965, 29949, 5348, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 18340, 29965, 29949, 5348, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 2672, 29963, 29963, 1430, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 2672, 29963, 29963, 1430, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 2672, 29963, 29963, 1430, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 17723, 29918, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29918, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 29091, 2965, 29965, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 29091, 2965, 29965, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 29091, 2965, 29965, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 2287, 29909, 29918, 27998, 1001, 518, 383, 3927, 1299, 4514, 1919, 323, 2891, 2287, 29909, 29918, 2303, 2190, 518, 383, 3927, 1299, 4514, 1919, 323, 2891, 2287, 29909, 29918, 4897, 13171, 518, 383, 3927, 1299, 4514, 891, 435, 29950, 29965, 29918, 3217, 13044, 29918, 29896, 29929, 29918, 15307, 6304, 29059, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 518, 383, 3927, 1299, 4514, 1919, 29134, 29918, 11116, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 21122, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 22471, 28483, 1430, 4741, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 383, 5690, 29903, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 365, 1299, 518, 383, 3927, 1299, 4514, 1919, 365, 20614, 518, 383, 3927, 1299, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 12279, 29925, 1307, 29918, 6720, 29933, 6227, 11937, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 22471, 28483, 1430, 4741, 518, 383, 3927, 1299, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 10014, 2190, 5550, 8476, 8098, 29918, 11116, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 9609, 29924, 29918, 29928, 7024, 29918, 3217, 13044, 29896, 29929, 29918, 2287, 6040, 6227, 29903, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 4741, 8127, 29928, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 4741, 8127, 29928, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 28657, 11282, 1692, 29928, 29918, 9606, 29909, 20566, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 28657, 11282, 1692, 29928, 29918, 9606, 29909, 20566, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29832, 2303, 29918, 3235, 5607, 8098, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29832, 2303, 29918, 3235, 5607, 8098, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 19578, 1430, 5425, 12064, 29918, 5454, 1525, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 19578, 1430, 5425, 12064, 29918, 5454, 1525, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 29091, 29918, 24815, 1806, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 6058, 29923, 29918, 1430, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 6058, 29923, 29918, 1806, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17067, 1254, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17067, 1254, 3352, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 23487, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 23487, 29903, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 24815, 1806, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 24815, 1806, 18474, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 399, 8187, 29918, 15307, 6304, 29059, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29134, 29903, 29918, 29911, 2891, 1964, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29134, 29903, 29918, 29911, 2891, 1964, 29918, 13171, 29918, 29896, 29900, 29900, 29900, 29900, 29900, 518, 383, 3927, 1299, 4514, 1919, 20231, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 5012, 7534, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29903, 29918, 29911, 2891, 1964, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29903, 29918, 29911, 2891, 1964, 29918, 13171, 29918, 29896, 29900, 29900, 29900, 29900, 29900, 518, 383, 3927, 1299, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 10014, 2190, 29903, 10403, 13507, 29918, 13875, 1799, 6545, 28541, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 7307, 29907, 29918, 29925, 5607, 2965, 29979, 29918, 2303, 3289, 11499, 29903, 584, 4810, 7428, 3919, 29903, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 21122, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 383, 5690, 29903, 29918, 16524, 518, 383, 3927, 1299, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 349, 5607, 2965, 29979, 29918, 1307, 29963, 6670, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 349, 5607, 2965, 29979, 29918, 11116, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 7791, 4574, 4741, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 6850, 8322, 29918, 1254, 4590, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 6850, 3040, 29918, 1367, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 1964, 29918, 19689, 8127, 518, 383, 3927, 1299, 4514, 891, 9609, 29924, 29918, 29928, 7024, 29918, 3217, 13044, 29896, 29929, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 29134, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29134, 29918, 11116, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 22471, 28483, 1430, 4741, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 365, 1299, 518, 383, 3927, 1299, 4514, 1919, 365, 20614, 518, 383, 3927, 1299, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 21947, 29949, 29954, 29918, 29954, 28902, 1964, 29918, 6720, 29933, 6227, 11937, 29918, 1525, 15082, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 402, 1672, 29907, 24422, 29918, 9468, 29918, 19689, 1718, 1529, 29907, 29979, 29918, 3210, 24336, 29918, 13171, 29907, 518, 383, 3927, 1299, 4514, 1919, 17723, 29918, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29918, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 349, 1718, 17557, 29918, 3210, 24336, 29918, 13171, 29907, 518, 383, 3927, 1299, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 390, 2890, 1367, 3919, 25758, 29918, 3210, 24336, 29918, 13171, 29907, 518, 383, 3927, 1299, 4514, 1919, 28081, 29909, 6227, 29918, 9468, 29918, 1525, 22245, 8098, 29918, 3210, 24336, 29918, 13171, 29907, 518, 383, 3927, 1299, 4514, 1919, 27092, 29918, 18166, 2725, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 10014, 2190, 29903, 1806, 29918, 1254, 8098, 29903, 29918, 3210, 24336, 29918, 13171, 29907, 518, 383, 3927, 1299, 4514, 1919, 399, 1955, 29968, 7390, 2477, 2890, 29918, 3210, 24336, 29918, 13171, 29907, 518, 383, 3927, 1299, 4514, 891, 26637, 29918, 3308, 29918, 3217, 13044, 29918, 18267, 29903, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 7534, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 5971, 3308, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 29907, 5005, 13309, 1299, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 29907, 5005, 13309, 1299, 18474, 1177, 22245, 8127, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 22484, 29450, 16786, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 22484, 29450, 16786, 1177, 22245, 8127, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 5971, 3308, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 2965, 23129, 5005, 13309, 1299, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 2965, 23129, 5005, 13309, 1299, 18474, 1177, 22245, 8127, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 2965, 29965, 22484, 29450, 16786, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 2965, 29965, 22484, 29450, 16786, 1177, 22245, 8127, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 405, 11787, 1299, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 405, 11787, 1299, 18474, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 5971, 3308, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 6732, 29963, 3919, 6227, 1299, 1955, 29907, 5005, 13309, 1299, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 6732, 29963, 3919, 6227, 1299, 1955, 29907, 5005, 13309, 1299, 18474, 1177, 22245, 8127, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 6732, 29963, 3919, 6227, 1299, 1955, 22484, 29450, 16786, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 6732, 29963, 3919, 6227, 1299, 1955, 22484, 29450, 16786, 1177, 22245, 8127, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 11794, 4214, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 11794, 4214, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 5971, 3308, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 3267, 1806, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 3267, 1806, 18474, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 5971, 3308, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 1964, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 5971, 3308, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 7307, 29907, 29918, 1525, 15082, 3352, 29918, 29925, 1299, 29902, 3919, 29918, 29902, 3580, 17923, 584, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 518, 383, 3927, 1299, 4514, 1919, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 29928, 1430, 6488, 1177, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 13967, 1001, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 29918, 1164, 10490, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 379, 3267, 29925, 1806, 1964, 29918, 1164, 10490, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 17171, 29928, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 17171, 29928, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 518, 383, 3927, 1299, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 29928, 1430, 6488, 1177, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29918, 3217, 13044, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 13967, 1001, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1001, 29907, 3919, 29918, 9800, 29918, 1177, 29925, 1299, 29902, 3919, 29903, 29918, 29956, 13054, 29918, 3217, 13044, 518, 383, 3927, 1299, 4514, 1919, 349, 1001, 29907, 3919, 29918, 9800, 29918, 1177, 29925, 1299, 29902, 3919, 29903, 29918, 29956, 13054, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1001, 29907, 3919, 29918, 9800, 29918, 1177, 29925, 1299, 29902, 3919, 29903, 29918, 29956, 13054, 29918, 3217, 13044, 29918, 29928, 1430, 6488, 1177, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1001, 29907, 3919, 29918, 9800, 29918, 1177, 29925, 1299, 29902, 3919, 29903, 29918, 29956, 13054, 29918, 3217, 13044, 29918, 13967, 1001, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 3035, 8647, 29918, 3217, 13044, 29918, 6007, 3738, 29934, 2303, 29928, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 3035, 8647, 29918, 3217, 13044, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 3035, 8647, 29918, 3217, 13044, 29918, 29903, 3308, 4162, 1783, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 3035, 8647, 29918, 3217, 13044, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 3217, 13044, 29918, 6007, 3738, 29934, 2303, 29928, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 3217, 13044, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 3217, 13044, 29918, 29903, 3308, 4162, 1783, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 1525, 29963, 5971, 3308, 29918, 28658, 29918, 3035, 10403, 13507, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 3217, 13044, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 317, 6040, 4198, 3352, 29918, 2965, 29965, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 9468, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 317, 6040, 4198, 3352, 29918, 2965, 29965, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 9468, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 317, 6040, 4198, 3352, 29918, 2965, 29965, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 317, 6040, 4198, 3352, 29918, 2965, 29965, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 9468, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 9468, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 3035, 8647, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 9468, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 9468, 29918, 29903, 3308, 4162, 1783, 3352, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 13044, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29925, 3352, 29902, 1299, 29934, 2965, 29918, 29925, 1299, 29902, 3919, 29903, 29918, 29950, 3267, 29925, 1806, 1964, 26664, 3352, 29918, 6007, 3738, 29934, 2303, 29928, 29918, 3217, 13044, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 2692, 29902, 5265, 29999, 8098, 518, 383, 3927, 1299, 4514, 1919, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 29928, 1430, 6488, 1177, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 11033, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 13967, 1001, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 15600, 1806, 2965, 1964, 29918, 1254, 29909, 4198, 4214, 29918, 7068, 1955, 6040, 1692, 29918, 13566, 2965, 5690, 3040, 29928, 29918, 29956, 13054, 1177, 29918, 8851, 29923, 29968, 29918, 6632, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 15600, 1806, 2965, 1964, 29918, 1254, 29909, 4198, 4214, 29918, 7068, 1955, 6040, 1692, 29918, 13566, 2965, 5690, 3040, 29928, 29918, 29956, 13054, 1177, 29918, 8851, 29923, 29968, 29918, 12256, 29918, 1525, 15082, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 15600, 1806, 2965, 1964, 29918, 1254, 29909, 4198, 4214, 29918, 7068, 1955, 6040, 1692, 29918, 13566, 2965, 5690, 3040, 29928, 29918, 29956, 13054, 1177, 29918, 8851, 29923, 29968, 29918, 21143, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 15600, 1806, 2965, 1964, 29918, 1254, 29909, 4198, 4214, 29918, 7068, 1955, 6040, 1692, 29918, 4986, 28658, 29918, 6632, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 15600, 1806, 2965, 1964, 29918, 1254, 29909, 4198, 4214, 29918, 7068, 1955, 6040, 1692, 29918, 4986, 28658, 29918, 12256, 29918, 1525, 15082, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 15600, 1806, 2965, 1964, 29918, 1254, 29909, 4198, 4214, 29918, 7068, 1955, 6040, 1692, 29918, 4986, 28658, 29918, 21143, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 20231, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 17171, 29928, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 17171, 29928, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 2692, 29902, 5265, 29999, 8098, 518, 383, 3927, 1299, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 29928, 1430, 6488, 1177, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 2692, 29902, 5265, 29999, 8098, 29918, 13967, 1001, 1299, 1955, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 317, 6040, 4198, 3352, 29918, 3035, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 29949, 4174, 4897, 2190, 29907, 29979, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 317, 6040, 4198, 3352, 29918, 3035, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29918, 29949, 4174, 4897, 2190, 29907, 29979, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 6850, 3040, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1254, 29909, 4198, 3352, 29918, 3035, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1254, 29909, 4198, 3352, 29918, 3035, 8647, 29918, 2965, 29965, 29918, 29933, 3352, 29903, 29918, 3217, 5348, 10461, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 23526, 29907, 29918, 9606, 1964, 4690, 29918, 18267, 29903, 584, 19937, 29918, 23487, 29918, 18736, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 3217, 13044, 29918, 18267, 29903, 518, 383, 3927, 1299, 4514, 1919, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 383, 5690, 29903, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 6248, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 16999, 4571, 3738, 3352, 29918, 29999, 1783, 29909, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 349, 1001, 29907, 3919, 29918, 24815, 1806, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 27640, 2882, 2190, 29968, 29918, 2287, 6720, 14345, 3301, 29950, 2965, 29903, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 21122, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 383, 5690, 29903, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 365, 1299, 1806, 29965, 2287, 518, 383, 3927, 1299, 4514, 1919, 365, 20614, 1806, 29965, 2287, 518, 383, 3927, 1299, 4514, 1919, 6850, 3040, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 1964, 29918, 16359, 1529, 1307, 29918, 29925, 4590, 13309, 8098, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1529, 1307, 29918, 29925, 4590, 13309, 8098, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 29925, 4590, 13309, 8098, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 7307, 29907, 29918, 1177, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 3217, 13044, 29918, 29896, 29929, 584, 20231, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 1177, 29918, 17171, 29918, 29925, 1783, 518, 383, 3927, 1299, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 1177, 29918, 17171, 29918, 29925, 1783, 29918, 27998, 1001, 29918, 8456, 18783, 518, 383, 3927, 1299, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 1177, 29918, 17171, 29918, 29925, 1783, 29918, 4897, 13171, 29918, 8456, 18783, 518, 383, 3927, 1299, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 27998, 1001, 29918, 8456, 18783, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 29949, 4174, 29965, 2227, 3352, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 2672, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 4897, 13171, 29918, 8456, 18783, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 6850, 3040, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1177, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1177, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 27998, 1001, 29918, 8456, 18783, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1177, 29925, 1299, 29902, 3919, 29918, 29933, 3352, 29903, 29918, 4897, 13171, 29918, 8456, 18783, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 7307, 29907, 29918, 18267, 4214, 584, 20231, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 2672, 6007, 6154, 3308, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 405, 11787, 1299, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 349, 3267, 1806, 18474, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 317, 9295, 29918, 15349, 29918, 2287, 6040, 29902, 20566, 29918, 29950, 3267, 29925, 1806, 1964, 3235, 8098, 29903, 584, 20231, 518, 20231, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29941, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 29091, 29918, 1177, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29091, 29918, 12015, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 27759, 29918, 1525, 15082, 4214, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 13756, 29963, 1177, 4741, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 5195, 29954, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1177, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1177, 29918, 11206, 6720, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1177, 29918, 2965, 29965, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 323, 2891, 1964, 29918, 1177, 29918, 1525, 5550, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 891, 476, 4198, 29918, 3308, 29918, 1525, 4590, 1430, 4214, 29918, 15307, 18521, 29918, 1177, 22245, 13780, 584, 4810, 3904, 5659, 29979, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 6850, 1299, 3308, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 379, 5005, 29918, 1525, 1254, 3960, 9838, 29903, 29918, 23869, 2241, 8895, 584, 4810, 3904, 5659, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 319, 29902, 2241, 8895, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 365, 1299, 518, 383, 3927, 1299, 4514, 1919, 365, 20614, 518, 383, 3927, 1299, 4514, 1919, 349, 7466, 29931, 3235, 29950, 3352, 518, 20231, 4514, 1919, 16759, 3960, 9838, 29918, 16975, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 7791, 4574, 27266, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 23526, 29911, 29918, 3308, 29918, 3217, 13044, 29896, 29929, 584, 29134, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 29134, 29903, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 21122, 29979, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 7534, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 5012, 7534, 29903, 29918, 29903, 1177, 4741, 29918, 15094, 29963, 29918, 28658, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 383, 5690, 29903, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 6850, 3040, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 478, 29950, 29918, 29907, 2190, 29918, 2287, 6040, 29902, 20566, 584, 29134, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 20231, 518, 20231, 4514, 1919, 5012, 7534, 29903, 518, 28019, 13635, 29898, 29941, 29947, 29892, 29871, 29900, 29897, 4514, 1919, 17714, 1964, 4690, 5454, 1525, 29918, 18166, 2725, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29896, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17723, 29941, 29896, 29953, 29953, 29918, 29906, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 1919, 17900, 1254, 29918, 1525, 15082, 3352, 29918, 26516, 518, 16437, 29949, 1307, 2190, 4514, 1919, 17900, 1254, 29918, 14474, 29928, 29918, 6248, 518, 323, 8890, 1254, 19297, 29918, 20321, 29999, 4514, 1919, 13756, 29963, 1177, 4741, 29918, 19713, 518, 21748, 29898, 29896, 29953, 29955, 29955, 29955, 29906, 29896, 29953, 29897, 4514, 891, 317, 9295, 29918, 15349, 29918, 2287, 6040, 29902, 20566, 29918, 8618, 29963, 1177, 4741, 29918, 23487, 29918, 3217, 3904 diff --git a/examples/cpp/multi_gpu_gpt/gpt_example_utils.cc b/examples/cpp/multi_gpu_gpt/gpt_example_utils.cc index fc24fd26f..8557c1c87 100644 --- a/examples/cpp/multi_gpu_gpt/gpt_example_utils.cc +++ b/examples/cpp/multi_gpu_gpt/gpt_example_utils.cc @@ -45,7 +45,7 @@ int read_start_ids(size_t batch_size, int i1 = 0; std::vector tmp_vec; while (std::getline(lineStream, vals, ',')) { - printf("vals: %s\n", vals.c_str()); + // printf("vals: %s\n", vals.c_str()); tmp_vec.push_back(std::stoi(vals)); i1++; } diff --git a/examples/pytorch/bart/utils/ft_encoder.py b/examples/pytorch/bart/utils/ft_encoder.py index 968119c77..0170818de 100644 --- a/examples/pytorch/bart/utils/ft_encoder.py +++ b/examples/pytorch/bart/utils/ft_encoder.py @@ -16,7 +16,7 @@ import torch.nn as nn import torch.distributed as dist import numpy as np -from transformers import MBartForConditionalGeneration, BartModel +from transformers import MBartForConditionalGeneration, BartModel, LlamaForCausalLM class FTBartEncoderWeight(object): def __init__( diff --git a/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h b/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h index 1e32d3d22..5da6823ba 100644 --- a/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h +++ b/src/fastertransformer/kernels/decoder_masked_multihead_attention_utils.h @@ -1326,7 +1326,7 @@ inline __device__ void zero(T& dst) inline __device__ float2 rotary_embedding_coefficient(const int zid, const int rot_embed_dim, const float rope_theta, const float t_step) { - const float inv_freq = t_step / pow(rope_theta, zid / (float)rot_embed_dim); + const float inv_freq = t_step / pow(1000000.f, zid / (float)rot_embed_dim); return {cos(inv_freq), sin(inv_freq)}; } diff --git a/src/fastertransformer/kernels/llama/decoder_masked_groupedquery_attention/decoder_masked_groupedquery_attention_128.cu b/src/fastertransformer/kernels/llama/decoder_masked_groupedquery_attention/decoder_masked_groupedquery_attention_128.cu index 3a673d3e3..2c972b411 100644 --- a/src/fastertransformer/kernels/llama/decoder_masked_groupedquery_attention/decoder_masked_groupedquery_attention_128.cu +++ b/src/fastertransformer/kernels/llama/decoder_masked_groupedquery_attention/decoder_masked_groupedquery_attention_128.cu @@ -25,10 +25,13 @@ //////////////////////////////////////////////////////////////////////////////////////////////////// #define MGQA_LAUNCH_KERNEL( \ - T, Dh, Dh_MAX, THDS_PER_KEY, THDS_PER_VALUE, THDS_PER_BLOCK, HAS_BEAMS, stream) \ - size_t smem_sz = mmha::smem_size_in_bytes(params, THDS_PER_VALUE, THDS_PER_BLOCK); \ + T, Dh, Dh_MAX, THDS_PER_KEY, THDS_PER_VALUE, THDS_PER_BLOCK, HAS_BEAMS, stream) \ + size_t smem_sz = mmha::smem_size_in_bytes(params, THDS_PER_VALUE, THDS_PER_BLOCK); \ dim3 grid(params.num_heads, params.batch_size); \ - mmha::masked_groupedquery_attention_kernel, \ + cudaFuncAttributeMaxDynamicSharedMemorySize, smem_sz); \ + mmha::masked_groupedquery_attention_kernel::allocateBuffer(size_t batch_size, size_t seq FT_LOG_DEBUG(__PRETTY_FUNCTION__); // const auto type_size = int8_mode_ == 2 ? sizeof(int8_t) : sizeof(T); // NOTE (perkzz): use sizeof(T) here for cutlass int8 kernels. + printf("local_hidden_units_: %d\n", local_hidden_units_); const auto type_size = sizeof(T); + printf("%ld\n", type_size * 30 * batch_size * seq_len * local_hidden_units_); + printf("%ld\n", type_size * 3 * batch_size * seq_len * local_hidden_units_); qkv_buf_ = (T*)allocator_->reMalloc(qkv_buf_, type_size * 3 * batch_size * seq_len * local_hidden_units_, true); + printf("qkv_buf_\n"); if (local_kv_head_num_ != local_head_num_) { size_t local_qkv_size = local_hidden_units_ + 2 * local_kv_head_num_ * size_per_head_; qkv_buf_tmp_ = (T*)allocator_->reMalloc(qkv_buf_tmp_, type_size * batch_size * seq_len * local_qkv_size, true); @@ -737,18 +741,25 @@ void LlamaContextAttentionLayer::allocateBuffer(size_t batch_size, size_t seq qkv_buf_tmp_ = qkv_buf_; } q_buf_2_ = (T*)allocator_->reMalloc(q_buf_2_, sizeof(T) * batch_size * seq_len * 3 * local_hidden_units_, true); + printf("q_buf_2_\n"); k_buf_2_ = q_buf_2_ + batch_size * seq_len * local_hidden_units_; v_buf_2_ = k_buf_2_ + batch_size * seq_len * local_hidden_units_; // save memory usage when using fmha if (allocate_qk_buf) { + printf("allocate_qk_buf\n"); + auto x = sizeof(T) * batch_size * local_head_num_ * seq_len * seq_len; + printf("%ld\n", x); qk_buf_ = (T*)allocator_->reMalloc(qk_buf_, sizeof(T) * batch_size * local_head_num_ * seq_len * seq_len, true); } else { allocator_->free((void**)(&qk_buf_)); } + printf("qkv_buf_2_\n"); qkv_buf_2_ = (T*)allocator_->reMalloc(qkv_buf_2_, sizeof(T) * batch_size * seq_len * local_hidden_units_, true); + printf("qkv_buf_2_\n"); qkv_buf_3_ = (T*)allocator_->reMalloc(qkv_buf_3_, type_size * batch_size * seq_len * local_hidden_units_, true); + printf("qkv_buf_3_\n"); if (is_qk_buf_float_ == true) { if (allocate_qk_buf) { diff --git a/src/fastertransformer/models/llama/Llama.cc b/src/fastertransformer/models/llama/Llama.cc index 82630079d..269087216 100644 --- a/src/fastertransformer/models/llama/Llama.cc +++ b/src/fastertransformer/models/llama/Llama.cc @@ -762,6 +762,23 @@ void Llama::forward(std::unordered_map* output_ten gpt_context_decoder_->forward( &decoder_output_tensors, &decoder_input_tensors, &gpt_weights->decoder_layer_weights); sync_check_cuda_error(); + { + T* buf; + int st = 1; + for (int k=0; k::forward(std::unordered_map* output_ten decoder_output_buf_ + hidden_units_offset}}, {"key_cache", Tensor{MEMORY_GPU, data_type, self_k_cache_shape, key_cache_}}, {"value_cache", Tensor{MEMORY_GPU, data_type, self_v_cache_shape, value_cache_}}}; + // for (int i=0; i< self_k_cache_shape.size(); i++) { + // printf("self_k_cache_shape: %d\n", self_k_cache_shape[i]); + // } + // for (int i=0; i< self_v_cache_shape.size(); i++) { + // printf("self_v_cache_shape: %d\n", self_v_cache_shape[i]); + // } gpt_decoder_->forward( &decoder_output_tensors, &decoder_input_tensors, &gpt_weights->decoder_layer_weights); } + if (step <= max_input_length + 1) { + T* buf; + int st = 1; + for (int k=0; k::forward(std::unordered_map* output_ten stream_); sync_check_cuda_error(); + // if (step == max_input_length) { + // T* buf; + // int st = hidden_units_; + // buf = new T[st]; + // cudaMemcpy(buf, normed_decoder_output_buf_, sizeof(T) * st, cudaMemcpyDeviceToHost); + // printf("normed_decoder_output_buf_ at step: %d\n", step); + // for (int i=0; i::forward(std::unordered_map* output_ten } dynamic_decode_output_tensors.insert(*t); } - + if (step == max_input_length && 0) { + float* buf; + int st = vocab_size_padded_; + buf = new float[st]; + cudaMemcpy(buf, logits_buf_, sizeof(float) * st, cudaMemcpyDeviceToHost); + printf("logits_buf_ at step: %d\n", step); + for (int i=0; i<10; i++) { + printf("%f ", double(buf[i])); + } + printf("\n"); + } dynamic_decode_layer_->forward(&dynamic_decode_output_tensors, &dynamic_decode_input_tensors); *generation_should_stop_ &= subbatch_should_stop; } diff --git a/src/fastertransformer/models/llama/LlamaContextDecoder.cc b/src/fastertransformer/models/llama/LlamaContextDecoder.cc index 59919e9d4..0d7f1e515 100644 --- a/src/fastertransformer/models/llama/LlamaContextDecoder.cc +++ b/src/fastertransformer/models/llama/LlamaContextDecoder.cc @@ -593,6 +593,23 @@ void LlamaContextDecoder::forward(std::unordered_map* request_batch_size, hidden_units_, stream_); + + // { + // T* buf; + // int st = seq_len*8192; + // buf = new T[st]; + // cudaMemcpy(buf, output_tensors->at("decoder_output").getPtr(), sizeof(T) * st, cudaMemcpyDeviceToHost); + // printf("decoder_output at step\n"); + // for (int s=0; s::forward(std::unordered_map* Tensor& v_cache = output_tensors->at("value_cache"); std::vector self_k_cache_size; self_k_cache_size.push_back(local_batch_size); + size_t a = 1; + int b = 1; for (auto t = k_cache.shape.begin() + 2; t != k_cache.shape.end(); ++t) { self_k_cache_size.push_back(*t); } + for (auto t = k_cache.shape.begin(); t != k_cache.shape.end(); ++t) { + a *= *t; + b *= *t; + } + // printf("a b: %ld %d\n", a, b); std::vector self_v_cache_size; self_v_cache_size.push_back(local_batch_size); for (auto t = v_cache.shape.begin() + 2; t != v_cache.shape.end(); ++t) { @@ -297,6 +304,7 @@ void LlamaDecoder::forward(std::unordered_map* ite_cache_offset *= *t; } cache_offset += ite_cache_offset; + // printf("cache_offset %ld ite_cache_offset %ld\n", cache_offset, ite_cache_offset); TensorMap self_attention_output_tensors{ {"hidden_features", Tensor{MEMORY_GPU, data_type, {local_batch_size, hidden_units_}, self_attn_output_}},