From 3fe16df454a72566555198770ab5f3e15788d518 Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Wed, 19 Nov 2025 23:35:57 +0000 Subject: [PATCH 1/9] squash --- repro_attention_dp.py | 1698 +++++++++++++++++ tpu_inference/layers/common/sharding.py | 14 +- tpu_inference/layers/vllm/fused_moe.py | 135 +- .../layers/vllm/quantization/common.py | 28 +- .../layers/vllm/quantization/unquantized.py | 24 +- tpu_inference/layers/vllm/sharding.py | 11 +- 6 files changed, 1841 insertions(+), 69 deletions(-) create mode 100644 repro_attention_dp.py diff --git a/repro_attention_dp.py b/repro_attention_dp.py new file mode 100644 index 000000000..584e676a3 --- /dev/null +++ b/repro_attention_dp.py @@ -0,0 +1,1698 @@ +""" +Reproduction script to compare attention kernel performance and HLO +with and without data parallelism. + +Usage: + python repro_attention_dp.py +""" + +import argparse +import time +from pathlib import Path +import numpy as np + +import jax +import jax.numpy as jnp +from jax.sharding import Mesh, PartitionSpec as P +from jax import NamedSharding, shard_map +from tpu_inference.layers.vllm.attention import _jax_attn_func +# import sys +# sys.path.insert(0, '/home/wenxindong_google_com/tpu-inference') + +from tpu_inference.kernels.ragged_paged_attention.v3.kernel_hd64 import ( + get_kv_cache_shape as get_kv_cache_shape_h64, +) +from tpu_inference.kernels.ragged_paged_attention.v3.kernel import get_kv_cache_shape +from tpu_inference.layers.common.sharding import ShardingAxisName +from tpu_inference.layers.common.attention_metadata import AttentionMetadata + + +def create_test_inputs( + batch_size: int = 256, + max_num_tokens: int = 256, + num_q_heads: int = 32, + num_kv_heads: int = 8, + head_dim: int = 64, + page_size: int = 256, + total_num_pages: int = 8282, + dp_size: int = 1, +): + + total_num_pages = total_num_pages * dp_size + # Q, K, V tensors + q = jnp.concatenate([jax.random.normal( + jax.random.PRNGKey(0), + (max_num_tokens, num_q_heads, head_dim), + dtype=jnp.bfloat16, + ) for i in range(dp_size)], axis=0) + k = jnp.concatenate([jax.random.normal( + jax.random.PRNGKey(0), + (max_num_tokens, num_kv_heads, head_dim), + dtype=jnp.bfloat16, + ) for i in range(dp_size)], axis=0) + v = jnp.concatenate([jax.random.normal( + jax.random.PRNGKey(0), + (max_num_tokens, num_kv_heads, head_dim), + dtype=jnp.bfloat16, + ) for i in range(dp_size)], axis=0) + + # KV cache + get_kv_cache_shape_fn = get_kv_cache_shape_h64 if head_dim == 64 else get_kv_cache_shape + kv_cache_shape = get_kv_cache_shape_fn( + total_num_pages, + page_size, + num_kv_heads, + head_dim, + jnp.bfloat16, + ) + kv_cache = jax.random.normal(jax.random.PRNGKey(0), kv_cache_shape, dtype=jnp.bfloat16) + + + # Actual seq_lens from the log for step 849 + seq_lens_rank0 = [2897, 2896, 2896, 2896, 2895, 2895, 2895, 2895, 2894, 2894, 2894, 2894, 2893, 2893, + 2893, 2893, 2892, 2892, 2892, 2892, 2891, 2891, 2891, 2891, 2890, 2890, 2890, 2890, + 2889, 2889, 2889, 2889, 2888, 2888, 2888, 2888, 2887, 2887, 2887, 2887, 2886, 2886, + 2886, 2886, 2885, 2885, 2885, 2885, 2884, 2884, 2884, 2884, 2883, 2883, 2883, 2883, + 2882, 2882, 2882, 2882, 2881, 2881, 2881, 2881, 2880, 2880, 2880, 2880, 2879, 2879, + 2879, 2879, 2878, 2878, 2878, 2878, 2877, 2877, 2877, 2877, 2876, 2876, 2876, 2876, + 2875, 2875, 2875, 2875, 2874, 2874, 2874, 2874, 2873, 2873, 2873, 2873, 2872, 2872, + 2872, 2872, 2871, 2871, 2871, 2871, 2870, 2870, 2870, 2870, 2869, 2869, 2869, 2869, + 2868, 2868, 2868, 2868, 2867, 2867, 2867, 2867, 2866, 2866, 2866, 2866, 2865, 2865, + 2865, 2865, 2864, 2864, 2864, 2863, 2863, 2863, 2863, 2862, 2862, 2862, 2862, 2861, + 2861, 2861, 2861, 2860, 2860, 2860, 2860, 2859, 2859, 2859, 2859, 2858, 2858, 2858, + 2858, 2857, 2857, 2857, 2857, 2856, 2856, 2856, 2856, 2855, 2855, 2855, 2855, 2854, + 2854, 2854, 2854, 2853, 2853, 2853, 2853, 2852, 2852, 2852, 2852, 2851, 2851, 2851, + 2850, 2850, 2850, 2850, 2849, 2849, 2849, 2849, 2848, 2848, 2848, 2848, 2847, 2847, + 2847, 2847, 2846, 2846, 2846, 2846, 2845, 2845, 2845, 2845, 2844, 2844, 2844, 2844, + 2843, 2843, 2843, 2843, 2842, 2842, 2842, 2842, 2841, 2841, 2841, 2841, 2840, 2840, + 2840, 2839, 2839, 2839, 2839, 2838, 2838, 2838, 2838, 2837, 2837, 2837, 2837, 2836, + 2836, 2836, 2836, 2835, 2835, 2835, 2835, 2834, 2834, 2834, 2834, 2833, 2833, 2833, + 2833, 2832, 2832, 2832] + + seq_lens_rank1 = [2896, 2896, 2896, 2896, 2895, 2895, 2895, 2894, 2894, 2894, 2894, 2893, 2893, 2893, + 2893, 2892, 2892, 2892, 2892, 2891, 2891, 2891, 2891, 2890, 2890, 2890, 2890, 2889, + 2889, 2889, 2889, 2888, 2888, 2888, 2888, 2887, 2887, 2887, 2887, 2886, 2886, 2886, + 2886, 2885, 2885, 2885, 2885, 2884, 2884, 2884, 2884, 2883, 2883, 2883, 2883, 2882, + 2882, 2882, 2882, 2881, 2881, 2881, 2881, 2880, 2880, 2880, 2880, 2879, 2879, 2879, + 2879, 2878, 2878, 2878, 2878, 2877, 2877, 2877, 2877, 2876, 2876, 2876, 2876, 2875, + 2875, 2875, 2875, 2874, 2874, 2874, 2874, 2873, 2873, 2873, 2873, 2872, 2872, 2872, + 2872, 2871, 2871, 2871, 2871, 2870, 2870, 2870, 2870, 2869, 2869, 2869, 2869, 2868, + 2868, 2868, 2868, 2867, 2867, 2867, 2867, 2866, 2866, 2866, 2866, 2865, 2865, 2865, + 2865, 2864, 2864, 2864, 2863, 2863, 2863, 2863, 2862, 2862, 2862, 2862, 2861, 2861, + 2861, 2861, 2860, 2860, 2860, 2860, 2859, 2859, 2859, 2859, 2858, 2858, 2858, 2858, + 2857, 2857, 2857, 2857, 2856, 2856, 2856, 2856, 2855, 2855, 2855, 2855, 2854, 2854, + 2854, 2854, 2853, 2853, 2853, 2853, 2852, 2852, 2852, 2852, 2851, 2851, 2851, 2851, + 2850, 2850, 2850, 2849, 2849, 2849, 2849, 2848, 2848, 2848, 2848, 2847, 2847, 2847, + 2847, 2846, 2846, 2846, 2846, 2845, 2845, 2845, 2845, 2844, 2844, 2844, 2844, 2843, + 2843, 2843, 2843, 2842, 2842, 2842, 2842, 2841, 2841, 2841, 2841, 2840, 2840, 2840, + 2839, 2839, 2839, 2839, 2838, 2838, 2838, 2838, 2837, 2837, 2837, 2837, 2836, 2836, + 2836, 2836, 2835, 2835, 2835, 2835, 2834, 2834, 2834, 2834, 2833, 2833, 2833, 2833, + 2832, 2832, 2832, 2831] + # from step 1698 + seq_len_no_dp = [3745, 3745, 3743, 3743, 3743, 3742, 3742, 3742, 3742, 3741, 3741, + 3741, 3741, 3740, 3740, 3740, 3740, 3739, 3739, 3739, 3739, 3738, + 3738, 3738, 3738, 3737, 3737, 3737, 3737, 3736, 3736, 3736, 3736, + 3735, 3735, 3735, 3735, 3734, 3734, 3734, 3734, 3733, 3733, 3733, + 3733, 3732, 3732, 3732, 3732, 3731, 3731, 3731, 3731, 3730, 3730, + 3730, 3730, 3729, 3729, 3729, 3729, 3728, 3728, 3728, 3728, 3727, + 3727, 3727, 3727, 3726, 3726, 3726, 3726, 3725, 3725, 3725, 3725, + 3724, 3724, 3724, 3724, 3723, 3723, 3723, 3723, 3722, 3722, 3722, + 3722, 3721, 3721, 3721, 3721, 3720, 3720, 3720, 3720, 3719, 3719, + 3719, 3719, 3718, 3718, 3718, 3718, 3717, 3717, 3717, 3717, 3716, + 3716, 3716, 3716, 3715, 3715, 3715, 3715, 3714, 3714, 3714, 3714, + 3713, 3713, 3713, 3713, 3712, 3712, 3712, 3712, 3711, 3711, 3711, + 3710, 3710, 3710, 3710, 3709, 3709, 3709, 3709, 3708, 3708, 3708, + 3708, 3707, 3707, 3707, 3707, 3706, 3706, 3706, 3706, 3705, 3705, + 3705, 3705, 3704, 3704, 3704, 3704, 3703, 3703, 3703, 3703, 3702, + 3702, 3702, 3702, 3701, 3701, 3701, 3701, 3700, 3700, 3700, 3700, + 3699, 3699, 3699, 3699, 3698, 3698, 3698, 3697, 3697, 3697, 3697, + 3696, 3696, 3696, 3696, 3695, 3695, 3695, 3695, 3694, 3694, 3694, + 3694, 3693, 3693, 3693, 3693, 3692, 3692, 3692, 3692, 3691, 3691, + 3691, 3691, 3690, 3690, 3690, 3690, 3689, 3689, 3689, 3689, 3688, + 3688, 3688, 3687, 3687, 3687, 3687, 3686, 3686, 3686, 3686, 3685, + 3685, 3685, 3685, 3684, 3684, 3684, 3684, 3683, 3683, 3683, 3683, + 3682, 3682, 3682, 3682, 3681, 3681, 3681, 3681, 3680, 3680, 3680, + 3680, 3679, 3679] + + # query_start_loc is just [0, 1, 2, ..., 256] for both ranks + query_start_loc_per_rank = list(range(257)) + + # Construct kv_lens and cu_q_lens based on dp_size + if dp_size == 1: + kv_lens = jnp.array(seq_len_no_dp, dtype=jnp.int32) + cu_q_lens = jnp.array(query_start_loc_per_rank, dtype=jnp.int32) + elif dp_size == 2: + kv_lens = jnp.array(seq_lens_rank0 + seq_lens_rank1, dtype=jnp.int32) + cu_q_lens = jnp.array(query_start_loc_per_rank + query_start_loc_per_rank, dtype=jnp.int32) + + # Distribution: [0, 0, 256] per rank (all decode mode) + distribution_list = [] + for _ in range(dp_size): + distribution_list.extend([0, 0, batch_size]) + distribution = jnp.array(distribution_list, dtype=jnp.int32) + + + page_indices_rank0 = [[ 1, 2, 3, 4, 5, 6, 7 , 8, 9, 2305, 2561, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 10, 11, 12, 13, 14, 15, 16, 17, 42, 2306, 2562, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 18, 19, 20, 21, 22, 23, 24, 25, 43, 2307, 2563, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 26, 27, 28, 29, 30, 31, 32, 33, 44, 2308, 2564, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 34, 35, 36, 37, 38, 39, 40, 41, 77, 2309, 2565, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 45, 46, 47, 48, 49, 50, 51, 52, 78, 2310, 2566, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 53, 54, 55, 56, 57, 58, 59, 60, 79, 2311, 2567, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 61, 62, 63, 64, 65, 66, 67, 68, 80, 2312, 2568, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 69, 70, 71, 72, 73, 74, 75, 76, 113, 2313, 2569, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 81, 82, 83, 84, 85, 86 , 87 , 88, 114, 2314, 2570, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 89 , 90 , 91 , 92, 93, 94, 95, 96, 115, 2315, 2571, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 97 , 98 , 99, 100, 101, 102, 103, 104, 116, 2316, 2572, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 105, 106, 107, 108, 109, 110, 111, 112, 149, 2317, 2573, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 117, 118, 119, 120, 121, 122, 123, 124, 150, 2318, 2574, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 125, 126, 127, 128, 129, 130, 131, 132, 151, 2319, 2575, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 133, 134, 135, 136, 137, 138, 139, 140, 152, 2320, 2576, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 141, 142, 143, 144, 145, 146, 147, 148, 185, 2321, 2577, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 153, 154, 155, 156, 157, 158, 159, 160, 186, 2322, 2578, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 161, 162, 163, 164, 165, 166, 167, 168, 187, 2323, 2579, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 169, 170, 171, 172, 173, 174, 175, 176, 188, 2324, 2580, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 177, 178, 179, 180, 181, 182, 183, 184, 221, 2325, 2581, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 189, 190, 191, 192, 193, 194, 195, 196, 222, 2326, 2582, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 197, 198, 199, 200, 201, 202, 203, 204, 223, 2327, 2583, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 205, 206, 207, 208, 209, 210, 211, 212, 224, 2328, 2584, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 213, 214, 215, 216, 217, 218, 219, 220, 257, 2329, 2585, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 225, 226, 227, 228, 229, 230, 231, 232, 258, 2330, 2586, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 233, 234, 235, 236, 237, 238, 239, 240, 259, 2331, 2587, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 241, 242, 243, 244, 245, 246, 247, 248, 260, 2332, 2588, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 249, 250, 251, 252, 253, 254, 255, 256, 293, 2333, 2589, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 261, 262, 263, 264, 265, 266, 267, 268, 294, 2334, 2590, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 269, 270, 271, 272, 273, 274, 275, 276, 295, 2335, 2591, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 277, 278, 279, 280, 281, 282, 283, 284, 296, 2336, 2592, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 285, 286, 287, 288, 289, 290, 291, 292, 329, 2337, 2593, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 297, 298, 299, 300, 301, 302, 303, 304, 330, 2338, 2594, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 305, 306, 307, 308, 309, 310, 311, 312, 331, 2339, 2595, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 313, 314, 315, 316, 317, 318, 319, 320, 332, 2340, 2596, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 321, 322, 323, 324, 325, 326, 327, 328, 365, 2341, 2597, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 333, 334, 335, 336, 337, 338, 339, 340, 366, 2342, 2598, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 341, 342, 343, 344, 345, 346, 347, 348, 367, 2343, 2599, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 349, 350, 351, 352, 353, 354, 355, 356, 368, 2344, 2600, 2856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 357, 358, 359, 360, 361, 362, 363, 364, 401, 2345, 2601, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 369, 370, 371, 372, 373, 374, 375, 376, 402, 2346, 2602, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 377, 378, 379, 380, 381, 382, 383, 384, 403, 2347, 2603, 2859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 385, 386, 387, 388, 389, 390, 391, 392, 404, 2348, 2604, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 393, 394, 395, 396, 397, 398, 399, 400, 436, 2349, 2605, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 405, 406, 407, 408, 409, 410, 411, 412, 437, 2350, 2606, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 413, 414, 415, 416, 417, 418, 419, 420, 438, 2351, 2607, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 421, 422, 423, 424, 425, 426, 427, 428, 439, 2352, 2608, 2864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 429, 430, 431, 432, 433, 434, 435, 440, 472, 2353, 2609, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 441, 442, 443, 444, 445, 446, 447, 448, 473, 2354, 2610, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 449, 450, 451, 452, 453, 454, 455, 456, 474, 2355, 2611, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 457, 458, 459, 460, 461, 462, 463, 464, 475, 2356, 2612, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 465, 466, 467, 468, 469, 470, 471, 476, 508, 2357, 2613, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 477, 478, 479, 480, 481, 482, 483, 484, 509, 2358, 2614, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 485, 486, 487, 488, 489, 490, 491, 492, 510, 2359, 2615, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 493, 494, 495, 496, 497, 498, 499, 500, 511, 2360, 2616, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 501, 502, 503, 504, 505, 506, 507, 512, 544, 2361, 2617, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 513, 514, 515, 516, 517, 518, 519, 520, 545, 2362, 2618, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 521, 522, 523, 524, 525, 526, 527, 528, 546, 2363, 2619, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 529, 530, 531, 532, 533, 534, 535, 536, 547, 2364, 2620, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 537, 538, 539, 540, 541, 542, 543, 548, 580, 2365, 2621, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 549, 550, 551, 552, 553, 554, 555, 556, 581, 2366, 2622, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 557, 558, 559, 560, 561, 562, 563, 564, 582, 2367, 2623, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 565, 566, 567, 568, 569, 570, 571, 572, 583, 2368, 2624, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 573, 574, 575, 576, 577, 578, 579, 584, 615, 2369, 2625, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 585, 586, 587, 588, 589, 590, 591, 592, 616, 2370, 2626, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 593, 594, 595, 596, 597, 598, 599, 600, 617, 2371, 2627, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 601, 602, 603, 604, 605, 606, 607, 608, 618, 2372, 2628, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 609, 610, 611, 612, 613, 614, 619, 620, 651, 2373, 2629, 2885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 621, 622, 623, 624, 625, 626, 627, 628, 652, 2374, 2630, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 629, 630, 631, 632, 633, 634, 635, 636, 653, 2375, 2631, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 637, 638, 639, 640, 641, 642, 643, 644, 654, 2376, 2632, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 645, 646, 647, 648, 649, 650, 655, 656, 687, 2377, 2633, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 657, 658, 659, 660, 661, 662, 663, 664, 688, 2378, 2634, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 665, 666, 667, 668, 669, 670, 671, 672, 689, 2379, 2635, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 673, 674, 675, 676, 677, 678, 679, 680, 690, 2380, 2636, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 681, 682, 683, 684, 685, 686, 691, 692, 723, 2381, 2637, 2893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 693, 694, 695, 696, 697, 698, 699, 700, 724, 2382, 2638, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 701, 702, 703, 704, 705, 706, 707, 708, 725, 2383, 2639, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 709, 710, 711, 712, 713, 714, 715, 716, 726, 2384, 2640, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 717, 718, 719, 720, 721, 722, 727, 728, 758, 2385, 2641, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 729, 730, 731, 732, 733, 734, 735, 736, 759, 2386, 2642, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 737, 738, 739, 740, 741, 742, 743, 744, 760, 2387, 2643, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 745, 746, 747, 748, 749, 750, 751, 752, 761, 2388, 2644, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 753, 754, 755, 756, 757, 762, 763, 764, 794, 2389, 2645, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 765, 766, 767, 768, 769, 770, 771, 772, 795, 2390, 2646, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 773, 774, 775, 776, 777, 778, 779, 780, 796, 2391, 2647, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 781, 782, 783, 784, 785, 786, 787, 788, 797, 2392, 2648, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 789, 790, 791, 792, 793, 798, 799, 800, 830, 2393, 2649, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 801, 802, 803, 804, 805, 806, 807, 808, 831, 2394, 2650, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 809, 810, 811, 812, 813, 814, 815, 816, 832, 2395, 2651, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 817, 818, 819, 820, 821, 822, 823, 824, 833, 2396, 2652, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 825, 826, 827, 828, 829, 834, 835, 836, 865, 2397, 2653, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 837, 838, 839, 840, 841, 842, 843, 844, 866, 2398, 2654, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 845, 846, 847, 848, 849, 850, 851, 852, 867, 2399, 2655, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 853, 854, 855, 856, 857, 858, 859, 860, 868, 2400, 2656, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 861, 862, 863, 864, 869, 870, 871, 872, 901, 2401, 2657, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 873, 874, 875, 876, 877, 878, 879, 880, 902, 2402, 2658, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 881, 882, 883, 884, 885, 886, 887, 888, 903, 2403, 2659, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 889, 890, 891, 892, 893, 894, 895, 896, 904, 2404, 2660, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 897, 898, 899, 900, 905, 906, 907, 908, 936, 2405, 2661, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 909, 910, 911, 912, 913, 914, 915, 916, 937, 2406, 2662, 2918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 917, 918, 919, 920, 921, 922, 923, 924, 938, 2407, 2663, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 925, 926, 927, 928, 929, 930, 931, 932, 939, 2408, 2664, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 933, 934, 935, 940, 941, 942, 943, 944, 972, 2409, 2665, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 945, 946, 947, 948, 949, 950, 951, 952, 973, 2410, 2666, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 953, 954, 955, 956, 957, 958, 959, 960, 974, 2411, 2667, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 961, 962, 963, 964, 965, 966, 967, 968, 975, 2412, 2668, 2924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 969, 970, 971, 976, 977, 978, 979, 980, 1008, 2413, 2669, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 981, 982, 983, 984, 985, 986, 987, 988, 1009, 2414, 2670, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 989, 990, 991, 992, 993, 994, 995, 996, 1010, 2415, 2671, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 997, 998, 999, 1000, 1001, 1002,1003, 1004, 1011, 2416, 2672, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1005, 1006, 1007, 1012,1013, 1014, 1015, 1016, 1043, 2417, 2673, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1017, 1018, 1019, 1020, 1021, 1022,1023, 1024, 1044, 2418, 2674, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032,1045, 2419, 2675, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1046, 2420, 2676, 2932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1041, 1042,1047, 1048, 1049, 1050, 1051, 1052,1079, 2421, 2677, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1080, 2422, 2678, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1061, 1062,1063, 1064, 1065, 1066, 1067, 1068, 1081, 2423, 2679, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1069, 1070, 1071, 1072,1073, 1074, 1075, 1076, 1082, 2424, 2680, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1077, 1078, 1083, 1084, 1085, 1086, 1087, 1088, 1114, 2425, 2681, 2937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1089, 1090, 1091, 1092,1093, 1094, 1095, 1096, 1115, 2426, 2682, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1097, 1098, 1099, 1100, 1101, 1102,1103, 1104, 1116, 2427, 2683, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112,1117, 2428, 2684, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1113, 1118, 1119, 1120, 1121, 1122,1123, 1124, 1150, 2429, 2685, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132,1151, 2430, 2686, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1152, 2431, 2687, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1141, 1142,1143, 1144, 1145, 1146, 1147, 1148, 1153, 2432, 2688, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1149, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1185, 2433, 2689, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1161, 1162,1163, 1164, 1165, 1166, 1167, 1168, 1186, 2434, 2690, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1169, 1170, 1171, 1172,1173, 1174, 1175, 1176, 1187, 2435, 2691, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1177, 1178, 1179, 1180, 1181, 1182,1183, 1184, 1220, 2436, 2692, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1188, 1189, 1190, 1191, 1192,1193, 1194, 1195, 1221, 2437, 2693, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1196, 1197, 1198, 1199, 1200, 1201, 1202,1203, 1222, 2438, 2694, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1223, 2439, 2695, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1212,1213, 1214, 1215, 1216, 1217, 1218, 1219, 1255, 2440, 2696, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1256, 2441, 2697, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1232,1233, 1234, 1235, 1236, 1237, 1238, 1239, 1257, 2442, 2698, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1240, 1241, 1242,1243, 1244, 1245, 1246, 1247, 1258, 2443, 2699, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1248, 1249, 1250, 1251, 1252,1253, 1254, 1259, 1291, 2444, 2700, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1260, 1261, 1262,1263, 1264, 1265, 1266, 1267, 1292, 2445, 2701, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1268, 1269, 1270, 1271, 1272,1273, 1274, 1275, 1293, 2446, 2702, 2958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1276, 1277, 1278, 1279, 1280, 1281, 1282,1283, 1294, 2447, 2703, 2959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1284, 1285, 1286, 1287, 1288, 1289, 1290, 1295, 1326, 2448, 2704, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1296, 1297, 1298, 1299, 1300, 1301, 1302,1303, 1327, 2449, 2705, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1328, 2450, 2706, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1312,1313, 1314, 1315, 1316, 1317, 1318, 1319, 1329, 2451, 2707, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1320, 1321, 1322,1323, 1324, 1325, 1330, 1331, 1362, 2452, 2708, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1332,1333, 1334, 1335, 1336, 1337, 1338, 1339, 1363, 2453, 2709, 2965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1340, 1341, 1342,1343, 1344, 1345, 1346, 1347, 1364, 2454, 2710, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1348, 1349, 1350, 1351, 1352,1353, 1354, 1355, 1365, 2455, 2711, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1356, 1357, 1358, 1359, 1360, 1361, 1366, 1367, 1397, 2456, 2712, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1368, 1369, 1370, 1371, 1372,1373, 1374, 1375, 1398, 2457, 2713, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1376, 1377, 1378, 1379, 1380, 1381, 1382,1383, 1399, 2458, 2714, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1400, 2459, 2715, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1392,1393, 1394, 1395, 1396, 1401, 1402,1403, 1432, 2460, 2716, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1433, 2461, 2717, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1412,1413, 1414, 1415, 1416, 1417, 1418, 1419, 1434, 2462, 2718, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1420, 1421, 1422,1423, 1424, 1425, 1426, 1427, 1435, 2463, 2719, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1428, 1429, 1430, 1431, 1436, 1437, 1438, 1439, 1468, 2464, 2720, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1440, 1441, 1442,1443, 1444, 1445, 1446, 1447, 1469, 2465, 2721, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1448, 1449, 1450, 1451, 1452,1453, 1454, 1455, 1470, 2466, 2722, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1456, 1457, 1458, 1459, 1460, 1461, 1462,1463, 1471, 2467, 2723, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1464, 1465, 1466, 1467, 1472,1473, 1474, 1475, 1503, 2468, 2724, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1476, 1477, 1478, 1479, 1480, 1481, 1482,1483, 1504, 2469, 2725, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1505, 2470, 2726, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1492,1493, 1494, 1495, 1496, 1497, 1498, 1499, 1506, 2471, 2727, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1500, 1501, 1502,1507, 1508, 1509, 1510, 1511, 1538, 2472, 2728, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1512,1513, 1514, 1515, 1516, 1517, 1518, 1519, 1539, 2473, 2729, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1520, 1521, 1522,1523, 1524, 1525, 1526, 1527, 1540, 2474, 2730, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1528, 1529, 1530, 1531, 1532,1533, 1534, 1535, 1541, 2475, 2731, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1536, 1537, 1542,1543, 1544, 1545, 1546, 1547, 1574, 2476, 2732, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1548, 1549, 1550, 1551, 1552,1553, 1554, 1555, 1575, 2477, 2733, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1556, 1557, 1558, 1559, 1560, 1561, 1562,1563, 1576, 2478, 2734, 2990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1564, 1565, 1566, 1567, 1568, 1569, 1570, 1571, 1577, 2479, 2735, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1572,1573, 1578, 1579, 1580, 1581, 1582,1583, 1609, 2480, 2736, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1610, 2481, 2737, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1592,1593, 1594, 1595, 1596, 1597, 1598, 1599, 1611, 2482, 2738, 2994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1600, 1601, 1602,1603, 1604, 1605, 1606, 1607, 1612, 2483, 2739, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1608, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1644, 2484, 2740, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1620, 1621, 1622,1623, 1624, 1625, 1626, 1627, 1645, 2485, 2741, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1628, 1629, 1630, 1631, 1632,1633, 1634, 1635, 1646, 2486, 2742, 2998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1636, 1637, 1638, 1639, 1640, 1641, 1642,1643, 1679, 2487, 2743, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1647, 1648, 1649, 1650, 1651, 1652,1653, 1654, 1680, 2488, 2744, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662,1681, 2489, 2745, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1682, 2490, 2746, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1671, 1672,1673, 1674, 1675, 1676, 1677, 1678, 1714, 2491, 2747, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, 1715, 2492, 2748, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1691, 1692,1693, 1694, 1695, 1696, 1697, 1698, 1716, 2493, 2749, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1699, 1700, 1701, 1702,1703, 1704, 1705, 1706, 1717, 2494, 2750, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1707, 1708, 1709, 1710, 1711, 1712,1713, 1718, 1749, 2495, 2751, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1719, 1720, 1721, 1722,1723, 1724, 1725, 1726, 1750, 2496, 2752, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1727, 1728, 1729, 1730, 1731, 1732,1733, 1734, 1751, 2497, 2753, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742,1752, 2498, 2754, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1743, 1744, 1745, 1746, 1747, 1748, 1753, 1754, 1784, 2499, 2755, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762,1785, 2500, 2756, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1763, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1786, 2501, 2757, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1771, 1772,1773, 1774, 1775, 1776, 1777, 1778, 1787, 2502, 2758, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1779, 1780, 1781, 1782,1783, 1788, 1789, 1790, 1820, 2503, 2759, 3015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1791, 1792,1793, 1794, 1795, 1796, 1797, 1798, 1821, 2504, 2760, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1799, 1800, 1801, 1802,1803, 1804, 1805, 1806, 1822, 2505, 2761, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1807, 1808, 1809, 1810, 1811, 1812,1813, 1814, 1823, 2506, 2762, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1815, 1816, 1817, 1818, 1819, 1824, 1825, 1826, 1855, 2507, 2763, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1827, 1828, 1829, 1830, 1831, 1832,1833, 1834, 1856, 2508, 2764, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1835, 1836, 1837, 1838, 1839, 1840, 1841, 1842,1857, 2509, 2765, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1858, 2510, 2766, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1851, 1852,1853, 1854, 1859, 1860, 1861, 1862,1890, 2511, 2767, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1891, 2512, 2768, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1871, 1872,1873, 1874, 1875, 1876, 1877, 1878, 1892, 2513, 2769, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1879, 1880, 1881, 1882,1883, 1884, 1885, 1886, 1893, 2514, 2770, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1887, 1888, 1889, 1894, 1895, 1896, 1897, 1898, 1925, 2515, 2771, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1899, 1900, 1901, 1902,1903, 1904, 1905, 1906, 1926, 2516, 2772, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1907, 1908, 1909, 1910, 1911, 1912,1913, 1914, 1927, 2517, 2773, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922,1928, 2518, 2774, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1923, 1924, 1929, 1930, 1931, 1932,1933, 1934, 1960, 2519, 2775, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942,1961, 2520, 2776, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1943, 1944, 1945, 1946, 1947, 1948, 1949, 1950, 1962, 2521, 2777, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1951, 1952,1953, 1954, 1955, 1956, 1957, 1958, 1963, 2522, 2778, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1959, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1996, 2523, 2779, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1971, 1972,1973, 1974, 1975, 1976, 1977, 1978, 1997, 2524, 2780, 3036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1979, 1980, 1981, 1982,1983, 1984, 1985, 1986, 1998, 2525, 2781, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1987, 1988, 1989, 1990, 1991, 1992,1993, 1994, 1999, 2526, 2782, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1995, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2031, 2527, 2783, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2032, 2528, 2784, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2033, 2529, 2785, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2065, 2530, 2786, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2066, 2531, 2787, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2067, 2532, 2788, 3044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2068, 2533, 2789, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2058, 2059, 2060, 2061, 2062, 2063, 2064, 2069, 2100, 2534, 2790, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2101, 2535, 2791, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, 2102, 2536, 2792, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2086, 2087, 2088, 2089, 2090, 2091, 2092, 2093, 2103, 2537, 2793, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2094, 2095, 2096, 2097, 2098, 2099, 2104, 2105, 2135, 2538, 2794, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2106, 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2136, 2539, 2795, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2137, 2540, 2796, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2138, 2541, 2797, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2130, 2131, 2132, 2133, 2134, 2139, 2140, 2141, 2170, 2542, 2798, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, 2171, 2543, 2799, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, 2172, 2544, 2800, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2173, 2545, 2801, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2166, 2167, 2168, 2169, 2174, 2175, 2176, 2177, 2205, 2546, 2802, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2178, 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2206, 2547, 2803, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2186, 2187, 2188, 2189, 2190, 2191, 2192, 2193, 2207, 2548, 2804, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2208, 2549, 2805, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2202, 2203, 2204, 2209, 2210, 2211, 2212, 2213, 2240, 2550, 2806, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2241, 2551, 2807, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2222, 2223, 2224, 2225, 2226, 2227, 2228, 2229, 2242, 2552, 2808, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2243, 2553, 2809, 3065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2238, 2239, 2244, 2245, 2246, 2247, 2248, 2249, 2275, 2554, 2810, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2276, 2555, 2811, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2277, 2556, 2812, 3068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2278, 2557, 2813, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2274, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2302, 2558, 2814, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2303, 2559, 2815, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2304, 2560, 2816, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] + + + page_indices_rank1 = [[ 1, 2, 3, 4, 5, 6, 7, 8, 33, 2305, 2561, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 9, 10, 11, 12, 13, 14, 15, 16, 34, 2306, 2562, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 17, 18, 19, 20, 21, 22, 23, 24, 35, 2307, 2563, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 25, 26, 27, 28, 29, 30, 31, 32, 36, 2308, 2564, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 53, 54, 55, 56, 57, 58, 59, 60, 71, 2311, 2567, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 37, 38, 39, 40, 41, 42, 43, 44, 69, 2309, 2565, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 45, 46, 47, 48, 49, 50, 51, 52, 70, 2310, 2566, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 61, 62, 63, 64, 65, 66, 67, 68, 104, 2312, 2568, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 88, 89 , 90 , 91, 92, 93, 94, 95, 107, 2315, 2571, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 72, 73, 74, 75, 76, 77, 78, 79, 105, 2313, 2569, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 80, 81, 82, 83, 84, 85 ,86, 87, 106, 2314, 2570, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 96, 97, 98, 99, 100, 101, 102, 103, 140, 2316, 2572, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 124, 125, 126, 127, 128, 129, 130, 131, 143, 2319, 2575, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 108, 109, 110, 111, 112, 113, 114, 115, 141, 2317, 2573, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 116, 117, 118, 119, 120, 121, 122, 123, 142, 2318, 2574, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 132, 133, 134, 135, 136, 137, 138, 139, 176, 2320, 2576, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 160, 161, 162, 163, 164, 165, 166, 167, 179, 2323, 2579, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 144, 145, 146, 147, 148, 149, 150, 151, 177, 2321, 2577, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 152, 153, 154, 155, 156, 157, 158, 159, 178, 2322, 2578, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 168, 169, 170, 171, 172, 173, 174, 175, 212, 2324, 2580, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 196, 197, 198, 199, 200, 201, 202, 203, 215, 2327, 2583, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 180, 181, 182, 183, 184, 185, 186, 187, 213, 2325, 2581, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 188, 189, 190, 191, 192, 193, 194, 195, 214, 2326, 2582, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 204, 205, 206, 207, 208, 209, 210, 211, 248, 2328, 2584, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 232, 233, 234, 235, 236, 237, 238, 239, 251, 2331, 2587, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 216, 217, 218, 219, 220, 221, 222, 223, 249, 2329, 2585, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 224, 225, 226, 227, 228, 229, 230, 231, 250, 2330, 2586, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 240, 241, 242, 243, 244, 245, 246, 247, 284, 2332, 2588, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 268, 269, 270, 271, 272, 273, 274, 275, 287, 2335, 2591, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 252, 253, 254, 255, 256, 257, 258, 259, 285, 2333, 2589, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 260, 261, 262, 263, 264, 265, 266, 267, 286, 2334, 2590, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 276, 277, 278, 279, 280, 281, 282, 283, 320, 2336, 2592, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 304, 305, 306, 307, 308, 309, 310, 311, 323, 2339, 2595, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 288, 289, 290, 291, 292, 293, 294, 295, 321, 2337, 2593, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 296, 297, 298, 299, 300, 301, 302, 303, 322, 2338, 2594, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 312, 313, 314, 315, 316, 317, 318, 319, 356, 2340, 2596, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 340, 341, 342, 343, 344, 345, 346, 347, 359, 2343, 2599, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 324, 325, 326, 327, 328, 329, 330, 331, 357, 2341, 2597, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 332, 333, 334, 335, 336, 337, 338, 339, 358, 2342, 2598, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 348, 349, 350, 351, 352, 353, 354, 355, 392, 2344, 2600, 2856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 376, 377, 378, 379, 380, 381, 382, 383, 395, 2347, 2603, 2859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 360, 361, 362, 363, 364, 365, 366, 367, 393, 2345, 2601, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 368, 369, 370, 371, 372, 373, 374, 375, 394, 2346, 2602, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 384, 385, 386, 387, 388, 389, 390, 391, 428, 2348, 2604, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 412, 413, 414, 415, 416, 417, 418, 419, 431, 2351, 2607, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 396, 397, 398, 399, 400, 401, 402, 403, 429, 2349, 2605, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 404, 405, 406, 407, 408, 409, 410, 411, 430, 2350, 2606, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 420, 421, 422, 423, 424, 425, 426, 427, 463, 2352, 2608, 2864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 448, 449, 450, 451, 452, 453, 454, 455, 466, 2355, 2611, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 432, 433, 434, 435, 436, 437, 438, 439, 464, 2353, 2609, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 440, 441, 442, 443, 444, 445, 446, 447, 465, 2354, 2610, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 456, 457, 458, 459, 460, 461, 462, 467, 499, 2356, 2612, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 484, 485, 486, 487, 488, 489, 490, 491, 502, 2359, 2615, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 468, 469, 470, 471, 472, 473, 474, 475, 500, 2357, 2613, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 476, 477, 478, 479, 480, 481, 482, 483, 501, 2358, 2614, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 492, 493, 494, 495, 496, 497, 498, 503, 535, 2360, 2616, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 520, 521, 522, 523, 524, 525, 526, 527, 538, 2363, 2619, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 504, 505, 506, 507, 508, 509, 510, 511, 536, 2361, 2617, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 512, 513, 514, 515, 516, 517, 518, 519, 537, 2362, 2618, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 528, 529, 530, 531, 532, 533, 534, 539, 571, 2364, 2620, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 556, 557, 558, 559, 560, 561, 562, 563, 574, 2367, 2623, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 540, 541, 542, 543, 544, 545, 546, 547, 572, 2365, 2621, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 548, 549, 550, 551, 552, 553, 554, 555, 573, 2366, 2622, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 564, 565, 566, 567, 568, 569, 570, 575, 606, 2368, 2624, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 592, 593, 594, 595, 596, 597, 598, 599, 609, 2371, 2627, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 576, 577, 578, 579, 580, 581, 582, 583, 607, 2369, 2625, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 584, 585, 586, 587, 588, 589, 590, 591, 608, 2370, 2626, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 600, 601, 602, 603, 604, 605, 610, 611, 642, 2372, 2628, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 628, 629, 630, 631, 632, 633, 634, 635, 645, 2375, 2631, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 612, 613, 614, 615, 616, 617, 618, 619, 643, 2373, 2629, 2885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 620, 621, 622, 623, 624, 625, 626, 627, 644, 2374, 2630, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 636, 637, 638, 639, 640, 641, 646, 647, 678, 2376, 2632, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 664, 665, 666, 667, 668, 669, 670, 671, 681, 2379, 2635, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 648, 649, 650, 651, 652, 653, 654, 655, 679, 2377, 2633, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 656, 657, 658, 659, 660, 661, 662, 663, 680, 2378, 2634, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 672, 673, 674, 675, 676, 677, 682, 683, 714, 2380, 2636, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 700, 701, 702, 703, 704, 705, 706, 707, 717, 2383, 2639, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 684, 685, 686, 687, 688, 689, 690, 691, 715, 2381, 2637, 2893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 692, 693, 694, 695, 696, 697, 698, 699, 716, 2382, 2638, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 708, 709, 710, 711, 712, 713, 718, 719, 749, 2384, 2640, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 736, 737, 738, 739, 740, 741, 742, 743, 752, 2387, 2643, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 720, 721, 722, 723, 724, 725, 726, 727, 750, 2385, 2641, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 728, 729, 730, 731, 732, 733, 734, 735, 751, 2386, 2642, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 744, 745, 746, 747, 748, 753, 754, 755, 785, 2388, 2644, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 772, 773, 774, 775, 776, 777, 778, 779, 788, 2391, 2647, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 756, 757, 758, 759, 760, 761, 762, 763, 786, 2389, 2645, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 764, 765, 766, 767, 768, 769, 770, 771, 787, 2390, 2646, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 780, 781, 782, 783, 784, 789, 790, 791, 821, 2392, 2648, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 808, 809, 810, 811, 812, 813, 814, 815, 824, 2395, 2651, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 792, 793, 794, 795, 796, 797, 798, 799, 822, 2393, 2649, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 800, 801, 802, 803, 804, 805, 806, 807, 823, 2394, 2650, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 816, 817, 818, 819, 820, 825, 826, 827, 856, 2396, 2652, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 844, 845, 846, 847, 848, 849, 850, 851, 859, 2399, 2655, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 828, 829, 830, 831, 832, 833, 834, 835, 857, 2397, 2653, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 836, 837, 838, 839, 840, 841, 842, 843, 858, 2398, 2654, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 852, 853, 854, 855, 860, 861, 862, 863, 892, 2400, 2656, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 880, 881, 882, 883, 884, 885, 886, 887, 895, 2403, 2659, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 864, 865, 866, 867, 868, 869, 870, 871, 893, 2401, 2657, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 872, 873, 874, 875, 876, 877, 878, 879, 894, 2402, 2658, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 888, 889, 890, 891, 896, 897, 898, 899, 928, 2404, 2660, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 916, 917, 918, 919, 920, 921, 922, 923, 931, 2407, 2663, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 900, 901, 902, 903, 904, 905, 906, 907, 929, 2405, 2661, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 908, 909, 910, 911, 912, 913, 914, 915, 930, 2406, 2662, 2918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 924, 925, 926, 927, 932, 933, 934, 935, 963, 2408, 2664, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 952, 953, 954, 955, 956, 957, 958, 959, 966, 2411, 2667, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 936, 937, 938, 939, 940, 941, 942, 943, 964, 2409, 2665, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 944, 945, 946, 947, 948, 949, 950, 951, 965, 2410, 2666, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 960, 961, 962, 967, 968, 969, 970, 971, 999, 2412, 2668, 2924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 988, 989, 990, 991, 992, 993, 994, 995, 1002, 2415, 2671, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 972, 973, 974, 975, 976, 977, 978, 979, 1000, 2413, 2669, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 980, 981, 982, 983, 984, 985, 986, 987, 1001, 2414, 2670, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 996, 997, 998, 1003, 1004, 1005, 1006, 1007, 1034, 2416, 2672, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1037, 2419, 2675, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1008, 1009, 1010, 1011, 1012,1013, 1014, 1015, 1035, 2417, 2673, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1016, 1017, 1018, 1019, 1020, 1021, 1022,1023, 1036, 2418, 2674, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1032,1033, 1038, 1039, 1040, 1041, 1042,1043, 1070, 2420, 2676, 2932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1060, 1061, 1062,1063, 1064, 1065, 1066, 1067, 1073, 2423, 2679, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1071, 2421, 2677, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1052,1053, 1054, 1055, 1056, 1057, 1058, 1059, 1072, 2422, 2678, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1068, 1069, 1074, 1075, 1076, 1077, 1078, 1079, 1105, 2424, 2680, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1096, 1097, 1098, 1099, 1100, 1101, 1102,1103, 1108, 2427, 2683, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1080, 1081, 1082,1083, 1084, 1085, 1086, 1087, 1106, 2425, 2681, 2937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1088, 1089, 1090, 1091, 1092,1093, 1094, 1095, 1107, 2426, 2682, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1104, 1109, 1110, 1111, 1112,1113, 1114, 1115, 1141, 2428, 2684, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1132,1133, 1134, 1135, 1136, 1137, 1138, 1139, 1144, 2431, 2687, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1116, 1117, 1118, 1119, 1120, 1121, 1122,1123, 1142, 2429, 2685, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1143, 2430, 2686, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1140, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1176, 2432, 2688, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1160, 1161, 1162,1163, 1164, 1165, 1166, 1167, 1178, 2434, 2690, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1152,1153, 1154, 1155, 1156, 1157, 1158, 1159, 1177, 2433, 2689, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1168, 1169, 1170, 1171, 1172,1173, 1174, 1175, 1211, 2435, 2691, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202,1214, 2438, 2694, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1179, 1180, 1181, 1182,1183, 1184, 1185, 1186, 1212, 2436, 2692, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1187, 1188, 1189, 1190, 1191, 1192,1193, 1194, 1213, 2437, 2693, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1246, 2439, 2695, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1231, 1232,1233, 1234, 1235, 1236, 1237, 1238, 1249, 2442, 2698, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222,1247, 2440, 2696, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1248, 2441, 2697, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1239, 1240, 1241, 1242,1243, 1244, 1245, 1250, 1282, 2443, 2699, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1267, 1268, 1269, 1270, 1271, 1272,1273, 1274, 1285, 2446, 2702, 2958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1251, 1252,1253, 1254, 1255, 1256, 1257, 1258, 1283, 2444, 2700, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1259, 1260, 1261, 1262,1263, 1264, 1265, 1266, 1284, 2445, 2701, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1275, 1276, 1277, 1278, 1279, 1280, 1281, 1286, 1317, 2447, 2703, 2959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1320, 2450, 2706, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1287, 1288, 1289, 1290, 1291, 1292,1293, 1294, 1318, 2448, 2704, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302,1319, 2449, 2705, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1311, 1312,1313, 1314, 1315, 1316, 1321, 1322,1353, 2451, 2707, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1339, 1340, 1341, 1342,1343, 1344, 1345, 1346, 1356, 2454, 2710, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1354, 2452, 2708, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1331, 1332,1333, 1334, 1335, 1336, 1337, 1338, 1355, 2453, 2709, 2965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1347, 1348, 1349, 1350, 1351, 1352,1357, 1358, 1388, 2455, 2711, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382,1391, 2458, 2714, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1359, 1360, 1361, 1362,1363, 1364, 1365, 1366, 1389, 2456, 2712, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1367, 1368, 1369, 1370, 1371, 1372,1373, 1374, 1390, 2457, 2713, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1383, 1384, 1385, 1386, 1387, 1392,1393, 1394, 1423, 2459, 2715, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1411, 1412,1413, 1414, 1415, 1416, 1417, 1418, 1426, 2462, 2718, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402,1424, 2460, 2716, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1425, 2461, 2717, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1419, 1420, 1421, 1422,1427, 1428, 1429, 1430, 1459, 2463, 2719, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1447, 1448, 1449, 1450, 1451, 1452,1453, 1454, 1462, 2466, 2722, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1431, 1432,1433, 1434, 1435, 1436, 1437, 1438, 1460, 2464, 2720, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1439, 1440, 1441, 1442,1443, 1444, 1445, 1446, 1461, 2465, 2721, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1455, 1456, 1457, 1458, 1463, 1464, 1465, 1466, 1494, 2467, 2723, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1497, 2470, 2726, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1467, 1468, 1469, 1470, 1471, 1472,1473, 1474, 1495, 2468, 2724, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482,1496, 2469, 2725, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1491, 1492,1493, 1498, 1499, 1500, 1501, 1502,1530, 2471, 2727, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1519, 1520, 1521, 1522,1523, 1524, 1525, 1526, 1533, 2474, 2730, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1531, 2472, 2728, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1511, 1512,1513, 1514, 1515, 1516, 1517, 1518, 1532, 2473, 2729, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1527, 1528, 1529, 1534, 1535, 1536, 1537, 1538, 1565, 2475, 2731, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562,1568, 2478, 2734, 2990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1539, 1540, 1541, 1542,1543, 1544, 1545, 1546, 1566, 2476, 2732, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1547, 1548, 1549, 1550, 1551, 1552,1553, 1554, 1567, 2477, 2733, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1563, 1564, 1569, 1570, 1571, 1572,1573, 1574, 1600, 2479, 2735, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1591, 1592,1593, 1594, 1595, 1596, 1597, 1598, 1603, 2482, 2738, 2994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1575, 1576, 1577, 1578, 1579, 1580, 1581, 1582,1601, 2480, 2736, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1583, 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1602, 2481, 2737, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1599, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1636, 2483, 2739, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1627, 1628, 1629, 1630, 1631, 1632,1633, 1634, 1639, 2486, 2742, 2998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1611, 1612,1613, 1614, 1615, 1616, 1617, 1618, 1637, 2484, 2740, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1619, 1620, 1621, 1622,1623, 1624, 1625, 1626, 1638, 2485, 2741, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1635, 1640, 1641, 1642,1643, 1644, 1645, 1646, 1671, 2487, 2743, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662,1673, 2489, 2745, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1647, 1648, 1649, 1650, 1651, 1652,1653, 1654, 1672, 2488, 2744, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1705, 2490, 2746, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1690, 1691, 1692,1693, 1694, 1695, 1696, 1697, 1708, 2493, 2749, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1674, 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1706, 2491, 2747, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1682,1683, 1684, 1685, 1686, 1687, 1688, 1689, 1707, 2492, 2748, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1698, 1699, 1700, 1701, 1702,1703, 1704, 1709, 1740, 2494, 2750, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1726, 1727, 1728, 1729, 1730, 1731, 1732,1733, 1743, 2497, 2753, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1710, 1711, 1712,1713, 1714, 1715, 1716, 1717, 1741, 2495, 2751, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1718, 1719, 1720, 1721, 1722,1723, 1724, 1725, 1742, 2496, 2752, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1734, 1735, 1736, 1737, 1738, 1739, 1744, 1745, 1776, 2498, 2754, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1762,1763, 1764, 1765, 1766, 1767, 1768, 1769, 1779, 2501, 2757, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1746, 1747, 1748, 1749, 1750, 1751, 1752,1753, 1777, 2499, 2755, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1754, 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1778, 2500, 2756, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1770, 1771, 1772,1773, 1774, 1775, 1780, 1781, 1811, 2502, 2758, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1798, 1799, 1800, 1801, 1802,1803, 1804, 1805, 1814, 2505, 2761, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1782,1783, 1784, 1785, 1786, 1787, 1788, 1789, 1812, 2503, 2759, 3015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1790, 1791, 1792,1793, 1794, 1795, 1796, 1797, 1813, 2504, 2760, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1806, 1807, 1808, 1809, 1810, 1815, 1816, 1817, 1846, 2506, 2762, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1834, 1835, 1836, 1837, 1838, 1839, 1840, 1841, 1849, 2509, 2765, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1818, 1819, 1820, 1821, 1822,1823, 1824, 1825, 1847, 2507, 2763, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1826, 1827, 1828, 1829, 1830, 1831, 1832,1833, 1848, 2508, 2764, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1842,1843, 1844, 1845, 1850, 1851, 1852,1853, 1881, 2510, 2766, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1870, 1871, 1872,1873, 1874, 1875, 1876, 1877, 1884, 2513, 2769, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, 1882, 2511, 2767, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1862,1863, 1864, 1865, 1866, 1867, 1868, 1869, 1883, 2512, 2768, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1878, 1879, 1880, 1885, 1886, 1887, 1888, 1889, 1916, 2514, 2770, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1906, 1907, 1908, 1909, 1910, 1911, 1912,1913, 1919, 2517, 2773, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1890, 1891, 1892,1893, 1894, 1895, 1896, 1897, 1917, 2515, 2771, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1898, 1899, 1900, 1901, 1902,1903, 1904, 1905, 1918, 2516, 2772, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1914, 1915, 1920, 1921, 1922,1923, 1924, 1925, 1952, 2518, 2774, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1942,1943, 1944, 1945, 1946, 1947, 1948, 1949, 1955, 2521, 2777, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1926, 1927, 1928, 1929, 1930, 1931, 1932,1933, 1953, 2519, 2775, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1954, 2520, 2776, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1950, 1951, 1956, 1957, 1958, 1959, 1960, 1961, 1987, 2522, 2778, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1978, 1979, 1980, 1981, 1982,1983, 1984, 1985, 1990, 2525, 2781, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1962,1963, 1964, 1965, 1966, 1967, 1968, 1969, 1988, 2523, 2779, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1970, 1971, 1972,1973, 1974, 1975, 1976, 1977, 1989, 2524, 2780, 3036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1986, 1991, 1992,1993, 1994, 1995, 1996, 1997, 2022, 2526, 2782, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2024, 2528, 2784, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2023, 2527, 2783, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2056, 2529, 2785, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2059, 2532, 2788, 3044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2057, 2530, 2786, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2058, 2531, 2787, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2049, 2050, 2051, 2052, 2053, 2054, 2055, 2060, 2091, 2533, 2789, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2077, 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2094, 2536, 2792, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2092, 2534, 2790, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2093, 2535, 2791, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2085, 2086, 2087, 2088, 2089, 2090, 2095, 2096, 2126, 2537, 2793, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2129, 2540, 2796, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2097, 2098, 2099, 2100, 2101, 2102, 2103, 2104, 2127, 2538, 2794, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2105, 2106, 2107, 2108, 2109, 2110, 2111, 2112, 2128, 2539, 2795, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2121, 2122, 2123, 2124, 2125, 2130, 2131, 2132, 2161, 2541, 2797, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2164, 2544, 2800, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2133, 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2162, 2542, 2798, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2141, 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2163, 2543, 2799, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2157, 2158, 2159, 2160, 2165, 2166, 2167, 2168, 2196, 2545, 2801, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, 2199, 2548, 2804, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2197, 2546, 2802, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2177, 2178, 2179, 2180, 2181, 2182, 2183, 2184, 2198, 2547, 2803, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2193, 2194, 2195, 2200, 2201, 2202, 2203, 2204, 2231, 2549, 2805, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2221, 2222, 2223, 2224, 2225, 2226, 2227, 2228, 2234, 2552, 2808, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2205, 2206, 2207, 2208, 2209, 2210, 2211, 2212, 2232, 2550, 2806, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2233, 2551, 2807, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2229, 2230, 2235, 2236, 2237, 2238, 2239, 2240, 2266, 2553, 2809, 3065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2269, 2556, 2812, 3068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2267, 2554, 2810, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2268, 2555, 2811, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2265, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2301, 2557, 2813, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2302, 2558, 2814, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2303, 2559, 2815, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2304, 2560, 2816, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] + page_indices_rank0 = sum(page_indices_rank0, []) + page_indices_rank1 = sum(page_indices_rank1, []) + page_indices_no_dp = [[ 1, 2, 3, 4, 5, 6, 7, 8, 17, 2305, 2561, + 2817, 3073, 3329, 3585, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 9, 10, 11, 12, 13, 14, 15, 16, 18, 2306, 2562, + 2818, 3074, 3330, 3586, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 19, 20, 21, 22, 23, 24, 25, 26, 51, 2307, 2563, + 2819, 3075, 3331, 3587, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 27, 28, 29, 30, 31, 32, 33, 34, 52, 2308, 2564, + 2820, 3076, 3332, 3588, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 35, 36, 37, 38, 39, 40, 41, 42, 53, 2309, 2565, + 2821, 3077, 3333, 3589, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 43, 44, 45, 46, 47, 48, 49, 50, 86, 2310, 2566, + 2822, 3078, 3334, 3590, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 54, 55, 56, 57, 58, 59, 60, 61, 87, 2311, 2567, + 2823, 3079, 3335, 3591, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 62, 63, 64, 65, 66, 67, 68, 69, 88, 2312, 2568, + 2824, 3080, 3336, 3592, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 70, 71, 72, 73, 74, 75, 76, 77, 89, 2313, 2569, + 2825, 3081, 3337, 3593, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 78, 79, 80, 81, 82, 83, 84, 85, 122, 2314, 2570, + 2826, 3082, 3338, 3594, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 90, 91, 92, 93, 94, 95, 96, 97, 123, 2315, 2571, + 2827, 3083, 3339, 3595, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 98, 99, 100, 101, 102, 103, 104, 105, 124, 2316, 2572, + 2828, 3084, 3340, 3596, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 106, 107, 108, 109, 110, 111, 112, 113, 125, 2317, 2573, + 2829, 3085, 3341, 3597, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 114, 115, 116, 117, 118, 119, 120, 121, 158, 2318, 2574, + 2830, 3086, 3342, 3598, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 126, 127, 128, 129, 130, 131, 132, 133, 159, 2319, 2575, + 2831, 3087, 3343, 3599, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 134, 135, 136, 137, 138, 139, 140, 141, 160, 2320, 2576, + 2832, 3088, 3344, 3600, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 142, 143, 144, 145, 146, 147, 148, 149, 161, 2321, 2577, + 2833, 3089, 3345, 3601, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 150, 151, 152, 153, 154, 155, 156, 157, 194, 2322, 2578, + 2834, 3090, 3346, 3602, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 162, 163, 164, 165, 166, 167, 168, 169, 195, 2323, 2579, + 2835, 3091, 3347, 3603, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 170, 171, 172, 173, 174, 175, 176, 177, 196, 2324, 2580, + 2836, 3092, 3348, 3604, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 178, 179, 180, 181, 182, 183, 184, 185, 197, 2325, 2581, + 2837, 3093, 3349, 3605, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 186, 187, 188, 189, 190, 191, 192, 193, 230, 2326, 2582, + 2838, 3094, 3350, 3606, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 198, 199, 200, 201, 202, 203, 204, 205, 231, 2327, 2583, + 2839, 3095, 3351, 3607, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 206, 207, 208, 209, 210, 211, 212, 213, 232, 2328, 2584, + 2840, 3096, 3352, 3608, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 214, 215, 216, 217, 218, 219, 220, 221, 233, 2329, 2585, + 2841, 3097, 3353, 3609, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 222, 223, 224, 225, 226, 227, 228, 229, 266, 2330, 2586, + 2842, 3098, 3354, 3610, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 234, 235, 236, 237, 238, 239, 240, 241, 267, 2331, 2587, + 2843, 3099, 3355, 3611, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 242, 243, 244, 245, 246, 247, 248, 249, 268, 2332, 2588, + 2844, 3100, 3356, 3612, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 250, 251, 252, 253, 254, 255, 256, 257, 269, 2333, 2589, + 2845, 3101, 3357, 3613, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 258, 259, 260, 261, 262, 263, 264, 265, 302, 2334, 2590, + 2846, 3102, 3358, 3614, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 270, 271, 272, 273, 274, 275, 276, 277, 303, 2335, 2591, + 2847, 3103, 3359, 3615, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 278, 279, 280, 281, 282, 283, 284, 285, 304, 2336, 2592, + 2848, 3104, 3360, 3616, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 286, 287, 288, 289, 290, 291, 292, 293, 305, 2337, 2593, + 2849, 3105, 3361, 3617, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 294, 295, 296, 297, 298, 299, 300, 301, 338, 2338, 2594, + 2850, 3106, 3362, 3618, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 306, 307, 308, 309, 310, 311, 312, 313, 339, 2339, 2595, + 2851, 3107, 3363, 3619, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 314, 315, 316, 317, 318, 319, 320, 321, 340, 2340, 2596, + 2852, 3108, 3364, 3620, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 322, 323, 324, 325, 326, 327, 328, 329, 341, 2341, 2597, + 2853, 3109, 3365, 3621, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 330, 331, 332, 333, 334, 335, 336, 337, 374, 2342, 2598, + 2854, 3110, 3366, 3622, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 342, 343, 344, 345, 346, 347, 348, 349, 375, 2343, 2599, + 2855, 3111, 3367, 3623, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 350, 351, 352, 353, 354, 355, 356, 357, 376, 2344, 2600, + 2856, 3112, 3368, 3624, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 358, 359, 360, 361, 362, 363, 364, 365, 377, 2345, 2601, + 2857, 3113, 3369, 3625, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 366, 367, 368, 369, 370, 371, 372, 373, 410, 2346, 2602, + 2858, 3114, 3370, 3626, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 378, 379, 380, 381, 382, 383, 384, 385, 411, 2347, 2603, + 2859, 3115, 3371, 3627, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 386, 387, 388, 389, 390, 391, 392, 393, 412, 2348, 2604, + 2860, 3116, 3372, 3628, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 394, 395, 396, 397, 398, 399, 400, 401, 413, 2349, 2605, + 2861, 3117, 3373, 3629, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 402, 403, 404, 405, 406, 407, 408, 409, 445, 2350, 2606, + 2862, 3118, 3374, 3630, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 414, 415, 416, 417, 418, 419, 420, 421, 446, 2351, 2607, + 2863, 3119, 3375, 3631, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 422, 423, 424, 425, 426, 427, 428, 429, 447, 2352, 2608, + 2864, 3120, 3376, 3632, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 430, 431, 432, 433, 434, 435, 436, 437, 448, 2353, 2609, + 2865, 3121, 3377, 3633, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 438, 439, 440, 441, 442, 443, 444, 449, 481, 2354, 2610, + 2866, 3122, 3378, 3634, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 450, 451, 452, 453, 454, 455, 456, 457, 482, 2355, 2611, + 2867, 3123, 3379, 3635, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 458, 459, 460, 461, 462, 463, 464, 465, 483, 2356, 2612, + 2868, 3124, 3380, 3636, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 466, 467, 468, 469, 470, 471, 472, 473, 484, 2357, 2613, + 2869, 3125, 3381, 3637, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 474, 475, 476, 477, 478, 479, 480, 485, 517, 2358, 2614, + 2870, 3126, 3382, 3638, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 486, 487, 488, 489, 490, 491, 492, 493, 518, 2359, 2615, + 2871, 3127, 3383, 3639, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 494, 495, 496, 497, 498, 499, 500, 501, 519, 2360, 2616, + 2872, 3128, 3384, 3640, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 502, 503, 504, 505, 506, 507, 508, 509, 520, 2361, 2617, + 2873, 3129, 3385, 3641, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 510, 511, 512, 513, 514, 515, 516, 521, 553, 2362, 2618, + 2874, 3130, 3386, 3642, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 522, 523, 524, 525, 526, 527, 528, 529, 554, 2363, 2619, + 2875, 3131, 3387, 3643, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 530, 531, 532, 533, 534, 535, 536, 537, 555, 2364, 2620, + 2876, 3132, 3388, 3644, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 538, 539, 540, 541, 542, 543, 544, 545, 556, 2365, 2621, + 2877, 3133, 3389, 3645, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 546, 547, 548, 549, 550, 551, 552, 557, 589, 2366, 2622, + 2878, 3134, 3390, 3646, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 558, 559, 560, 561, 562, 563, 564, 565, 590, 2367, 2623, + 2879, 3135, 3391, 3647, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 566, 567, 568, 569, 570, 571, 572, 573, 591, 2368, 2624, + 2880, 3136, 3392, 3648, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 574, 575, 576, 577, 578, 579, 580, 581, 592, 2369, 2625, + 2881, 3137, 3393, 3649, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 582, 583, 584, 585, 586, 587, 588, 593, 624, 2370, 2626, + 2882, 3138, 3394, 3650, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 594, 595, 596, 597, 598, 599, 600, 601, 625, 2371, 2627, + 2883, 3139, 3395, 3651, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 602, 603, 604, 605, 606, 607, 608, 609, 626, 2372, 2628, + 2884, 3140, 3396, 3652, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 610, 611, 612, 613, 614, 615, 616, 617, 627, 2373, 2629, + 2885, 3141, 3397, 3653, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 618, 619, 620, 621, 622, 623, 628, 629, 660, 2374, 2630, + 2886, 3142, 3398, 3654, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 630, 631, 632, 633, 634, 635, 636, 637, 661, 2375, 2631, + 2887, 3143, 3399, 3655, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 638, 639, 640, 641, 642, 643, 644, 645, 662, 2376, 2632, + 2888, 3144, 3400, 3656, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 646, 647, 648, 649, 650, 651, 652, 653, 663, 2377, 2633, + 2889, 3145, 3401, 3657, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 654, 655, 656, 657, 658, 659, 664, 665, 696, 2378, 2634, + 2890, 3146, 3402, 3658, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 666, 667, 668, 669, 670, 671, 672, 673, 697, 2379, 2635, + 2891, 3147, 3403, 3659, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 674, 675, 676, 677, 678, 679, 680, 681, 698, 2380, 2636, + 2892, 3148, 3404, 3660, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 682, 683, 684, 685, 686, 687, 688, 689, 699, 2381, 2637, + 2893, 3149, 3405, 3661, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 690, 691, 692, 693, 694, 695, 700, 701, 731, 2382, 2638, + 2894, 3150, 3406, 3662, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 702, 703, 704, 705, 706, 707, 708, 709, 732, 2383, 2639, + 2895, 3151, 3407, 3663, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 710, 711, 712, 713, 714, 715, 716, 717, 733, 2384, 2640, + 2896, 3152, 3408, 3664, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 718, 719, 720, 721, 722, 723, 724, 725, 734, 2385, 2641, + 2897, 3153, 3409, 3665, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 726, 727, 728, 729, 730, 735, 736, 737, 767, 2386, 2642, + 2898, 3154, 3410, 3666, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 738, 739, 740, 741, 742, 743, 744, 745, 768, 2387, 2643, + 2899, 3155, 3411, 3667, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 746, 747, 748, 749, 750, 751, 752, 753, 769, 2388, 2644, + 2900, 3156, 3412, 3668, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 754, 755, 756, 757, 758, 759, 760, 761, 770, 2389, 2645, + 2901, 3157, 3413, 3669, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 762, 763, 764, 765, 766, 771, 772, 773, 803, 2390, 2646, + 2902, 3158, 3414, 3670, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 774, 775, 776, 777, 778, 779, 780, 781, 804, 2391, 2647, + 2903, 3159, 3415, 3671, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 782, 783, 784, 785, 786, 787, 788, 789, 805, 2392, 2648, + 2904, 3160, 3416, 3672, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 790, 791, 792, 793, 794, 795, 796, 797, 806, 2393, 2649, + 2905, 3161, 3417, 3673, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 798, 799, 800, 801, 802, 807, 808, 809, 838, 2394, 2650, + 2906, 3162, 3418, 3674, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 810, 811, 812, 813, 814, 815, 816, 817, 839, 2395, 2651, + 2907, 3163, 3419, 3675, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 818, 819, 820, 821, 822, 823, 824, 825, 840, 2396, 2652, + 2908, 3164, 3420, 3676, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 826, 827, 828, 829, 830, 831, 832, 833, 841, 2397, 2653, + 2909, 3165, 3421, 3677, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 834, 835, 836, 837, 842, 843, 844, 845, 874, 2398, 2654, + 2910, 3166, 3422, 3678, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 846, 847, 848, 849, 850, 851, 852, 853, 875, 2399, 2655, + 2911, 3167, 3423, 3679, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 854, 855, 856, 857, 858, 859, 860, 861, 876, 2400, 2656, + 2912, 3168, 3424, 3680, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 862, 863, 864, 865, 866, 867, 868, 869, 877, 2401, 2657, + 2913, 3169, 3425, 3681, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 870, 871, 872, 873, 878, 879, 880, 881, 910, 2402, 2658, + 2914, 3170, 3426, 3682, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 882, 883, 884, 885, 886, 887, 888, 889, 911, 2403, 2659, + 2915, 3171, 3427, 3683, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 890, 891, 892, 893, 894, 895, 896, 897, 912, 2404, 2660, + 2916, 3172, 3428, 3684, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 898, 899, 900, 901, 902, 903, 904, 905, 913, 2405, 2661, + 2917, 3173, 3429, 3685, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 906, 907, 908, 909, 914, 915, 916, 917, 945, 2406, 2662, + 2918, 3174, 3430, 3686, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 918, 919, 920, 921, 922, 923, 924, 925, 946, 2407, 2663, + 2919, 3175, 3431, 3687, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 926, 927, 928, 929, 930, 931, 932, 933, 947, 2408, 2664, + 2920, 3176, 3432, 3688, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 934, 935, 936, 937, 938, 939, 940, 941, 948, 2409, 2665, + 2921, 3177, 3433, 3689, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 942, 943, 944, 949, 950, 951, 952, 953, 981, 2410, 2666, + 2922, 3178, 3434, 3690, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 954, 955, 956, 957, 958, 959, 960, 961, 982, 2411, 2667, + 2923, 3179, 3435, 3691, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 962, 963, 964, 965, 966, 967, 968, 969, 983, 2412, 2668, + 2924, 3180, 3436, 3692, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 970, 971, 972, 973, 974, 975, 976, 977, 984, 2413, 2669, + 2925, 3181, 3437, 3693, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 978, 979, 980, 985, 986, 987, 988, 989, 1016, 2414, 2670, + 2926, 3182, 3438, 3694, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 990, 991, 992, 993, 994, 995, 996, 997, 1017, 2415, 2671, + 2927, 3183, 3439, 3695, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1018, 2416, 2672, + 2928, 3184, 3440, 3696, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1019, 2417, 2673, + 2929, 3185, 3441, 3697, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1014, 1015, 1020, 1021, 1022, 1023, 1024, 1025, 1052, 2418, 2674, + 2930, 3186, 3442, 3698, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1053, 2419, 2675, + 2931, 3187, 3443, 3699, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1054, 2420, 2676, + 2932, 3188, 3444, 3700, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1055, 2421, 2677, + 2933, 3189, 3445, 3701, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1050, 1051, 1056, 1057, 1058, 1059, 1060, 1061, 1088, 2422, 2678, + 2934, 3190, 3446, 3702, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1089, 2423, 2679, + 2935, 3191, 3447, 3703, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1090, 2424, 2680, + 2936, 3192, 3448, 3704, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1091, 2425, 2681, + 2937, 3193, 3449, 3705, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1086, 1087, 1092, 1093, 1094, 1095, 1096, 1097, 1123, 2426, 2682, + 2938, 3194, 3450, 3706, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1124, 2427, 2683, + 2939, 3195, 3451, 3707, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1125, 2428, 2684, + 2940, 3196, 3452, 3708, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1126, 2429, 2685, + 2941, 3197, 3453, 3709, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1122, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1159, 2430, 2686, + 2942, 3198, 3454, 3710, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1160, 2431, 2687, + 2943, 3199, 3455, 3711, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1161, 2432, 2688, + 2944, 3200, 3456, 3712, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1162, 2433, 2689, + 2945, 3201, 3457, 3713, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1158, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1194, 2434, 2690, + 2946, 3202, 3458, 3714, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1195, 2435, 2691, + 2947, 3203, 3459, 3715, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1196, 2436, 2692, + 2948, 3204, 3460, 3716, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1229, 2437, 2693, + 2949, 3205, 3461, 3717, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1230, 2438, 2694, + 2950, 3206, 3462, 3718, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1231, 2439, 2695, + 2951, 3207, 3463, 3719, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1232, 2440, 2696, + 2952, 3208, 3464, 3720, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1264, 2441, 2697, + 2953, 3209, 3465, 3721, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1265, 2442, 2698, + 2954, 3210, 3466, 3722, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1266, 2443, 2699, + 2955, 3211, 3467, 3723, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1267, 2444, 2700, + 2956, 3212, 3468, 3724, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1257, 1258, 1259, 1260, 1261, 1262, 1263, 1268, 1300, 2445, 2701, + 2957, 3213, 3469, 3725, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1301, 2446, 2702, + 2958, 3214, 3470, 3726, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1302, 2447, 2703, + 2959, 3215, 3471, 3727, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1303, 2448, 2704, + 2960, 3216, 3472, 3728, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1293, 1294, 1295, 1296, 1297, 1298, 1299, 1304, 1335, 2449, 2705, + 2961, 3217, 3473, 3729, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1336, 2450, 2706, + 2962, 3218, 3474, 3730, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1337, 2451, 2707, + 2963, 3219, 3475, 3731, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1338, 2452, 2708, + 2964, 3220, 3476, 3732, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1329, 1330, 1331, 1332, 1333, 1334, 1339, 1340, 1370, 2453, 2709, + 2965, 3221, 3477, 3733, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1371, 2454, 2710, + 2966, 3222, 3478, 3734, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1372, 2455, 2711, + 2967, 3223, 3479, 3735, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1373, 2456, 2712, + 2968, 3224, 3480, 3736, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1365, 1366, 1367, 1368, 1369, 1374, 1375, 1376, 1406, 2457, 2713, + 2969, 3225, 3481, 3737, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1407, 2458, 2714, + 2970, 3226, 3482, 3738, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1408, 2459, 2715, + 2971, 3227, 3483, 3739, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1409, 2460, 2716, + 2972, 3228, 3484, 3740, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1401, 1402, 1403, 1404, 1405, 1410, 1411, 1412, 1441, 2461, 2717, + 2973, 3229, 3485, 3741, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1442, 2462, 2718, + 2974, 3230, 3486, 3742, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1421, 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1443, 2463, 2719, + 2975, 3231, 3487, 3743, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1444, 2464, 2720, + 2976, 3232, 3488, 3744, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1437, 1438, 1439, 1440, 1445, 1446, 1447, 1448, 1477, 2465, 2721, + 2977, 3233, 3489, 3745, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1449, 1450, 1451, 1452, 1453, 1454, 1455, 1456, 1478, 2466, 2722, + 2978, 3234, 3490, 3746, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1479, 2467, 2723, + 2979, 3235, 3491, 3747, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1465, 1466, 1467, 1468, 1469, 1470, 1471, 1472, 1480, 2468, 2724, + 2980, 3236, 3492, 3748, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1473, 1474, 1475, 1476, 1481, 1482, 1483, 1484, 1512, 2469, 2725, + 2981, 3237, 3493, 3749, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1513, 2470, 2726, + 2982, 3238, 3494, 3750, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1493, 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1514, 2471, 2727, + 2983, 3239, 3495, 3751, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1515, 2472, 2728, + 2984, 3240, 3496, 3752, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1509, 1510, 1511, 1516, 1517, 1518, 1519, 1520, 1547, 2473, 2729, + 2985, 3241, 3497, 3753, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1548, 2474, 2730, + 2986, 3242, 3498, 3754, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1529, 1530, 1531, 1532, 1533, 1534, 1535, 1536, 1549, 2475, 2731, + 2987, 3243, 3499, 3755, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1550, 2476, 2732, + 2988, 3244, 3500, 3756, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1545, 1546, 1551, 1552, 1553, 1554, 1555, 1556, 1583, 2477, 2733, + 2989, 3245, 3501, 3757, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1584, 2478, 2734, + 2990, 3246, 3502, 3758, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1565, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1585, 2479, 2735, + 2991, 3247, 3503, 3759, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1586, 2480, 2736, + 2992, 3248, 3504, 3760, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1581, 1582, 1587, 1588, 1589, 1590, 1591, 1592, 1618, 2481, 2737, + 2993, 3249, 3505, 3761, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1593, 1594, 1595, 1596, 1597, 1598, 1599, 1600, 1619, 2482, 2738, + 2994, 3250, 3506, 3762, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1620, 2483, 2739, + 2995, 3251, 3507, 3763, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1621, 2484, 2740, + 2996, 3252, 3508, 3764, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1617, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1653, 2485, 2741, + 2997, 3253, 3509, 3765, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1654, 2486, 2742, + 2998, 3254, 3510, 3766, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1655, 2487, 2743, + 2999, 3255, 3511, 3767, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1645, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1687, 2488, 2744, + 3000, 3256, 3512, 3768, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1688, 2489, 2745, + 3001, 3257, 3513, 3769, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1689, 2490, 2746, + 3002, 3258, 3514, 3770, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1690, 2491, 2747, + 3003, 3259, 3515, 3771, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1680, 1681, 1682, 1683, 1684, 1685, 1686, 1691, 1723, 2492, 2748, + 3004, 3260, 3516, 3772, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699, 1724, 2493, 2749, + 3005, 3261, 3517, 3773, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1725, 2494, 2750, + 3006, 3262, 3518, 3774, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1708, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1726, 2495, 2751, + 3007, 3263, 3519, 3775, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1716, 1717, 1718, 1719, 1720, 1721, 1722, 1727, 1758, 2496, 2752, + 3008, 3264, 3520, 3776, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1728, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1759, 2497, 2753, + 3009, 3265, 3521, 3777, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, 1760, 2498, 2754, + 3010, 3266, 3522, 3778, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1761, 2499, 2755, + 3011, 3267, 3523, 3779, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1752, 1753, 1754, 1755, 1756, 1757, 1762, 1763, 1793, 2500, 2756, + 3012, 3268, 3524, 3780, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1764, 1765, 1766, 1767, 1768, 1769, 1770, 1771, 1794, 2501, 2757, + 3013, 3269, 3525, 3781, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1795, 2502, 2758, + 3014, 3270, 3526, 3782, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1796, 2503, 2759, + 3015, 3271, 3527, 3783, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1788, 1789, 1790, 1791, 1792, 1797, 1798, 1799, 1828, 2504, 2760, + 3016, 3272, 3528, 3784, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1829, 2505, 2761, + 3017, 3273, 3529, 3785, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1830, 2506, 2762, + 3018, 3274, 3530, 3786, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1831, 2507, 2763, + 3019, 3275, 3531, 3787, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1824, 1825, 1826, 1827, 1832, 1833, 1834, 1835, 1864, 2508, 2764, + 3020, 3276, 3532, 3788, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1836, 1837, 1838, 1839, 1840, 1841, 1842, 1843, 1865, 2509, 2765, + 3021, 3277, 3533, 3789, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1866, 2510, 2766, + 3022, 3278, 3534, 3790, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1852, 1853, 1854, 1855, 1856, 1857, 1858, 1859, 1867, 2511, 2767, + 3023, 3279, 3535, 3791, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1860, 1861, 1862, 1863, 1868, 1869, 1870, 1871, 1899, 2512, 2768, + 3024, 3280, 3536, 3792, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, 1900, 2513, 2769, + 3025, 3281, 3537, 3793, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1901, 2514, 2770, + 3026, 3282, 3538, 3794, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1902, 2515, 2771, + 3027, 3283, 3539, 3795, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1896, 1897, 1898, 1903, 1904, 1905, 1906, 1907, 1934, 2516, 2772, + 3028, 3284, 3540, 3796, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1935, 2517, 2773, + 3029, 3285, 3541, 3797, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1936, 2518, 2774, + 3030, 3286, 3542, 3798, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1937, 2519, 2775, + 3031, 3287, 3543, 3799, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1932, 1933, 1938, 1939, 1940, 1941, 1942, 1943, 1969, 2520, 2776, + 3032, 3288, 3544, 3800, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1944, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1970, 2521, 2777, + 3033, 3289, 3545, 3801, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, 1971, 2522, 2778, + 3034, 3290, 3546, 3802, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1972, 2523, 2779, + 3035, 3291, 3547, 3803, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1968, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 2004, 2524, 2780, + 3036, 3292, 3548, 3804, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2005, 2525, 2781, + 3037, 3293, 3549, 3805, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 2006, 2526, 2782, + 3038, 3294, 3550, 3806, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2038, 2527, 2783, + 3039, 3295, 3551, 3807, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2039, 2528, 2784, + 3040, 3296, 3552, 3808, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2040, 2529, 2785, + 3041, 3297, 3553, 3809, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2041, 2530, 2786, + 3042, 3298, 3554, 3810, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2031, 2032, 2033, 2034, 2035, 2036, 2037, 2042, 2074, 2531, 2787, + 3043, 3299, 3555, 3811, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2075, 2532, 2788, + 3044, 3300, 3556, 3812, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2076, 2533, 2789, + 3045, 3301, 3557, 3813, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2077, 2534, 2790, + 3046, 3302, 3558, 3814, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2067, 2068, 2069, 2070, 2071, 2072, 2073, 2078, 2109, 2535, 2791, + 3047, 3303, 3559, 3815, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2079, 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2110, 2536, 2792, + 3048, 3304, 3560, 3816, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2087, 2088, 2089, 2090, 2091, 2092, 2093, 2094, 2111, 2537, 2793, + 3049, 3305, 3561, 3817, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2095, 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2112, 2538, 2794, + 3050, 3306, 3562, 3818, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2103, 2104, 2105, 2106, 2107, 2108, 2113, 2114, 2144, 2539, 2795, + 3051, 3307, 3563, 3819, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2145, 2540, 2796, + 3052, 3308, 3564, 3820, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2146, 2541, 2797, + 3053, 3309, 3565, 3821, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2147, 2542, 2798, + 3054, 3310, 3566, 3822, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2139, 2140, 2141, 2142, 2143, 2148, 2149, 2150, 2179, 2543, 2799, + 3055, 3311, 3567, 3823, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2151, 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2180, 2544, 2800, + 3056, 3312, 3568, 3824, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2181, 2545, 2801, + 3057, 3313, 3569, 3825, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2182, 2546, 2802, + 3058, 3314, 3570, 3826, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2175, 2176, 2177, 2178, 2183, 2184, 2185, 2186, 2214, 2547, 2803, + 3059, 3315, 3571, 3827, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2187, 2188, 2189, 2190, 2191, 2192, 2193, 2194, 2215, 2548, 2804, + 3060, 3316, 3572, 3828, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2216, 2549, 2805, + 3061, 3317, 3573, 3829, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2203, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2217, 2550, 2806, + 3062, 3318, 3574, 3830, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2211, 2212, 2213, 2218, 2219, 2220, 2221, 2222, 2249, 2551, 2807, + 3063, 3319, 3575, 3831, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2223, 2224, 2225, 2226, 2227, 2228, 2229, 2230, 2250, 2552, 2808, + 3064, 3320, 3576, 3832, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2251, 2553, 2809, + 3065, 3321, 3577, 3833, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2252, 2554, 2810, + 3066, 3322, 3578, 3834, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2247, 2248, 2253, 2254, 2255, 2256, 2257, 2258, 2284, 2555, 2811, + 3067, 3323, 3579, 3835, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2285, 2556, 2812, + 3068, 3324, 3580, 3836, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2286, 2557, 2813, + 3069, 3325, 3581, 3837, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2287, 2558, 2814, + 3070, 3326, 3582, 3838, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2283, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2303, 2559, 2815, + 3071, 3327, 3583, 3839, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2304, 2560, 2816, + 3072, 3328, 3584, 3840, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] + + page_indices_no_dp = sum(page_indices_no_dp, [] ) + if dp_size == 1: + page_indices = jnp.array(page_indices_no_dp, dtype=jnp.int32) + elif dp_size == 2: + page_indices = jnp.array(page_indices_rank0 + page_indices_rank1, dtype=jnp.int32) + + attention_sink = None + sm_scale = head_dim ** -0.5 + + print(f"\nInput creation (dp_size={dp_size}):") + print(f" q: {q.shape}") + print(f" kv_cache: {kv_cache.shape}") + print(f" kv_lens: {kv_lens.shape} = {kv_lens}") + print(f" page_indices: {page_indices.shape}", page_indices) + print(f" cu_q_lens: {cu_q_lens.shape} = {cu_q_lens}") + print(f" distribution: {distribution.shape} = {distribution}") + + return { + 'q': q, + 'k': k, + 'v': v, + 'kv_cache': kv_cache, + 'kv_lens': kv_lens, + 'page_indices': page_indices, + 'cu_q_lens': cu_q_lens, + 'distribution': distribution, + 'attention_sink': attention_sink, + 'sm_scale': sm_scale, + } + +def run(inputs, dump_dir, dp_size=2, num_devices=8, dp=True): + + print("\n" + "="*80) + if dp: + print(f"Running WITH Data Parallelism (dp_size={dp_size}, num_devices={num_devices})") + else: + print(f"Running WITHOUT Data Parallelism (dp_size={dp_size}, num_devices={num_devices})") + print("="*80) + + print(f"Input shapes:") + for key, val in inputs.items(): + if isinstance(val, jax.Array): + print(f" {key}: {val.shape} {val.dtype}") + + devices = jax.devices() + dp_size = 2 + if dp: + device_array = np.array(devices[:8]).reshape(dp_size, -1) + else: + device_array = np.array(devices[:4]).reshape(1, -1) + mesh = Mesh( + device_array, + axis_names=(ShardingAxisName.ATTN_DATA, ShardingAxisName.ATTN_HEAD), + ) + print(f"Mesh: {mesh}") + + def _attention_wrapper(q, k, v, kv_cache, kv_lens, page_indices, cu_q_lens, distribution): + + attention_metadata = AttentionMetadata( + input_positions=None, + block_tables=page_indices, + seq_lens=kv_lens, + query_start_loc=cu_q_lens, + request_distribution=distribution, + ) + + batch_size, num_heads, head_dim = q.shape + num_kv_heads = k.shape[1] + + # Reshape q, k, v to match _jax_attn_func expectations + q_reshaped = q.reshape(batch_size, num_heads * head_dim) + k_reshaped = k.reshape(batch_size, num_kv_heads * head_dim) + v_reshaped = v.reshape(batch_size, num_kv_heads * head_dim) + + new_kv_cache, outputs = _jax_attn_func( + kv_cache=kv_cache, + q=q_reshaped, + k=k_reshaped, + v=v_reshaped, + sinks=None, + attention_metadata=attention_metadata, + mesh=mesh, + scale=inputs['sm_scale'], + head_size=head_dim, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + q_scale=None, + k_scale=None, + v_scale=None, + sliding_window=None, + ) + + outputs = outputs.reshape(batch_size, num_heads, head_dim) + + return outputs, new_kv_cache + + attention_fn = jax.jit(_attention_wrapper) + + # Lower and dump HLO + print("\nLowering computation...") + lowered = attention_fn.lower( + inputs['q'], + inputs['k'], + inputs['v'], + inputs['kv_cache'], + inputs['kv_lens'], + inputs['page_indices'], + inputs['cu_q_lens'], + inputs['distribution'], + ) + + # Dump HLO + hlo_text = lowered.as_text() + if dp: + hlo_path = Path(dump_dir) / f"with_dp_{dp_size}_hlo.txt" + else: + hlo_path = Path(dump_dir) / f"no_dp_hlo.txt" + hlo_path.write_text(hlo_text) + print(f"✓ HLO dumped to: {hlo_path}") + + qkv_spec = P(ShardingAxisName.ATTN_DATA, ShardingAxisName.ATTN_HEAD, None) + kv_cache_spec = P(ShardingAxisName.ATTN_DATA, None, ShardingAxisName.ATTN_HEAD, None, None) + + inputs['q'] = jax.device_put(inputs['q'], NamedSharding(mesh, qkv_spec)) + inputs['k'] = jax.device_put(inputs['k'], NamedSharding(mesh, qkv_spec)) + inputs['v'] = jax.device_put(inputs['v'], NamedSharding(mesh, qkv_spec)) + inputs['kv_cache'] = jax.device_put(inputs['kv_cache'], NamedSharding(mesh, kv_cache_spec)) + inputs['kv_lens'] = jax.device_put(inputs['kv_lens'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) + inputs['page_indices'] = jax.device_put(inputs['page_indices'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) + inputs['cu_q_lens'] = jax.device_put(inputs['cu_q_lens'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) + inputs['distribution'] = jax.device_put(inputs['distribution'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) + + # Warm up + print("Warming up...") + for _ in range(3): + result = attention_fn( + inputs['q'], + inputs['k'], + inputs['v'], + inputs['kv_cache'], + inputs['kv_lens'], + inputs['page_indices'], + inputs['cu_q_lens'], + inputs['distribution'], + ) + jax.block_until_ready(result) + + # Benchmark + print("Benchmarking ...") + times = [] + dump_dir = f"gs://wenxindong-vm/trace/debug/gptoss/kernel/dp{dp}" + jax.profiler.start_trace(dump_dir) + for i in range(10): + start = time.time() + result = attention_fn( + inputs['q'], + inputs['k'], + inputs['v'], + inputs['kv_cache'], + inputs['kv_lens'], + inputs['page_indices'], + inputs['cu_q_lens'], + inputs['distribution'], + ) + jax.block_until_ready(result) + elapsed = time.time() - start + times.append(elapsed) + print(f" Run {i+1}: {elapsed*1000:.2f} ms") + jax.profiler.stop_trace() + avg_time = sum(times) / len(times) + print(f"\n✓ Average time: {avg_time*1000:.2f} ms") + + return result, avg_time + + +def main(): + parser = argparse.ArgumentParser(description="Reproduce attention DP issue") + parser.add_argument( + '--mode', + type=str, + choices=['no_dp', 'with_dp', 'both'], + default='both', + help='Which mode to run' + ) + parser.add_argument( + '--dump_dir', + type=str, + default='/tmp/attention_dp_repro', + help='Directory to dump HLO files' + ) + args = parser.parse_args() + + # Create dump directory + dump_dir = Path(args.dump_dir) + dump_dir.mkdir(parents=True, exist_ok=True) + print(f"HLO dump directory: {dump_dir}") + + # Note: We'll create DP-specific inputs when running with_dp mode + inputs_no_dp = None + inputs_with_dp = None + + print(f"\nJAX version: {jax.__version__}") + print(f"Available devices: {jax.devices()}") + print(f"Device count: {jax.device_count()}") + + results = {} + + # Run without DP + if args.mode in ['no_dp', 'both']: + print("\nCreating inputs for no_dp mode...") + inputs_no_dp = create_test_inputs(dp_size=1) + result, avg_time = run(inputs_no_dp, dump_dir, dp=False) + results['no_dp'] = {'result': result, 'time': avg_time} + + # Run with DP + if args.mode in ['with_dp', 'both']: + print(f"\nCreating inputs for with_dp mode (dp_size={2})...") + inputs_with_dp = create_test_inputs( dp_size=2) + result, avg_time = run(inputs_with_dp, dump_dir, dp=True) + results['with_dp'] = {'result': result, 'time': avg_time} + + # Compare results + if args.mode == 'both': + print("\n" + "="*80) + print("COMPARISON") + print("="*80) + + no_dp_time = results['no_dp']['time'] + with_dp_time = results['with_dp']['time'] + speedup = no_dp_time / with_dp_time + slowdown = with_dp_time / no_dp_time + + print(f"No DP time: {no_dp_time*1000:.2f} ms") + print(f"With DP time: {with_dp_time*1000:.2f} ms") + + if speedup > 1.0: + print(f"✓ DP is {speedup:.2f}x FASTER") + else: + print(f"✗ DP is {slowdown:.2f}x SLOWER") + + print("\n" + "="*80) + print(f"\n1. Compare HLO files in: {dump_dir}") + print(f" - no_dp_hlo.txt") + print(f" - with_dp_2_hlo.txt") + print(f"\n2. Or use XLA dump for more detail:") + print(f" XLA_FLAGS='--xla_dump_to={dump_dir}/xla_dump --xla_dump_hlo_as_text' python {__file__} --mode {args.mode}") + print() + + +if __name__ == '__main__': + main() + + +# ================================================================================ +# COMPARISON +# ================================================================================ +# No DP time: 2.63 ms +# With DP time: 2.62 ms +# ✓ DP is 1.00x FASTER \ No newline at end of file diff --git a/tpu_inference/layers/common/sharding.py b/tpu_inference/layers/common/sharding.py index 1a1a8d169..930096156 100644 --- a/tpu_inference/layers/common/sharding.py +++ b/tpu_inference/layers/common/sharding.py @@ -121,6 +121,7 @@ def from_vllm_config(cls, if enable_dp_attention: # Replicate attention layer when num_kv_heads < TP num_kv_heads = vllm_config.model_config.get_total_num_kv_heads() + kv_dtype = utils.get_jax_dtype_from_str_dtype( vllm_config.cache_config.cache_dtype) or jnp.bfloat16 packing = 4 // jnp.dtype(kv_dtype).itemsize @@ -128,6 +129,11 @@ def from_vllm_config(cls, # duplicate KV heads across devices, wasting kv cache memory. # Use attention DP instead to reduce per-device num_kv_heads and # eliminate this waste. + + # if head_dim is 64, multiply packing by 2 + if vllm_config.model_config.get_head_size() == 64: + packing *= 2 + num_kv_heads_per_device_in_kv_cache = (num_kv_heads * 2) / packing attn_dp = max( int(tensor_parallelism // num_kv_heads_per_device_in_kv_cache), @@ -166,10 +172,10 @@ def validate(cls, vllm_config, sharding_strategy): f"LoRA is not supported with data parallelism " f"(DP size: {total_dp_size}). Please disable LoRA or " f"set data parallelism to 1.") - if not os.environ.get("NEW_MODEL_DESIGN", False): - raise ValueError( - "Must run DP with NEW_MODEL_DESIGN enabled. Please set the " - "NEW_MODEL_DESIGN=True.") + # if not os.environ.get("NEW_MODEL_DESIGN", False): + # raise ValueError( + # "Must run DP with NEW_MODEL_DESIGN enabled. Please set the " + # "NEW_MODEL_DESIGN=True.") @property def total_dp_size(self) -> int: diff --git a/tpu_inference/layers/vllm/fused_moe.py b/tpu_inference/layers/vllm/fused_moe.py index fa9a45288..9f676f4db 100644 --- a/tpu_inference/layers/vllm/fused_moe.py +++ b/tpu_inference/layers/vllm/fused_moe.py @@ -8,7 +8,7 @@ from tpu_inference.layers.vllm.linear_common import \ slice_sharded_tensor_for_concatenation - +from tpu_inference.layers.common.sharding import ShardingAxisName P = PartitionSpec @@ -110,7 +110,7 @@ def tensor_sharded_gmm_merged_column_parallel( # adapted from https://github.com/pytorch/xla/blob/1d409399474197c484894be90b75d9855393dda5/torch_xla/experimental/custom_kernel.py#L1401 m, k, g = lhs.shape[0], lhs.shape[1], rhs.shape[0] n = rhs.shape[1] if transpose_rhs else rhs.shape[2] - tm, tk, tn = _get_tiling_size_for_gmm_kernel(m, k, n, g) + tm, tk, tn = _get_tiling_size_for_gmm_kernel(m//mesh.shape["data"], k, n, g) _gmm = functools.partial( gmm, @@ -123,16 +123,25 @@ def tensor_sharded_gmm_merged_column_parallel( gmm_result = shard_map( _gmm, mesh=mesh, - in_specs=(P(), P(None, "model", None), P()), - out_specs=(P(None, "model")), + in_specs=(P(ShardingAxisName.MLP_DATA, None), P(None, ShardingAxisName.MLP_TENSOR, None), P(ShardingAxisName.MLP_DATA)), + out_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR)), check_rep=False, )(lhs, rhs, group_sizes) if rhs_bias is not None: - rhs_bis = jnp.repeat(rhs_bias, group_sizes, 0, total_repeat_length=m) - gmm_result = (gmm_result + rhs_bis).astype(gmm_result.dtype) + def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): + rhs_bis = jnp.repeat(rhs_bias_local, group_sizes_global, 0, total_repeat_length=m//mesh.shape["data"]) + return (gmm_result_local + rhs_bis).astype(gmm_result_local.dtype) + + gmm_result = shard_map( + _add_bias, + mesh=mesh, + in_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR), P(None, ShardingAxisName.MLP_TENSOR), P(ShardingAxisName.MLP_DATA)), + out_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR)), + check_rep=False, + )(gmm_result, rhs_bias, group_sizes) - n_shards = mesh.shape["model"] + n_shards = mesh.shape['model'] * mesh.shape.get('attn_dp', 1) output_sizes = [intermediate_size, intermediate_size] return slice_sharded_tensor_for_concatenation(gmm_result, output_sizes, @@ -150,7 +159,7 @@ def tensor_sharded_gmm_row_parallel( # adapted from https://github.com/pytorch/xla/blob/1d409399474197c484894be90b75d9855393dda5/torch_xla/experimental/custom_kernel.py#L1401 m, k, g = lhs.shape[0], lhs.shape[1], rhs.shape[0] n = rhs.shape[1] if transpose_rhs else rhs.shape[2] - tm, tk, tn = _get_tiling_size_for_gmm_kernel(m, k, n, g) + tm, tk, tn = _get_tiling_size_for_gmm_kernel(m//mesh.shape["data"], k, n, g) _gmm = functools.partial( gmm, @@ -162,19 +171,29 @@ def tensor_sharded_gmm_row_parallel( def _gmm_all_reduce(lhs, rhs, group_sizes): r = _gmm(lhs, rhs, group_sizes) - return jax.lax.psum(r, axis_name="model") + return jax.lax.psum(r, axis_name=ShardingAxisName.MLP_TENSOR) gmm_result = shard_map( _gmm_all_reduce, mesh=mesh, - in_specs=(P(None, "model"), P(None, None, "model"), P()), - out_specs=(P()), - check_rep=False, + in_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR), + P(None, None, ShardingAxisName.MLP_TENSOR), P(ShardingAxisName.MLP_DATA)), + out_specs=(P(ShardingAxisName.MLP_DATA)), + check_rep=False, )(lhs, rhs, group_sizes) - + # jax.debug.print("gmm_result before bias {} {}", gmm_result.sum(), gmm_result.ravel()[:10]) if rhs_bias is not None: - rhs_bias = jnp.repeat(rhs_bias, group_sizes, 0, total_repeat_length=m) - gmm_result = (gmm_result + rhs_bias).astype(gmm_result.dtype) + def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): + rhs_bis = jnp.repeat(rhs_bias_local, group_sizes_global, 0, total_repeat_length=m//mesh.shape["data"]) + return (gmm_result_local + rhs_bis).astype(gmm_result_local.dtype) + + gmm_result = shard_map( + _add_bias, + mesh=mesh, + in_specs=(P(ShardingAxisName.MLP_DATA), P(), P(ShardingAxisName.MLP_DATA)), + out_specs=(P(ShardingAxisName.MLP_DATA)), + check_rep=False, + )(gmm_result, rhs_bias, group_sizes) return gmm_result @@ -191,13 +210,12 @@ def expert_sharded_gmm( # adapted from https://github.com/pytorch/xla/blob/1d409399474197c484894be90b75d9855393dda5/torch_xla/experimental/custom_kernel.py#L1401 m, k, g = lhs.shape[0], lhs.shape[1], rhs.shape[0] n = rhs.shape[1] if transpose_rhs else rhs.shape[2] - tm, tk, tn = _get_tiling_size_for_gmm_kernel(m, k, n, g) + tm, tk, tn = _get_tiling_size_for_gmm_kernel(m//mesh.shape["data"], k, n, g) num_experts_per_shard = num_experts // ep_size group_offset = jnp.arange(0, num_experts, num_experts_per_shard) group_offset = jax.lax.with_sharding_constraint( - group_offset, NamedSharding(mesh, P("model"))) - + group_offset, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) def _gmm(lhs, rhs, group_sizes, group_offset): # Group offset for this shard. `group_offset` is sharded, and in this # sharded function, it has only 1 element and `group_offset.shape` is @@ -236,8 +254,9 @@ def _gmm(lhs, rhs, group_sizes, group_offset): gmm_res = shard_map( _gmm, mesh=mesh, - in_specs=(P(), P("model", None, None), P(), P("model")), - out_specs=(P("model", None)), + in_specs=(P(ShardingAxisName.MLP_DATA, None), P(ShardingAxisName.EXPERT, None, + None), P(ShardingAxisName.MLP_DATA), P(ShardingAxisName.EXPERT)), + out_specs=(P(ShardingAxisName.EXPERT, None)), check_rep=False, )(lhs, rhs, group_sizes, group_offset) @@ -256,12 +275,11 @@ def _gmm(lhs, rhs, group_sizes, group_offset): recv_sizes = send_sizes input_offsets = jax.lax.with_sharding_constraint( - input_offsets, NamedSharding(mesh, P("model"))) + input_offsets, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) send_sizes = jax.lax.with_sharding_constraint( - send_sizes, NamedSharding(mesh, P("model"))) + send_sizes, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) output_offsets = jax.lax.with_sharding_constraint( - output_offsets, NamedSharding(mesh, P("model"))) - + output_offsets, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) def _ragged_all_to_all(operand, input_offsets, send_sizes, output_offsets, recv_sizes): output = jnp.zeros_like(operand) @@ -292,8 +310,7 @@ def _ragged_all_to_all(operand, input_offsets, send_sizes, output_offsets, send_sizes_of_shard, output_offsets_of_shard, recv_sizes_of_shard, - axis_name="model") - + axis_name=ShardingAxisName.EXPERT) # Use ragged_all_to_all to send the result from gmm for each expert to all # the shards. In the working example, the result would be: # A, A, A, A A, A, A, A A, A, A, A A, A, A, A @@ -314,7 +331,8 @@ def _ragged_all_to_all(operand, input_offsets, send_sizes, output_offsets, return shard_map( _ragged_all_to_all, mesh=mesh, - in_specs=(P("model", None), P("model"), P("model"), P("model"), P()), + in_specs=(P(ShardingAxisName.EXPERT, None), P(ShardingAxisName.EXPERT), + P(ShardingAxisName.EXPERT), P(ShardingAxisName.EXPERT), P()), out_specs=(P()), check_rep=False, )(gmm_res, input_offsets, send_sizes, output_offsets, recv_sizes) @@ -350,13 +368,12 @@ def fused_moe_func( hidden_size = hidden_states.shape[-1] num_tokens = hidden_states.size // hidden_size assert global_num_experts == w1.shape[0] - ep_size = mesh.shape["model"] # only used if use_ep is True. + ep_size = mesh.shape['model'] * mesh.shape.get("attn_dp", 1) # only used if use_ep is True. intermediate_size = w2.shape[-1] dtype = hidden_states.dtype assert (num_tokens * topk) % 16 == 0, ( "The kernel requires num_tokens * topk to be a multiple of " f"16 but got {num_tokens}*{topk}={num_tokens*topk}") - hidden_states = hidden_states.reshape(num_tokens, hidden_size) gating_output = gating_output.reshape(num_tokens, global_num_experts) @@ -366,15 +383,28 @@ def fused_moe_func( topk_weights = topk_weights / topk_weights.sum(axis=-1, keepdims=True) topk_weights = topk_weights.astype(dtype) - topk_indices_flat = topk_indices.flatten() - topk_argsort_indices = jnp.argsort(topk_indices_flat) - topk_argsort_revert_indices = jnp.argsort(topk_argsort_indices) - token_indices = jnp.arange(num_tokens, dtype=jnp.int32).repeat(topk) - token_indices_sorted = token_indices[topk_argsort_indices] - group_sizes = jnp.bincount(topk_indices_flat, length=global_num_experts) - - x = hidden_states[token_indices_sorted] - + def _process_tokens_locally(hidden_states_local, topk_indices_local): + num_tokens_local = hidden_states_local.shape[0] + topk_indices_flat = topk_indices_local.flatten() + topk_argsort_indices = jnp.argsort(topk_indices_flat) + topk_argsort_revert_indices = jnp.argsort(topk_argsort_indices) + token_indices = jnp.arange(num_tokens_local, dtype=jnp.int32).repeat(topk) + token_indices_sorted = token_indices[topk_argsort_indices] + group_sizes_local = jnp.bincount(topk_indices_flat, length=global_num_experts) + + x = hidden_states_local[token_indices_sorted] + return x, group_sizes_local, topk_argsort_revert_indices + + x, group_sizes, topk_argsort_revert_indices = shard_map( + _process_tokens_locally, + mesh=mesh, + in_specs=(P(ShardingAxisName.ATTN_DATA, None), P(ShardingAxisName.ATTN_DATA, None)), + out_specs=(P(ShardingAxisName.ATTN_DATA, None), P(ShardingAxisName.ATTN_DATA), P(ShardingAxisName.ATTN_DATA)), + check_rep=False, + )(hidden_states, topk_indices) + + # jax.debug.print("hidden_state before gmm {} {}", x.sum(), x.ravel()[:10]) + # jax.debug.print("group_sizes {} {}", group_sizes.sum(), group_sizes) if use_ep: x = expert_sharded_gmm( x, @@ -396,9 +426,11 @@ def fused_moe_func( mesh=mesh, intermediate_size=intermediate_size, ) + # jax.debug.print("hidden_state after first gmm x1 {} {}", x1.sum(), x1.ravel()[:10]) + # jax.debug.print("hidden_state after first gmm x2 {} {}", x2.sum(), x2.ravel()[:10]) x = activation_fn(activation, x1, x2) - + # jax.debug.print("hidden_state after activation {} {}", x.sum(), x.ravel()[:10]) if use_ep: x = expert_sharded_gmm( x, @@ -411,7 +443,7 @@ def fused_moe_func( ) else: x = jax.lax.with_sharding_constraint( - x, NamedSharding(mesh, P(None, "model"))) + x, NamedSharding(mesh, P(ShardingAxisName.MLP_DATA, "model"))) x = tensor_sharded_gmm_row_parallel( x, w2, @@ -420,14 +452,27 @@ def fused_moe_func( transpose_rhs=True, mesh=mesh, ) - - x = x[topk_argsort_revert_indices].reshape(-1, topk, hidden_size) - x = x * jnp.expand_dims(topk_weights, axis=-1) - x = x.sum(axis=-2) + # jax.debug.print("hidden_state after second gmm {} {}", x.sum(), x.ravel()[:10]) + + def _finalize_output(x_local, topk_argsort_revert_indices_local, topk_weights_local): + x_local = x_local[topk_argsort_revert_indices_local].reshape(-1, topk, hidden_size) + x_local = x_local * jnp.expand_dims(topk_weights_local, axis=-1) + x_local = x_local.sum(axis=-2) + return x_local + + x = shard_map( + _finalize_output, + mesh=mesh, + in_specs=(P(ShardingAxisName.ATTN_DATA, None), P(ShardingAxisName.ATTN_DATA), P(ShardingAxisName.ATTN_DATA, None)), + out_specs=(P(ShardingAxisName.ATTN_DATA, None)), + check_rep=False, + )(x, topk_argsort_revert_indices, topk_weights) + # jax.debug.print("hidden_state after finalize output {} {}", x.sum(), x.ravel()[:10]) x = x.reshape(orig_shape) if reduce_results: - x = jax.lax.with_sharding_constraint(x, NamedSharding(mesh, P())) + x = jax.lax.with_sharding_constraint(x, NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) + # jax.debug.print("hidden_state after reducing result {} {}", x.sum(), x.ravel()[:10]) return x diff --git a/tpu_inference/layers/vllm/quantization/common.py b/tpu_inference/layers/vllm/quantization/common.py index 381dce392..7872c4aaa 100644 --- a/tpu_inference/layers/vllm/quantization/common.py +++ b/tpu_inference/layers/vllm/quantization/common.py @@ -11,6 +11,7 @@ ReplicatedLinear, RowParallelLinear) +from tpu_inference.layers.common.sharding import ShardingAxisName from tpu_inference.layers.vllm.linear_common import \ get_model_matmul_fusion_assignment from tpu_inference.utils import TPU_SECOND_LAST_MINOR @@ -34,15 +35,23 @@ def __init__(self, vllm_config: VllmConfig, mesh: Mesh, layer: LinearBase): self.enable_sequence_parallelism = vllm_config.compilation_config.pass_config.enable_sequence_parallelism self.input_sharding = None self.output_sharding = None + self.tp_size = self.mesh.shape['model'] * self.mesh.shape.get( + 'attn_dp', 1) if isinstance(layer, RowParallelLinear): - self.weight_sharding = P(None, "model") + self.weight_sharding = P(None, ShardingAxisName.MLP_TENSOR) if self.enable_sequence_parallelism: - self.output_sharding = P("model", None) + self.output_sharding = P(ShardingAxisName.MLP_TENSOR, None) elif isinstance(layer, ColumnParallelLinear): - self.weight_sharding = P("model", None) + if isinstance(layer, QKVParallelLinear): + self.input_sharding = P(ShardingAxisName.ATTN_DATA, None) + self.weight_sharding = P('model', None) + self.output_sharding = P(ShardingAxisName.ATTN_DATA, "model") + else: + self.weight_sharding = P(ShardingAxisName.MLP_TENSOR, None) + if self.enable_sequence_parallelism: - self.input_sharding = P("model", None) + self.input_sharding = P(ShardingAxisName.MLP_TENSOR, None) if isinstance(layer, MergedColumnParallelLinear) or isinstance( layer, QKVParallelLinear): @@ -61,13 +70,18 @@ def __init__(self, vllm_config: VllmConfig, mesh: Mesh, layer: LinearBase): " bad performance.", type(layer)) self.bias_sharding = P(self.weight_sharding[0]) - self.n_shards = self.mesh.shape.get(self.weight_sharding[0], 1) + if isinstance(self.weight_sharding[0], tuple): + self.n_shards = 1 + for axis in self.weight_sharding[0]: + self.n_shards *= self.mesh.shape.get(axis, 1) + else: + self.n_shards = self.mesh.shape.get(self.weight_sharding[0], 1) def get_input_sharding(self, x: torchax.tensor.Tensor): if self.enable_sequence_parallelism: token_num = x.shape[0] # NOTE(chengjiyao): make sure the sharded token_num is larger than TPU_SECOND_LAST_MINOR - if token_num // self.mesh.shape["model"] >= TPU_SECOND_LAST_MINOR: + if token_num // self.tp_size >= TPU_SECOND_LAST_MINOR: return self.input_sharding else: return None @@ -77,7 +91,7 @@ def get_output_sharding(self, x: torchax.tensor.Tensor): if self.enable_sequence_parallelism: token_num = x.shape[0] # NOTE(chengjiyao): make sure the sharded token_num is larger than TPU_SECOND_LAST_MINOR - if token_num // self.mesh.shape["model"] >= TPU_SECOND_LAST_MINOR: + if token_num // self.tp_size >= TPU_SECOND_LAST_MINOR: return self.output_sharding else: return None diff --git a/tpu_inference/layers/vllm/quantization/unquantized.py b/tpu_inference/layers/vllm/quantization/unquantized.py index 8f2a622cd..674c9b139 100644 --- a/tpu_inference/layers/vllm/quantization/unquantized.py +++ b/tpu_inference/layers/vllm/quantization/unquantized.py @@ -25,6 +25,8 @@ from tpu_inference.kernels.fused_moe.v1.kernel import fused_ep_moe from tpu_inference.layers.common.quant_methods import (UNQUANTIZED, get_tpu_quant_method) +from tpu_inference.layers.common.sharding import ShardingAxisName + from tpu_inference.layers.vllm.fused_moe import fused_moe_func_padded from tpu_inference.layers.vllm.linear_common import ( reorder_concatenated_tensor_for_sharding, @@ -260,38 +262,42 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: w13_weight = jax.device_put( w13_weight, Format(Layout((0, 1, 2)), - NamedSharding(self.mesh, P("model", None, None)))) + NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None, None)))) w2_weight = jax.device_put( w2_weight, Format(Layout((0, 1, 2)), - NamedSharding(self.mesh, P("model", None, None)))) + NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None, None)))) if self.moe.has_bias: w13_bias = jax.device_put( w13_bias, Format(Layout((0, 1)), - NamedSharding(self.mesh, P("model", None)))) + NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None)))) w2_bias = jax.device_put( w2_bias, Format(Layout((0, 1)), - NamedSharding(self.mesh, P("model", None)))) + NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None)))) else: intermediate_size = w13_weight.shape[1] // 2 assert intermediate_size == w2_weight.shape[-1] output_sizes = [intermediate_size, intermediate_size] - n_shards = self.mesh.shape["model"] + n_shards = self.mesh.shape['model'] * self.mesh.shape.get( + "attn_dp", 1) assert intermediate_size % n_shards == 0 w13_weight = reorder_concatenated_tensor_for_sharding( w13_weight, output_sizes, n_shards, dim=1) w13_weight = jax.device_put( w13_weight, - Format(Layout((0, 1, 2)), - NamedSharding(self.mesh, P(None, "model", None)))) + Format( + Layout((0, 1, 2)), + NamedSharding( + self.mesh, + P(None, ShardingAxisName.MLP_TENSOR, None)))) w2_weight = jax.device_put( w2_weight, Format(Layout((0, 1, 2)), - NamedSharding(self.mesh, P(None, None, "model")))) + NamedSharding(self.mesh, P(None, None,ShardingAxisName.MLP_TENSOR)))) if self.moe.has_bias: w13_bias = reorder_concatenated_tensor_for_sharding( @@ -299,7 +305,7 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: w13_bias = jax.device_put( w13_bias, Format(Layout((0, 1)), - NamedSharding(self.mesh, P(None, "model")))) + NamedSharding(self.mesh, P(None, ShardingAxisName.MLP_TENSOR)))) w2_bias = jax.device_put( w2_bias, Format(Layout((0, 1)), diff --git a/tpu_inference/layers/vllm/sharding.py b/tpu_inference/layers/vllm/sharding.py index b9fd4fdd9..043e6a64f 100644 --- a/tpu_inference/layers/vllm/sharding.py +++ b/tpu_inference/layers/vllm/sharding.py @@ -19,6 +19,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding) +from tpu_inference.layers.common.sharding import ShardingAxisName from tpu_inference import envs from tpu_inference.logger import init_logger @@ -109,7 +110,8 @@ def _shard_tensor_to_tpu_replicated(tensor: torch.Tensor, def _shard_vocab_parallel_embedding(layer: VocabParallelEmbedding, mesh: Mesh) -> None: weight = _convert_to_torchax_and_shard( - layer.weight, NamedSharding(mesh, P('model', None))) + layer.weight, NamedSharding(mesh, P(ShardingAxisName.MLP_TENSOR, + None))) layer.weight = Parameter(weight, requires_grad=False) @@ -118,11 +120,12 @@ def _shard_lm_head(layer: ParallelLMHead, mesh: Mesh): # if that config is set, then we should not create new weights but reuse the # weight from VocabParallelEmbedding weight = _convert_to_torchax_and_shard( - layer.weight, NamedSharding(mesh, P('model', None))) + layer.weight, NamedSharding(mesh, P(ShardingAxisName.MLP_TENSOR, + None))) layer.weight = Parameter(weight, requires_grad=False) if layer.bias is not None: - bias = _convert_to_torchax_and_shard(layer.bias, - NamedSharding(mesh, P('model'))) + bias = _convert_to_torchax_and_shard( + layer.bias, NamedSharding(mesh, P(ShardingAxisName.MLP_TENSOR))) layer.bias = Parameter(bias, requires_grad=False) From 3eada3d9fe2e603ba6ac80f4aeba271f9dc3e6cc Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Wed, 19 Nov 2025 23:37:02 +0000 Subject: [PATCH 2/9] wip --- repro_attention_dp.py | 1698 ----------------------------------------- 1 file changed, 1698 deletions(-) delete mode 100644 repro_attention_dp.py diff --git a/repro_attention_dp.py b/repro_attention_dp.py deleted file mode 100644 index 584e676a3..000000000 --- a/repro_attention_dp.py +++ /dev/null @@ -1,1698 +0,0 @@ -""" -Reproduction script to compare attention kernel performance and HLO -with and without data parallelism. - -Usage: - python repro_attention_dp.py -""" - -import argparse -import time -from pathlib import Path -import numpy as np - -import jax -import jax.numpy as jnp -from jax.sharding import Mesh, PartitionSpec as P -from jax import NamedSharding, shard_map -from tpu_inference.layers.vllm.attention import _jax_attn_func -# import sys -# sys.path.insert(0, '/home/wenxindong_google_com/tpu-inference') - -from tpu_inference.kernels.ragged_paged_attention.v3.kernel_hd64 import ( - get_kv_cache_shape as get_kv_cache_shape_h64, -) -from tpu_inference.kernels.ragged_paged_attention.v3.kernel import get_kv_cache_shape -from tpu_inference.layers.common.sharding import ShardingAxisName -from tpu_inference.layers.common.attention_metadata import AttentionMetadata - - -def create_test_inputs( - batch_size: int = 256, - max_num_tokens: int = 256, - num_q_heads: int = 32, - num_kv_heads: int = 8, - head_dim: int = 64, - page_size: int = 256, - total_num_pages: int = 8282, - dp_size: int = 1, -): - - total_num_pages = total_num_pages * dp_size - # Q, K, V tensors - q = jnp.concatenate([jax.random.normal( - jax.random.PRNGKey(0), - (max_num_tokens, num_q_heads, head_dim), - dtype=jnp.bfloat16, - ) for i in range(dp_size)], axis=0) - k = jnp.concatenate([jax.random.normal( - jax.random.PRNGKey(0), - (max_num_tokens, num_kv_heads, head_dim), - dtype=jnp.bfloat16, - ) for i in range(dp_size)], axis=0) - v = jnp.concatenate([jax.random.normal( - jax.random.PRNGKey(0), - (max_num_tokens, num_kv_heads, head_dim), - dtype=jnp.bfloat16, - ) for i in range(dp_size)], axis=0) - - # KV cache - get_kv_cache_shape_fn = get_kv_cache_shape_h64 if head_dim == 64 else get_kv_cache_shape - kv_cache_shape = get_kv_cache_shape_fn( - total_num_pages, - page_size, - num_kv_heads, - head_dim, - jnp.bfloat16, - ) - kv_cache = jax.random.normal(jax.random.PRNGKey(0), kv_cache_shape, dtype=jnp.bfloat16) - - - # Actual seq_lens from the log for step 849 - seq_lens_rank0 = [2897, 2896, 2896, 2896, 2895, 2895, 2895, 2895, 2894, 2894, 2894, 2894, 2893, 2893, - 2893, 2893, 2892, 2892, 2892, 2892, 2891, 2891, 2891, 2891, 2890, 2890, 2890, 2890, - 2889, 2889, 2889, 2889, 2888, 2888, 2888, 2888, 2887, 2887, 2887, 2887, 2886, 2886, - 2886, 2886, 2885, 2885, 2885, 2885, 2884, 2884, 2884, 2884, 2883, 2883, 2883, 2883, - 2882, 2882, 2882, 2882, 2881, 2881, 2881, 2881, 2880, 2880, 2880, 2880, 2879, 2879, - 2879, 2879, 2878, 2878, 2878, 2878, 2877, 2877, 2877, 2877, 2876, 2876, 2876, 2876, - 2875, 2875, 2875, 2875, 2874, 2874, 2874, 2874, 2873, 2873, 2873, 2873, 2872, 2872, - 2872, 2872, 2871, 2871, 2871, 2871, 2870, 2870, 2870, 2870, 2869, 2869, 2869, 2869, - 2868, 2868, 2868, 2868, 2867, 2867, 2867, 2867, 2866, 2866, 2866, 2866, 2865, 2865, - 2865, 2865, 2864, 2864, 2864, 2863, 2863, 2863, 2863, 2862, 2862, 2862, 2862, 2861, - 2861, 2861, 2861, 2860, 2860, 2860, 2860, 2859, 2859, 2859, 2859, 2858, 2858, 2858, - 2858, 2857, 2857, 2857, 2857, 2856, 2856, 2856, 2856, 2855, 2855, 2855, 2855, 2854, - 2854, 2854, 2854, 2853, 2853, 2853, 2853, 2852, 2852, 2852, 2852, 2851, 2851, 2851, - 2850, 2850, 2850, 2850, 2849, 2849, 2849, 2849, 2848, 2848, 2848, 2848, 2847, 2847, - 2847, 2847, 2846, 2846, 2846, 2846, 2845, 2845, 2845, 2845, 2844, 2844, 2844, 2844, - 2843, 2843, 2843, 2843, 2842, 2842, 2842, 2842, 2841, 2841, 2841, 2841, 2840, 2840, - 2840, 2839, 2839, 2839, 2839, 2838, 2838, 2838, 2838, 2837, 2837, 2837, 2837, 2836, - 2836, 2836, 2836, 2835, 2835, 2835, 2835, 2834, 2834, 2834, 2834, 2833, 2833, 2833, - 2833, 2832, 2832, 2832] - - seq_lens_rank1 = [2896, 2896, 2896, 2896, 2895, 2895, 2895, 2894, 2894, 2894, 2894, 2893, 2893, 2893, - 2893, 2892, 2892, 2892, 2892, 2891, 2891, 2891, 2891, 2890, 2890, 2890, 2890, 2889, - 2889, 2889, 2889, 2888, 2888, 2888, 2888, 2887, 2887, 2887, 2887, 2886, 2886, 2886, - 2886, 2885, 2885, 2885, 2885, 2884, 2884, 2884, 2884, 2883, 2883, 2883, 2883, 2882, - 2882, 2882, 2882, 2881, 2881, 2881, 2881, 2880, 2880, 2880, 2880, 2879, 2879, 2879, - 2879, 2878, 2878, 2878, 2878, 2877, 2877, 2877, 2877, 2876, 2876, 2876, 2876, 2875, - 2875, 2875, 2875, 2874, 2874, 2874, 2874, 2873, 2873, 2873, 2873, 2872, 2872, 2872, - 2872, 2871, 2871, 2871, 2871, 2870, 2870, 2870, 2870, 2869, 2869, 2869, 2869, 2868, - 2868, 2868, 2868, 2867, 2867, 2867, 2867, 2866, 2866, 2866, 2866, 2865, 2865, 2865, - 2865, 2864, 2864, 2864, 2863, 2863, 2863, 2863, 2862, 2862, 2862, 2862, 2861, 2861, - 2861, 2861, 2860, 2860, 2860, 2860, 2859, 2859, 2859, 2859, 2858, 2858, 2858, 2858, - 2857, 2857, 2857, 2857, 2856, 2856, 2856, 2856, 2855, 2855, 2855, 2855, 2854, 2854, - 2854, 2854, 2853, 2853, 2853, 2853, 2852, 2852, 2852, 2852, 2851, 2851, 2851, 2851, - 2850, 2850, 2850, 2849, 2849, 2849, 2849, 2848, 2848, 2848, 2848, 2847, 2847, 2847, - 2847, 2846, 2846, 2846, 2846, 2845, 2845, 2845, 2845, 2844, 2844, 2844, 2844, 2843, - 2843, 2843, 2843, 2842, 2842, 2842, 2842, 2841, 2841, 2841, 2841, 2840, 2840, 2840, - 2839, 2839, 2839, 2839, 2838, 2838, 2838, 2838, 2837, 2837, 2837, 2837, 2836, 2836, - 2836, 2836, 2835, 2835, 2835, 2835, 2834, 2834, 2834, 2834, 2833, 2833, 2833, 2833, - 2832, 2832, 2832, 2831] - # from step 1698 - seq_len_no_dp = [3745, 3745, 3743, 3743, 3743, 3742, 3742, 3742, 3742, 3741, 3741, - 3741, 3741, 3740, 3740, 3740, 3740, 3739, 3739, 3739, 3739, 3738, - 3738, 3738, 3738, 3737, 3737, 3737, 3737, 3736, 3736, 3736, 3736, - 3735, 3735, 3735, 3735, 3734, 3734, 3734, 3734, 3733, 3733, 3733, - 3733, 3732, 3732, 3732, 3732, 3731, 3731, 3731, 3731, 3730, 3730, - 3730, 3730, 3729, 3729, 3729, 3729, 3728, 3728, 3728, 3728, 3727, - 3727, 3727, 3727, 3726, 3726, 3726, 3726, 3725, 3725, 3725, 3725, - 3724, 3724, 3724, 3724, 3723, 3723, 3723, 3723, 3722, 3722, 3722, - 3722, 3721, 3721, 3721, 3721, 3720, 3720, 3720, 3720, 3719, 3719, - 3719, 3719, 3718, 3718, 3718, 3718, 3717, 3717, 3717, 3717, 3716, - 3716, 3716, 3716, 3715, 3715, 3715, 3715, 3714, 3714, 3714, 3714, - 3713, 3713, 3713, 3713, 3712, 3712, 3712, 3712, 3711, 3711, 3711, - 3710, 3710, 3710, 3710, 3709, 3709, 3709, 3709, 3708, 3708, 3708, - 3708, 3707, 3707, 3707, 3707, 3706, 3706, 3706, 3706, 3705, 3705, - 3705, 3705, 3704, 3704, 3704, 3704, 3703, 3703, 3703, 3703, 3702, - 3702, 3702, 3702, 3701, 3701, 3701, 3701, 3700, 3700, 3700, 3700, - 3699, 3699, 3699, 3699, 3698, 3698, 3698, 3697, 3697, 3697, 3697, - 3696, 3696, 3696, 3696, 3695, 3695, 3695, 3695, 3694, 3694, 3694, - 3694, 3693, 3693, 3693, 3693, 3692, 3692, 3692, 3692, 3691, 3691, - 3691, 3691, 3690, 3690, 3690, 3690, 3689, 3689, 3689, 3689, 3688, - 3688, 3688, 3687, 3687, 3687, 3687, 3686, 3686, 3686, 3686, 3685, - 3685, 3685, 3685, 3684, 3684, 3684, 3684, 3683, 3683, 3683, 3683, - 3682, 3682, 3682, 3682, 3681, 3681, 3681, 3681, 3680, 3680, 3680, - 3680, 3679, 3679] - - # query_start_loc is just [0, 1, 2, ..., 256] for both ranks - query_start_loc_per_rank = list(range(257)) - - # Construct kv_lens and cu_q_lens based on dp_size - if dp_size == 1: - kv_lens = jnp.array(seq_len_no_dp, dtype=jnp.int32) - cu_q_lens = jnp.array(query_start_loc_per_rank, dtype=jnp.int32) - elif dp_size == 2: - kv_lens = jnp.array(seq_lens_rank0 + seq_lens_rank1, dtype=jnp.int32) - cu_q_lens = jnp.array(query_start_loc_per_rank + query_start_loc_per_rank, dtype=jnp.int32) - - # Distribution: [0, 0, 256] per rank (all decode mode) - distribution_list = [] - for _ in range(dp_size): - distribution_list.extend([0, 0, batch_size]) - distribution = jnp.array(distribution_list, dtype=jnp.int32) - - - page_indices_rank0 = [[ 1, 2, 3, 4, 5, 6, 7 , 8, 9, 2305, 2561, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 10, 11, 12, 13, 14, 15, 16, 17, 42, 2306, 2562, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 18, 19, 20, 21, 22, 23, 24, 25, 43, 2307, 2563, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 26, 27, 28, 29, 30, 31, 32, 33, 44, 2308, 2564, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 34, 35, 36, 37, 38, 39, 40, 41, 77, 2309, 2565, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 45, 46, 47, 48, 49, 50, 51, 52, 78, 2310, 2566, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 53, 54, 55, 56, 57, 58, 59, 60, 79, 2311, 2567, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 61, 62, 63, 64, 65, 66, 67, 68, 80, 2312, 2568, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 69, 70, 71, 72, 73, 74, 75, 76, 113, 2313, 2569, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 81, 82, 83, 84, 85, 86 , 87 , 88, 114, 2314, 2570, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 89 , 90 , 91 , 92, 93, 94, 95, 96, 115, 2315, 2571, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 97 , 98 , 99, 100, 101, 102, 103, 104, 116, 2316, 2572, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 105, 106, 107, 108, 109, 110, 111, 112, 149, 2317, 2573, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 117, 118, 119, 120, 121, 122, 123, 124, 150, 2318, 2574, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 125, 126, 127, 128, 129, 130, 131, 132, 151, 2319, 2575, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 133, 134, 135, 136, 137, 138, 139, 140, 152, 2320, 2576, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 141, 142, 143, 144, 145, 146, 147, 148, 185, 2321, 2577, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 153, 154, 155, 156, 157, 158, 159, 160, 186, 2322, 2578, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 161, 162, 163, 164, 165, 166, 167, 168, 187, 2323, 2579, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 169, 170, 171, 172, 173, 174, 175, 176, 188, 2324, 2580, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 177, 178, 179, 180, 181, 182, 183, 184, 221, 2325, 2581, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 189, 190, 191, 192, 193, 194, 195, 196, 222, 2326, 2582, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 197, 198, 199, 200, 201, 202, 203, 204, 223, 2327, 2583, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 205, 206, 207, 208, 209, 210, 211, 212, 224, 2328, 2584, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 213, 214, 215, 216, 217, 218, 219, 220, 257, 2329, 2585, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 225, 226, 227, 228, 229, 230, 231, 232, 258, 2330, 2586, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 233, 234, 235, 236, 237, 238, 239, 240, 259, 2331, 2587, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 241, 242, 243, 244, 245, 246, 247, 248, 260, 2332, 2588, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 249, 250, 251, 252, 253, 254, 255, 256, 293, 2333, 2589, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 261, 262, 263, 264, 265, 266, 267, 268, 294, 2334, 2590, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 269, 270, 271, 272, 273, 274, 275, 276, 295, 2335, 2591, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 277, 278, 279, 280, 281, 282, 283, 284, 296, 2336, 2592, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 285, 286, 287, 288, 289, 290, 291, 292, 329, 2337, 2593, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 297, 298, 299, 300, 301, 302, 303, 304, 330, 2338, 2594, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 305, 306, 307, 308, 309, 310, 311, 312, 331, 2339, 2595, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 313, 314, 315, 316, 317, 318, 319, 320, 332, 2340, 2596, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 321, 322, 323, 324, 325, 326, 327, 328, 365, 2341, 2597, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 333, 334, 335, 336, 337, 338, 339, 340, 366, 2342, 2598, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 341, 342, 343, 344, 345, 346, 347, 348, 367, 2343, 2599, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 349, 350, 351, 352, 353, 354, 355, 356, 368, 2344, 2600, 2856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 357, 358, 359, 360, 361, 362, 363, 364, 401, 2345, 2601, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 369, 370, 371, 372, 373, 374, 375, 376, 402, 2346, 2602, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 377, 378, 379, 380, 381, 382, 383, 384, 403, 2347, 2603, 2859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 385, 386, 387, 388, 389, 390, 391, 392, 404, 2348, 2604, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 393, 394, 395, 396, 397, 398, 399, 400, 436, 2349, 2605, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 405, 406, 407, 408, 409, 410, 411, 412, 437, 2350, 2606, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 413, 414, 415, 416, 417, 418, 419, 420, 438, 2351, 2607, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 421, 422, 423, 424, 425, 426, 427, 428, 439, 2352, 2608, 2864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 429, 430, 431, 432, 433, 434, 435, 440, 472, 2353, 2609, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 441, 442, 443, 444, 445, 446, 447, 448, 473, 2354, 2610, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 449, 450, 451, 452, 453, 454, 455, 456, 474, 2355, 2611, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 457, 458, 459, 460, 461, 462, 463, 464, 475, 2356, 2612, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 465, 466, 467, 468, 469, 470, 471, 476, 508, 2357, 2613, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 477, 478, 479, 480, 481, 482, 483, 484, 509, 2358, 2614, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 485, 486, 487, 488, 489, 490, 491, 492, 510, 2359, 2615, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 493, 494, 495, 496, 497, 498, 499, 500, 511, 2360, 2616, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 501, 502, 503, 504, 505, 506, 507, 512, 544, 2361, 2617, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 513, 514, 515, 516, 517, 518, 519, 520, 545, 2362, 2618, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 521, 522, 523, 524, 525, 526, 527, 528, 546, 2363, 2619, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 529, 530, 531, 532, 533, 534, 535, 536, 547, 2364, 2620, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 537, 538, 539, 540, 541, 542, 543, 548, 580, 2365, 2621, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 549, 550, 551, 552, 553, 554, 555, 556, 581, 2366, 2622, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 557, 558, 559, 560, 561, 562, 563, 564, 582, 2367, 2623, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 565, 566, 567, 568, 569, 570, 571, 572, 583, 2368, 2624, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 573, 574, 575, 576, 577, 578, 579, 584, 615, 2369, 2625, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 585, 586, 587, 588, 589, 590, 591, 592, 616, 2370, 2626, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 593, 594, 595, 596, 597, 598, 599, 600, 617, 2371, 2627, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 601, 602, 603, 604, 605, 606, 607, 608, 618, 2372, 2628, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 609, 610, 611, 612, 613, 614, 619, 620, 651, 2373, 2629, 2885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 621, 622, 623, 624, 625, 626, 627, 628, 652, 2374, 2630, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 629, 630, 631, 632, 633, 634, 635, 636, 653, 2375, 2631, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 637, 638, 639, 640, 641, 642, 643, 644, 654, 2376, 2632, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 645, 646, 647, 648, 649, 650, 655, 656, 687, 2377, 2633, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 657, 658, 659, 660, 661, 662, 663, 664, 688, 2378, 2634, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 665, 666, 667, 668, 669, 670, 671, 672, 689, 2379, 2635, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 673, 674, 675, 676, 677, 678, 679, 680, 690, 2380, 2636, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 681, 682, 683, 684, 685, 686, 691, 692, 723, 2381, 2637, 2893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 693, 694, 695, 696, 697, 698, 699, 700, 724, 2382, 2638, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 701, 702, 703, 704, 705, 706, 707, 708, 725, 2383, 2639, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 709, 710, 711, 712, 713, 714, 715, 716, 726, 2384, 2640, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 717, 718, 719, 720, 721, 722, 727, 728, 758, 2385, 2641, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 729, 730, 731, 732, 733, 734, 735, 736, 759, 2386, 2642, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 737, 738, 739, 740, 741, 742, 743, 744, 760, 2387, 2643, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 745, 746, 747, 748, 749, 750, 751, 752, 761, 2388, 2644, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 753, 754, 755, 756, 757, 762, 763, 764, 794, 2389, 2645, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 765, 766, 767, 768, 769, 770, 771, 772, 795, 2390, 2646, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 773, 774, 775, 776, 777, 778, 779, 780, 796, 2391, 2647, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 781, 782, 783, 784, 785, 786, 787, 788, 797, 2392, 2648, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 789, 790, 791, 792, 793, 798, 799, 800, 830, 2393, 2649, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 801, 802, 803, 804, 805, 806, 807, 808, 831, 2394, 2650, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 809, 810, 811, 812, 813, 814, 815, 816, 832, 2395, 2651, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 817, 818, 819, 820, 821, 822, 823, 824, 833, 2396, 2652, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 825, 826, 827, 828, 829, 834, 835, 836, 865, 2397, 2653, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 837, 838, 839, 840, 841, 842, 843, 844, 866, 2398, 2654, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 845, 846, 847, 848, 849, 850, 851, 852, 867, 2399, 2655, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 853, 854, 855, 856, 857, 858, 859, 860, 868, 2400, 2656, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 861, 862, 863, 864, 869, 870, 871, 872, 901, 2401, 2657, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 873, 874, 875, 876, 877, 878, 879, 880, 902, 2402, 2658, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 881, 882, 883, 884, 885, 886, 887, 888, 903, 2403, 2659, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 889, 890, 891, 892, 893, 894, 895, 896, 904, 2404, 2660, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 897, 898, 899, 900, 905, 906, 907, 908, 936, 2405, 2661, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 909, 910, 911, 912, 913, 914, 915, 916, 937, 2406, 2662, 2918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 917, 918, 919, 920, 921, 922, 923, 924, 938, 2407, 2663, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 925, 926, 927, 928, 929, 930, 931, 932, 939, 2408, 2664, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 933, 934, 935, 940, 941, 942, 943, 944, 972, 2409, 2665, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 945, 946, 947, 948, 949, 950, 951, 952, 973, 2410, 2666, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 953, 954, 955, 956, 957, 958, 959, 960, 974, 2411, 2667, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 961, 962, 963, 964, 965, 966, 967, 968, 975, 2412, 2668, 2924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 969, 970, 971, 976, 977, 978, 979, 980, 1008, 2413, 2669, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 981, 982, 983, 984, 985, 986, 987, 988, 1009, 2414, 2670, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 989, 990, 991, 992, 993, 994, 995, 996, 1010, 2415, 2671, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 997, 998, 999, 1000, 1001, 1002,1003, 1004, 1011, 2416, 2672, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1005, 1006, 1007, 1012,1013, 1014, 1015, 1016, 1043, 2417, 2673, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1017, 1018, 1019, 1020, 1021, 1022,1023, 1024, 1044, 2418, 2674, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032,1045, 2419, 2675, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1046, 2420, 2676, 2932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1041, 1042,1047, 1048, 1049, 1050, 1051, 1052,1079, 2421, 2677, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1080, 2422, 2678, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1061, 1062,1063, 1064, 1065, 1066, 1067, 1068, 1081, 2423, 2679, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1069, 1070, 1071, 1072,1073, 1074, 1075, 1076, 1082, 2424, 2680, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1077, 1078, 1083, 1084, 1085, 1086, 1087, 1088, 1114, 2425, 2681, 2937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1089, 1090, 1091, 1092,1093, 1094, 1095, 1096, 1115, 2426, 2682, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1097, 1098, 1099, 1100, 1101, 1102,1103, 1104, 1116, 2427, 2683, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112,1117, 2428, 2684, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1113, 1118, 1119, 1120, 1121, 1122,1123, 1124, 1150, 2429, 2685, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132,1151, 2430, 2686, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1152, 2431, 2687, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1141, 1142,1143, 1144, 1145, 1146, 1147, 1148, 1153, 2432, 2688, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1149, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1185, 2433, 2689, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1161, 1162,1163, 1164, 1165, 1166, 1167, 1168, 1186, 2434, 2690, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1169, 1170, 1171, 1172,1173, 1174, 1175, 1176, 1187, 2435, 2691, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1177, 1178, 1179, 1180, 1181, 1182,1183, 1184, 1220, 2436, 2692, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1188, 1189, 1190, 1191, 1192,1193, 1194, 1195, 1221, 2437, 2693, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1196, 1197, 1198, 1199, 1200, 1201, 1202,1203, 1222, 2438, 2694, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1223, 2439, 2695, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1212,1213, 1214, 1215, 1216, 1217, 1218, 1219, 1255, 2440, 2696, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1256, 2441, 2697, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1232,1233, 1234, 1235, 1236, 1237, 1238, 1239, 1257, 2442, 2698, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1240, 1241, 1242,1243, 1244, 1245, 1246, 1247, 1258, 2443, 2699, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1248, 1249, 1250, 1251, 1252,1253, 1254, 1259, 1291, 2444, 2700, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1260, 1261, 1262,1263, 1264, 1265, 1266, 1267, 1292, 2445, 2701, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1268, 1269, 1270, 1271, 1272,1273, 1274, 1275, 1293, 2446, 2702, 2958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1276, 1277, 1278, 1279, 1280, 1281, 1282,1283, 1294, 2447, 2703, 2959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1284, 1285, 1286, 1287, 1288, 1289, 1290, 1295, 1326, 2448, 2704, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1296, 1297, 1298, 1299, 1300, 1301, 1302,1303, 1327, 2449, 2705, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1328, 2450, 2706, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1312,1313, 1314, 1315, 1316, 1317, 1318, 1319, 1329, 2451, 2707, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1320, 1321, 1322,1323, 1324, 1325, 1330, 1331, 1362, 2452, 2708, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1332,1333, 1334, 1335, 1336, 1337, 1338, 1339, 1363, 2453, 2709, 2965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1340, 1341, 1342,1343, 1344, 1345, 1346, 1347, 1364, 2454, 2710, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1348, 1349, 1350, 1351, 1352,1353, 1354, 1355, 1365, 2455, 2711, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1356, 1357, 1358, 1359, 1360, 1361, 1366, 1367, 1397, 2456, 2712, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1368, 1369, 1370, 1371, 1372,1373, 1374, 1375, 1398, 2457, 2713, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1376, 1377, 1378, 1379, 1380, 1381, 1382,1383, 1399, 2458, 2714, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1400, 2459, 2715, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1392,1393, 1394, 1395, 1396, 1401, 1402,1403, 1432, 2460, 2716, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1433, 2461, 2717, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1412,1413, 1414, 1415, 1416, 1417, 1418, 1419, 1434, 2462, 2718, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1420, 1421, 1422,1423, 1424, 1425, 1426, 1427, 1435, 2463, 2719, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1428, 1429, 1430, 1431, 1436, 1437, 1438, 1439, 1468, 2464, 2720, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1440, 1441, 1442,1443, 1444, 1445, 1446, 1447, 1469, 2465, 2721, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1448, 1449, 1450, 1451, 1452,1453, 1454, 1455, 1470, 2466, 2722, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1456, 1457, 1458, 1459, 1460, 1461, 1462,1463, 1471, 2467, 2723, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1464, 1465, 1466, 1467, 1472,1473, 1474, 1475, 1503, 2468, 2724, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1476, 1477, 1478, 1479, 1480, 1481, 1482,1483, 1504, 2469, 2725, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1505, 2470, 2726, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1492,1493, 1494, 1495, 1496, 1497, 1498, 1499, 1506, 2471, 2727, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1500, 1501, 1502,1507, 1508, 1509, 1510, 1511, 1538, 2472, 2728, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1512,1513, 1514, 1515, 1516, 1517, 1518, 1519, 1539, 2473, 2729, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1520, 1521, 1522,1523, 1524, 1525, 1526, 1527, 1540, 2474, 2730, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1528, 1529, 1530, 1531, 1532,1533, 1534, 1535, 1541, 2475, 2731, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1536, 1537, 1542,1543, 1544, 1545, 1546, 1547, 1574, 2476, 2732, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1548, 1549, 1550, 1551, 1552,1553, 1554, 1555, 1575, 2477, 2733, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1556, 1557, 1558, 1559, 1560, 1561, 1562,1563, 1576, 2478, 2734, 2990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1564, 1565, 1566, 1567, 1568, 1569, 1570, 1571, 1577, 2479, 2735, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1572,1573, 1578, 1579, 1580, 1581, 1582,1583, 1609, 2480, 2736, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1610, 2481, 2737, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1592,1593, 1594, 1595, 1596, 1597, 1598, 1599, 1611, 2482, 2738, 2994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1600, 1601, 1602,1603, 1604, 1605, 1606, 1607, 1612, 2483, 2739, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1608, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1644, 2484, 2740, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1620, 1621, 1622,1623, 1624, 1625, 1626, 1627, 1645, 2485, 2741, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1628, 1629, 1630, 1631, 1632,1633, 1634, 1635, 1646, 2486, 2742, 2998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1636, 1637, 1638, 1639, 1640, 1641, 1642,1643, 1679, 2487, 2743, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1647, 1648, 1649, 1650, 1651, 1652,1653, 1654, 1680, 2488, 2744, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662,1681, 2489, 2745, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1682, 2490, 2746, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1671, 1672,1673, 1674, 1675, 1676, 1677, 1678, 1714, 2491, 2747, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, 1715, 2492, 2748, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1691, 1692,1693, 1694, 1695, 1696, 1697, 1698, 1716, 2493, 2749, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1699, 1700, 1701, 1702,1703, 1704, 1705, 1706, 1717, 2494, 2750, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1707, 1708, 1709, 1710, 1711, 1712,1713, 1718, 1749, 2495, 2751, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1719, 1720, 1721, 1722,1723, 1724, 1725, 1726, 1750, 2496, 2752, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1727, 1728, 1729, 1730, 1731, 1732,1733, 1734, 1751, 2497, 2753, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742,1752, 2498, 2754, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1743, 1744, 1745, 1746, 1747, 1748, 1753, 1754, 1784, 2499, 2755, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762,1785, 2500, 2756, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1763, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1786, 2501, 2757, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1771, 1772,1773, 1774, 1775, 1776, 1777, 1778, 1787, 2502, 2758, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1779, 1780, 1781, 1782,1783, 1788, 1789, 1790, 1820, 2503, 2759, 3015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1791, 1792,1793, 1794, 1795, 1796, 1797, 1798, 1821, 2504, 2760, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1799, 1800, 1801, 1802,1803, 1804, 1805, 1806, 1822, 2505, 2761, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1807, 1808, 1809, 1810, 1811, 1812,1813, 1814, 1823, 2506, 2762, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1815, 1816, 1817, 1818, 1819, 1824, 1825, 1826, 1855, 2507, 2763, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1827, 1828, 1829, 1830, 1831, 1832,1833, 1834, 1856, 2508, 2764, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1835, 1836, 1837, 1838, 1839, 1840, 1841, 1842,1857, 2509, 2765, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1858, 2510, 2766, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1851, 1852,1853, 1854, 1859, 1860, 1861, 1862,1890, 2511, 2767, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1891, 2512, 2768, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1871, 1872,1873, 1874, 1875, 1876, 1877, 1878, 1892, 2513, 2769, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1879, 1880, 1881, 1882,1883, 1884, 1885, 1886, 1893, 2514, 2770, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1887, 1888, 1889, 1894, 1895, 1896, 1897, 1898, 1925, 2515, 2771, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1899, 1900, 1901, 1902,1903, 1904, 1905, 1906, 1926, 2516, 2772, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1907, 1908, 1909, 1910, 1911, 1912,1913, 1914, 1927, 2517, 2773, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922,1928, 2518, 2774, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1923, 1924, 1929, 1930, 1931, 1932,1933, 1934, 1960, 2519, 2775, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942,1961, 2520, 2776, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1943, 1944, 1945, 1946, 1947, 1948, 1949, 1950, 1962, 2521, 2777, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1951, 1952,1953, 1954, 1955, 1956, 1957, 1958, 1963, 2522, 2778, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1959, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1996, 2523, 2779, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1971, 1972,1973, 1974, 1975, 1976, 1977, 1978, 1997, 2524, 2780, 3036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1979, 1980, 1981, 1982,1983, 1984, 1985, 1986, 1998, 2525, 2781, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1987, 1988, 1989, 1990, 1991, 1992,1993, 1994, 1999, 2526, 2782, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1995, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2031, 2527, 2783, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2032, 2528, 2784, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2033, 2529, 2785, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2065, 2530, 2786, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2034, 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2066, 2531, 2787, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2067, 2532, 2788, 3044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2068, 2533, 2789, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2058, 2059, 2060, 2061, 2062, 2063, 2064, 2069, 2100, 2534, 2790, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2101, 2535, 2791, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, 2102, 2536, 2792, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2086, 2087, 2088, 2089, 2090, 2091, 2092, 2093, 2103, 2537, 2793, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2094, 2095, 2096, 2097, 2098, 2099, 2104, 2105, 2135, 2538, 2794, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2106, 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2136, 2539, 2795, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2137, 2540, 2796, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2138, 2541, 2797, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2130, 2131, 2132, 2133, 2134, 2139, 2140, 2141, 2170, 2542, 2798, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, 2171, 2543, 2799, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, 2172, 2544, 2800, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2173, 2545, 2801, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2166, 2167, 2168, 2169, 2174, 2175, 2176, 2177, 2205, 2546, 2802, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2178, 2179, 2180, 2181, 2182, 2183, 2184, 2185, 2206, 2547, 2803, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2186, 2187, 2188, 2189, 2190, 2191, 2192, 2193, 2207, 2548, 2804, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2208, 2549, 2805, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2202, 2203, 2204, 2209, 2210, 2211, 2212, 2213, 2240, 2550, 2806, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2241, 2551, 2807, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2222, 2223, 2224, 2225, 2226, 2227, 2228, 2229, 2242, 2552, 2808, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2243, 2553, 2809, 3065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2238, 2239, 2244, 2245, 2246, 2247, 2248, 2249, 2275, 2554, 2810, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2276, 2555, 2811, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2277, 2556, 2812, 3068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, 2278, 2557, 2813, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2274, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2302, 2558, 2814, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2303, 2559, 2815, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2304, 2560, 2816, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] - - - page_indices_rank1 = [[ 1, 2, 3, 4, 5, 6, 7, 8, 33, 2305, 2561, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 9, 10, 11, 12, 13, 14, 15, 16, 34, 2306, 2562, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 17, 18, 19, 20, 21, 22, 23, 24, 35, 2307, 2563, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 25, 26, 27, 28, 29, 30, 31, 32, 36, 2308, 2564, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 53, 54, 55, 56, 57, 58, 59, 60, 71, 2311, 2567, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 37, 38, 39, 40, 41, 42, 43, 44, 69, 2309, 2565, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 45, 46, 47, 48, 49, 50, 51, 52, 70, 2310, 2566, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 61, 62, 63, 64, 65, 66, 67, 68, 104, 2312, 2568, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 88, 89 , 90 , 91, 92, 93, 94, 95, 107, 2315, 2571, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 72, 73, 74, 75, 76, 77, 78, 79, 105, 2313, 2569, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 80, 81, 82, 83, 84, 85 ,86, 87, 106, 2314, 2570, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 96, 97, 98, 99, 100, 101, 102, 103, 140, 2316, 2572, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 124, 125, 126, 127, 128, 129, 130, 131, 143, 2319, 2575, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 108, 109, 110, 111, 112, 113, 114, 115, 141, 2317, 2573, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 116, 117, 118, 119, 120, 121, 122, 123, 142, 2318, 2574, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 132, 133, 134, 135, 136, 137, 138, 139, 176, 2320, 2576, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 160, 161, 162, 163, 164, 165, 166, 167, 179, 2323, 2579, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 144, 145, 146, 147, 148, 149, 150, 151, 177, 2321, 2577, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 152, 153, 154, 155, 156, 157, 158, 159, 178, 2322, 2578, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 168, 169, 170, 171, 172, 173, 174, 175, 212, 2324, 2580, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 196, 197, 198, 199, 200, 201, 202, 203, 215, 2327, 2583, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 180, 181, 182, 183, 184, 185, 186, 187, 213, 2325, 2581, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 188, 189, 190, 191, 192, 193, 194, 195, 214, 2326, 2582, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 204, 205, 206, 207, 208, 209, 210, 211, 248, 2328, 2584, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 232, 233, 234, 235, 236, 237, 238, 239, 251, 2331, 2587, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 216, 217, 218, 219, 220, 221, 222, 223, 249, 2329, 2585, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 224, 225, 226, 227, 228, 229, 230, 231, 250, 2330, 2586, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 240, 241, 242, 243, 244, 245, 246, 247, 284, 2332, 2588, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 268, 269, 270, 271, 272, 273, 274, 275, 287, 2335, 2591, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 252, 253, 254, 255, 256, 257, 258, 259, 285, 2333, 2589, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 260, 261, 262, 263, 264, 265, 266, 267, 286, 2334, 2590, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 276, 277, 278, 279, 280, 281, 282, 283, 320, 2336, 2592, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 304, 305, 306, 307, 308, 309, 310, 311, 323, 2339, 2595, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 288, 289, 290, 291, 292, 293, 294, 295, 321, 2337, 2593, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 296, 297, 298, 299, 300, 301, 302, 303, 322, 2338, 2594, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 312, 313, 314, 315, 316, 317, 318, 319, 356, 2340, 2596, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 340, 341, 342, 343, 344, 345, 346, 347, 359, 2343, 2599, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 324, 325, 326, 327, 328, 329, 330, 331, 357, 2341, 2597, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 332, 333, 334, 335, 336, 337, 338, 339, 358, 2342, 2598, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 348, 349, 350, 351, 352, 353, 354, 355, 392, 2344, 2600, 2856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 376, 377, 378, 379, 380, 381, 382, 383, 395, 2347, 2603, 2859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 360, 361, 362, 363, 364, 365, 366, 367, 393, 2345, 2601, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 368, 369, 370, 371, 372, 373, 374, 375, 394, 2346, 2602, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 384, 385, 386, 387, 388, 389, 390, 391, 428, 2348, 2604, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 412, 413, 414, 415, 416, 417, 418, 419, 431, 2351, 2607, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 396, 397, 398, 399, 400, 401, 402, 403, 429, 2349, 2605, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 404, 405, 406, 407, 408, 409, 410, 411, 430, 2350, 2606, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 420, 421, 422, 423, 424, 425, 426, 427, 463, 2352, 2608, 2864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 448, 449, 450, 451, 452, 453, 454, 455, 466, 2355, 2611, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 432, 433, 434, 435, 436, 437, 438, 439, 464, 2353, 2609, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 440, 441, 442, 443, 444, 445, 446, 447, 465, 2354, 2610, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 456, 457, 458, 459, 460, 461, 462, 467, 499, 2356, 2612, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 484, 485, 486, 487, 488, 489, 490, 491, 502, 2359, 2615, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 468, 469, 470, 471, 472, 473, 474, 475, 500, 2357, 2613, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 476, 477, 478, 479, 480, 481, 482, 483, 501, 2358, 2614, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 492, 493, 494, 495, 496, 497, 498, 503, 535, 2360, 2616, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 520, 521, 522, 523, 524, 525, 526, 527, 538, 2363, 2619, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 504, 505, 506, 507, 508, 509, 510, 511, 536, 2361, 2617, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 512, 513, 514, 515, 516, 517, 518, 519, 537, 2362, 2618, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 528, 529, 530, 531, 532, 533, 534, 539, 571, 2364, 2620, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 556, 557, 558, 559, 560, 561, 562, 563, 574, 2367, 2623, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 540, 541, 542, 543, 544, 545, 546, 547, 572, 2365, 2621, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 548, 549, 550, 551, 552, 553, 554, 555, 573, 2366, 2622, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 564, 565, 566, 567, 568, 569, 570, 575, 606, 2368, 2624, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 592, 593, 594, 595, 596, 597, 598, 599, 609, 2371, 2627, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 576, 577, 578, 579, 580, 581, 582, 583, 607, 2369, 2625, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 584, 585, 586, 587, 588, 589, 590, 591, 608, 2370, 2626, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 600, 601, 602, 603, 604, 605, 610, 611, 642, 2372, 2628, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 628, 629, 630, 631, 632, 633, 634, 635, 645, 2375, 2631, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 612, 613, 614, 615, 616, 617, 618, 619, 643, 2373, 2629, 2885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 620, 621, 622, 623, 624, 625, 626, 627, 644, 2374, 2630, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 636, 637, 638, 639, 640, 641, 646, 647, 678, 2376, 2632, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 664, 665, 666, 667, 668, 669, 670, 671, 681, 2379, 2635, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 648, 649, 650, 651, 652, 653, 654, 655, 679, 2377, 2633, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 656, 657, 658, 659, 660, 661, 662, 663, 680, 2378, 2634, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 672, 673, 674, 675, 676, 677, 682, 683, 714, 2380, 2636, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 700, 701, 702, 703, 704, 705, 706, 707, 717, 2383, 2639, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 684, 685, 686, 687, 688, 689, 690, 691, 715, 2381, 2637, 2893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 692, 693, 694, 695, 696, 697, 698, 699, 716, 2382, 2638, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 708, 709, 710, 711, 712, 713, 718, 719, 749, 2384, 2640, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 736, 737, 738, 739, 740, 741, 742, 743, 752, 2387, 2643, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 720, 721, 722, 723, 724, 725, 726, 727, 750, 2385, 2641, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 728, 729, 730, 731, 732, 733, 734, 735, 751, 2386, 2642, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 744, 745, 746, 747, 748, 753, 754, 755, 785, 2388, 2644, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 772, 773, 774, 775, 776, 777, 778, 779, 788, 2391, 2647, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 756, 757, 758, 759, 760, 761, 762, 763, 786, 2389, 2645, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 764, 765, 766, 767, 768, 769, 770, 771, 787, 2390, 2646, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 780, 781, 782, 783, 784, 789, 790, 791, 821, 2392, 2648, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 808, 809, 810, 811, 812, 813, 814, 815, 824, 2395, 2651, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 792, 793, 794, 795, 796, 797, 798, 799, 822, 2393, 2649, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 800, 801, 802, 803, 804, 805, 806, 807, 823, 2394, 2650, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 816, 817, 818, 819, 820, 825, 826, 827, 856, 2396, 2652, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 844, 845, 846, 847, 848, 849, 850, 851, 859, 2399, 2655, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 828, 829, 830, 831, 832, 833, 834, 835, 857, 2397, 2653, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 836, 837, 838, 839, 840, 841, 842, 843, 858, 2398, 2654, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 852, 853, 854, 855, 860, 861, 862, 863, 892, 2400, 2656, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 880, 881, 882, 883, 884, 885, 886, 887, 895, 2403, 2659, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 864, 865, 866, 867, 868, 869, 870, 871, 893, 2401, 2657, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 872, 873, 874, 875, 876, 877, 878, 879, 894, 2402, 2658, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 888, 889, 890, 891, 896, 897, 898, 899, 928, 2404, 2660, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 916, 917, 918, 919, 920, 921, 922, 923, 931, 2407, 2663, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 900, 901, 902, 903, 904, 905, 906, 907, 929, 2405, 2661, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 908, 909, 910, 911, 912, 913, 914, 915, 930, 2406, 2662, 2918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 924, 925, 926, 927, 932, 933, 934, 935, 963, 2408, 2664, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 952, 953, 954, 955, 956, 957, 958, 959, 966, 2411, 2667, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 936, 937, 938, 939, 940, 941, 942, 943, 964, 2409, 2665, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 944, 945, 946, 947, 948, 949, 950, 951, 965, 2410, 2666, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 960, 961, 962, 967, 968, 969, 970, 971, 999, 2412, 2668, 2924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 988, 989, 990, 991, 992, 993, 994, 995, 1002, 2415, 2671, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 972, 973, 974, 975, 976, 977, 978, 979, 1000, 2413, 2669, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 980, 981, 982, 983, 984, 985, 986, 987, 1001, 2414, 2670, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 996, 997, 998, 1003, 1004, 1005, 1006, 1007, 1034, 2416, 2672, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1037, 2419, 2675, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1008, 1009, 1010, 1011, 1012,1013, 1014, 1015, 1035, 2417, 2673, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1016, 1017, 1018, 1019, 1020, 1021, 1022,1023, 1036, 2418, 2674, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1032,1033, 1038, 1039, 1040, 1041, 1042,1043, 1070, 2420, 2676, 2932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1060, 1061, 1062,1063, 1064, 1065, 1066, 1067, 1073, 2423, 2679, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1071, 2421, 2677, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1052,1053, 1054, 1055, 1056, 1057, 1058, 1059, 1072, 2422, 2678, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1068, 1069, 1074, 1075, 1076, 1077, 1078, 1079, 1105, 2424, 2680, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1096, 1097, 1098, 1099, 1100, 1101, 1102,1103, 1108, 2427, 2683, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1080, 1081, 1082,1083, 1084, 1085, 1086, 1087, 1106, 2425, 2681, 2937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1088, 1089, 1090, 1091, 1092,1093, 1094, 1095, 1107, 2426, 2682, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1104, 1109, 1110, 1111, 1112,1113, 1114, 1115, 1141, 2428, 2684, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1132,1133, 1134, 1135, 1136, 1137, 1138, 1139, 1144, 2431, 2687, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1116, 1117, 1118, 1119, 1120, 1121, 1122,1123, 1142, 2429, 2685, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1143, 2430, 2686, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1140, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1176, 2432, 2688, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1160, 1161, 1162,1163, 1164, 1165, 1166, 1167, 1178, 2434, 2690, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1152,1153, 1154, 1155, 1156, 1157, 1158, 1159, 1177, 2433, 2689, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1168, 1169, 1170, 1171, 1172,1173, 1174, 1175, 1211, 2435, 2691, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202,1214, 2438, 2694, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1179, 1180, 1181, 1182,1183, 1184, 1185, 1186, 1212, 2436, 2692, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1187, 1188, 1189, 1190, 1191, 1192,1193, 1194, 1213, 2437, 2693, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1246, 2439, 2695, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1231, 1232,1233, 1234, 1235, 1236, 1237, 1238, 1249, 2442, 2698, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222,1247, 2440, 2696, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1248, 2441, 2697, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1239, 1240, 1241, 1242,1243, 1244, 1245, 1250, 1282, 2443, 2699, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1267, 1268, 1269, 1270, 1271, 1272,1273, 1274, 1285, 2446, 2702, 2958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1251, 1252,1253, 1254, 1255, 1256, 1257, 1258, 1283, 2444, 2700, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1259, 1260, 1261, 1262,1263, 1264, 1265, 1266, 1284, 2445, 2701, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1275, 1276, 1277, 1278, 1279, 1280, 1281, 1286, 1317, 2447, 2703, 2959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1320, 2450, 2706, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1287, 1288, 1289, 1290, 1291, 1292,1293, 1294, 1318, 2448, 2704, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302,1319, 2449, 2705, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1311, 1312,1313, 1314, 1315, 1316, 1321, 1322,1353, 2451, 2707, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1339, 1340, 1341, 1342,1343, 1344, 1345, 1346, 1356, 2454, 2710, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1354, 2452, 2708, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1331, 1332,1333, 1334, 1335, 1336, 1337, 1338, 1355, 2453, 2709, 2965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1347, 1348, 1349, 1350, 1351, 1352,1357, 1358, 1388, 2455, 2711, 2967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382,1391, 2458, 2714, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1359, 1360, 1361, 1362,1363, 1364, 1365, 1366, 1389, 2456, 2712, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1367, 1368, 1369, 1370, 1371, 1372,1373, 1374, 1390, 2457, 2713, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1383, 1384, 1385, 1386, 1387, 1392,1393, 1394, 1423, 2459, 2715, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1411, 1412,1413, 1414, 1415, 1416, 1417, 1418, 1426, 2462, 2718, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402,1424, 2460, 2716, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1425, 2461, 2717, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1419, 1420, 1421, 1422,1427, 1428, 1429, 1430, 1459, 2463, 2719, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1447, 1448, 1449, 1450, 1451, 1452,1453, 1454, 1462, 2466, 2722, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1431, 1432,1433, 1434, 1435, 1436, 1437, 1438, 1460, 2464, 2720, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1439, 1440, 1441, 1442,1443, 1444, 1445, 1446, 1461, 2465, 2721, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1455, 1456, 1457, 1458, 1463, 1464, 1465, 1466, 1494, 2467, 2723, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1497, 2470, 2726, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1467, 1468, 1469, 1470, 1471, 1472,1473, 1474, 1495, 2468, 2724, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482,1496, 2469, 2725, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1491, 1492,1493, 1498, 1499, 1500, 1501, 1502,1530, 2471, 2727, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1519, 1520, 1521, 1522,1523, 1524, 1525, 1526, 1533, 2474, 2730, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1531, 2472, 2728, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1511, 1512,1513, 1514, 1515, 1516, 1517, 1518, 1532, 2473, 2729, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1527, 1528, 1529, 1534, 1535, 1536, 1537, 1538, 1565, 2475, 2731, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562,1568, 2478, 2734, 2990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1539, 1540, 1541, 1542,1543, 1544, 1545, 1546, 1566, 2476, 2732, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1547, 1548, 1549, 1550, 1551, 1552,1553, 1554, 1567, 2477, 2733, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1563, 1564, 1569, 1570, 1571, 1572,1573, 1574, 1600, 2479, 2735, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1591, 1592,1593, 1594, 1595, 1596, 1597, 1598, 1603, 2482, 2738, 2994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1575, 1576, 1577, 1578, 1579, 1580, 1581, 1582,1601, 2480, 2736, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1583, 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1602, 2481, 2737, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1599, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1636, 2483, 2739, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1627, 1628, 1629, 1630, 1631, 1632,1633, 1634, 1639, 2486, 2742, 2998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1611, 1612,1613, 1614, 1615, 1616, 1617, 1618, 1637, 2484, 2740, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1619, 1620, 1621, 1622,1623, 1624, 1625, 1626, 1638, 2485, 2741, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1635, 1640, 1641, 1642,1643, 1644, 1645, 1646, 1671, 2487, 2743, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662,1673, 2489, 2745, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1647, 1648, 1649, 1650, 1651, 1652,1653, 1654, 1672, 2488, 2744, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1663, 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1705, 2490, 2746, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1690, 1691, 1692,1693, 1694, 1695, 1696, 1697, 1708, 2493, 2749, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1674, 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1706, 2491, 2747, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1682,1683, 1684, 1685, 1686, 1687, 1688, 1689, 1707, 2492, 2748, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1698, 1699, 1700, 1701, 1702,1703, 1704, 1709, 1740, 2494, 2750, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1726, 1727, 1728, 1729, 1730, 1731, 1732,1733, 1743, 2497, 2753, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1710, 1711, 1712,1713, 1714, 1715, 1716, 1717, 1741, 2495, 2751, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1718, 1719, 1720, 1721, 1722,1723, 1724, 1725, 1742, 2496, 2752, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1734, 1735, 1736, 1737, 1738, 1739, 1744, 1745, 1776, 2498, 2754, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1762,1763, 1764, 1765, 1766, 1767, 1768, 1769, 1779, 2501, 2757, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1746, 1747, 1748, 1749, 1750, 1751, 1752,1753, 1777, 2499, 2755, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1754, 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1778, 2500, 2756, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1770, 1771, 1772,1773, 1774, 1775, 1780, 1781, 1811, 2502, 2758, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1798, 1799, 1800, 1801, 1802,1803, 1804, 1805, 1814, 2505, 2761, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1782,1783, 1784, 1785, 1786, 1787, 1788, 1789, 1812, 2503, 2759, 3015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1790, 1791, 1792,1793, 1794, 1795, 1796, 1797, 1813, 2504, 2760, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1806, 1807, 1808, 1809, 1810, 1815, 1816, 1817, 1846, 2506, 2762, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1834, 1835, 1836, 1837, 1838, 1839, 1840, 1841, 1849, 2509, 2765, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1818, 1819, 1820, 1821, 1822,1823, 1824, 1825, 1847, 2507, 2763, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1826, 1827, 1828, 1829, 1830, 1831, 1832,1833, 1848, 2508, 2764, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1842,1843, 1844, 1845, 1850, 1851, 1852,1853, 1881, 2510, 2766, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1870, 1871, 1872,1873, 1874, 1875, 1876, 1877, 1884, 2513, 2769, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, 1882, 2511, 2767, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1862,1863, 1864, 1865, 1866, 1867, 1868, 1869, 1883, 2512, 2768, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1878, 1879, 1880, 1885, 1886, 1887, 1888, 1889, 1916, 2514, 2770, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1906, 1907, 1908, 1909, 1910, 1911, 1912,1913, 1919, 2517, 2773, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1890, 1891, 1892,1893, 1894, 1895, 1896, 1897, 1917, 2515, 2771, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1898, 1899, 1900, 1901, 1902,1903, 1904, 1905, 1918, 2516, 2772, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1914, 1915, 1920, 1921, 1922,1923, 1924, 1925, 1952, 2518, 2774, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1942,1943, 1944, 1945, 1946, 1947, 1948, 1949, 1955, 2521, 2777, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1926, 1927, 1928, 1929, 1930, 1931, 1932,1933, 1953, 2519, 2775, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1954, 2520, 2776, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1950, 1951, 1956, 1957, 1958, 1959, 1960, 1961, 1987, 2522, 2778, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1978, 1979, 1980, 1981, 1982,1983, 1984, 1985, 1990, 2525, 2781, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1962,1963, 1964, 1965, 1966, 1967, 1968, 1969, 1988, 2523, 2779, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1970, 1971, 1972,1973, 1974, 1975, 1976, 1977, 1989, 2524, 2780, 3036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1986, 1991, 1992,1993, 1994, 1995, 1996, 1997, 2022, 2526, 2782, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2024, 2528, 2784, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2023, 2527, 2783, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2056, 2529, 2785, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2041, 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2059, 2532, 2788, 3044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2057, 2530, 2786, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2033, 2034, 2035, 2036, 2037, 2038, 2039, 2040, 2058, 2531, 2787, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2049, 2050, 2051, 2052, 2053, 2054, 2055, 2060, 2091, 2533, 2789, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2077, 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2094, 2536, 2792, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2092, 2534, 2790, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2093, 2535, 2791, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2085, 2086, 2087, 2088, 2089, 2090, 2095, 2096, 2126, 2537, 2793, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2129, 2540, 2796, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2097, 2098, 2099, 2100, 2101, 2102, 2103, 2104, 2127, 2538, 2794, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2105, 2106, 2107, 2108, 2109, 2110, 2111, 2112, 2128, 2539, 2795, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2121, 2122, 2123, 2124, 2125, 2130, 2131, 2132, 2161, 2541, 2797, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2164, 2544, 2800, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2133, 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2162, 2542, 2798, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2141, 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2163, 2543, 2799, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2157, 2158, 2159, 2160, 2165, 2166, 2167, 2168, 2196, 2545, 2801, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, 2199, 2548, 2804, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2197, 2546, 2802, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2177, 2178, 2179, 2180, 2181, 2182, 2183, 2184, 2198, 2547, 2803, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2193, 2194, 2195, 2200, 2201, 2202, 2203, 2204, 2231, 2549, 2805, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2221, 2222, 2223, 2224, 2225, 2226, 2227, 2228, 2234, 2552, 2808, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2205, 2206, 2207, 2208, 2209, 2210, 2211, 2212, 2232, 2550, 2806, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2233, 2551, 2807, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2229, 2230, 2235, 2236, 2237, 2238, 2239, 2240, 2266, 2553, 2809, 3065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2269, 2556, 2812, 3068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2267, 2554, 2810, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2268, 2555, 2811, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2265, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2301, 2557, 2813, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2302, 2558, 2814, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2303, 2559, 2815, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2304, 2560, 2816, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] - page_indices_rank0 = sum(page_indices_rank0, []) - page_indices_rank1 = sum(page_indices_rank1, []) - page_indices_no_dp = [[ 1, 2, 3, 4, 5, 6, 7, 8, 17, 2305, 2561, - 2817, 3073, 3329, 3585, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 9, 10, 11, 12, 13, 14, 15, 16, 18, 2306, 2562, - 2818, 3074, 3330, 3586, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 19, 20, 21, 22, 23, 24, 25, 26, 51, 2307, 2563, - 2819, 3075, 3331, 3587, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 27, 28, 29, 30, 31, 32, 33, 34, 52, 2308, 2564, - 2820, 3076, 3332, 3588, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 35, 36, 37, 38, 39, 40, 41, 42, 53, 2309, 2565, - 2821, 3077, 3333, 3589, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 43, 44, 45, 46, 47, 48, 49, 50, 86, 2310, 2566, - 2822, 3078, 3334, 3590, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 54, 55, 56, 57, 58, 59, 60, 61, 87, 2311, 2567, - 2823, 3079, 3335, 3591, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 62, 63, 64, 65, 66, 67, 68, 69, 88, 2312, 2568, - 2824, 3080, 3336, 3592, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 70, 71, 72, 73, 74, 75, 76, 77, 89, 2313, 2569, - 2825, 3081, 3337, 3593, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 78, 79, 80, 81, 82, 83, 84, 85, 122, 2314, 2570, - 2826, 3082, 3338, 3594, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 90, 91, 92, 93, 94, 95, 96, 97, 123, 2315, 2571, - 2827, 3083, 3339, 3595, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 98, 99, 100, 101, 102, 103, 104, 105, 124, 2316, 2572, - 2828, 3084, 3340, 3596, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 106, 107, 108, 109, 110, 111, 112, 113, 125, 2317, 2573, - 2829, 3085, 3341, 3597, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 114, 115, 116, 117, 118, 119, 120, 121, 158, 2318, 2574, - 2830, 3086, 3342, 3598, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 126, 127, 128, 129, 130, 131, 132, 133, 159, 2319, 2575, - 2831, 3087, 3343, 3599, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 134, 135, 136, 137, 138, 139, 140, 141, 160, 2320, 2576, - 2832, 3088, 3344, 3600, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 142, 143, 144, 145, 146, 147, 148, 149, 161, 2321, 2577, - 2833, 3089, 3345, 3601, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 150, 151, 152, 153, 154, 155, 156, 157, 194, 2322, 2578, - 2834, 3090, 3346, 3602, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 162, 163, 164, 165, 166, 167, 168, 169, 195, 2323, 2579, - 2835, 3091, 3347, 3603, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 170, 171, 172, 173, 174, 175, 176, 177, 196, 2324, 2580, - 2836, 3092, 3348, 3604, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 178, 179, 180, 181, 182, 183, 184, 185, 197, 2325, 2581, - 2837, 3093, 3349, 3605, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 186, 187, 188, 189, 190, 191, 192, 193, 230, 2326, 2582, - 2838, 3094, 3350, 3606, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 198, 199, 200, 201, 202, 203, 204, 205, 231, 2327, 2583, - 2839, 3095, 3351, 3607, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 206, 207, 208, 209, 210, 211, 212, 213, 232, 2328, 2584, - 2840, 3096, 3352, 3608, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 214, 215, 216, 217, 218, 219, 220, 221, 233, 2329, 2585, - 2841, 3097, 3353, 3609, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 222, 223, 224, 225, 226, 227, 228, 229, 266, 2330, 2586, - 2842, 3098, 3354, 3610, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 234, 235, 236, 237, 238, 239, 240, 241, 267, 2331, 2587, - 2843, 3099, 3355, 3611, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 242, 243, 244, 245, 246, 247, 248, 249, 268, 2332, 2588, - 2844, 3100, 3356, 3612, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 250, 251, 252, 253, 254, 255, 256, 257, 269, 2333, 2589, - 2845, 3101, 3357, 3613, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 258, 259, 260, 261, 262, 263, 264, 265, 302, 2334, 2590, - 2846, 3102, 3358, 3614, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 270, 271, 272, 273, 274, 275, 276, 277, 303, 2335, 2591, - 2847, 3103, 3359, 3615, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 278, 279, 280, 281, 282, 283, 284, 285, 304, 2336, 2592, - 2848, 3104, 3360, 3616, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 286, 287, 288, 289, 290, 291, 292, 293, 305, 2337, 2593, - 2849, 3105, 3361, 3617, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 294, 295, 296, 297, 298, 299, 300, 301, 338, 2338, 2594, - 2850, 3106, 3362, 3618, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 306, 307, 308, 309, 310, 311, 312, 313, 339, 2339, 2595, - 2851, 3107, 3363, 3619, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 314, 315, 316, 317, 318, 319, 320, 321, 340, 2340, 2596, - 2852, 3108, 3364, 3620, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 322, 323, 324, 325, 326, 327, 328, 329, 341, 2341, 2597, - 2853, 3109, 3365, 3621, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 330, 331, 332, 333, 334, 335, 336, 337, 374, 2342, 2598, - 2854, 3110, 3366, 3622, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 342, 343, 344, 345, 346, 347, 348, 349, 375, 2343, 2599, - 2855, 3111, 3367, 3623, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 350, 351, 352, 353, 354, 355, 356, 357, 376, 2344, 2600, - 2856, 3112, 3368, 3624, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 358, 359, 360, 361, 362, 363, 364, 365, 377, 2345, 2601, - 2857, 3113, 3369, 3625, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 366, 367, 368, 369, 370, 371, 372, 373, 410, 2346, 2602, - 2858, 3114, 3370, 3626, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 378, 379, 380, 381, 382, 383, 384, 385, 411, 2347, 2603, - 2859, 3115, 3371, 3627, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 386, 387, 388, 389, 390, 391, 392, 393, 412, 2348, 2604, - 2860, 3116, 3372, 3628, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 394, 395, 396, 397, 398, 399, 400, 401, 413, 2349, 2605, - 2861, 3117, 3373, 3629, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 402, 403, 404, 405, 406, 407, 408, 409, 445, 2350, 2606, - 2862, 3118, 3374, 3630, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 414, 415, 416, 417, 418, 419, 420, 421, 446, 2351, 2607, - 2863, 3119, 3375, 3631, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 422, 423, 424, 425, 426, 427, 428, 429, 447, 2352, 2608, - 2864, 3120, 3376, 3632, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 430, 431, 432, 433, 434, 435, 436, 437, 448, 2353, 2609, - 2865, 3121, 3377, 3633, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 438, 439, 440, 441, 442, 443, 444, 449, 481, 2354, 2610, - 2866, 3122, 3378, 3634, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 450, 451, 452, 453, 454, 455, 456, 457, 482, 2355, 2611, - 2867, 3123, 3379, 3635, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 458, 459, 460, 461, 462, 463, 464, 465, 483, 2356, 2612, - 2868, 3124, 3380, 3636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 466, 467, 468, 469, 470, 471, 472, 473, 484, 2357, 2613, - 2869, 3125, 3381, 3637, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 474, 475, 476, 477, 478, 479, 480, 485, 517, 2358, 2614, - 2870, 3126, 3382, 3638, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 486, 487, 488, 489, 490, 491, 492, 493, 518, 2359, 2615, - 2871, 3127, 3383, 3639, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 494, 495, 496, 497, 498, 499, 500, 501, 519, 2360, 2616, - 2872, 3128, 3384, 3640, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 502, 503, 504, 505, 506, 507, 508, 509, 520, 2361, 2617, - 2873, 3129, 3385, 3641, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 510, 511, 512, 513, 514, 515, 516, 521, 553, 2362, 2618, - 2874, 3130, 3386, 3642, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 522, 523, 524, 525, 526, 527, 528, 529, 554, 2363, 2619, - 2875, 3131, 3387, 3643, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 530, 531, 532, 533, 534, 535, 536, 537, 555, 2364, 2620, - 2876, 3132, 3388, 3644, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 538, 539, 540, 541, 542, 543, 544, 545, 556, 2365, 2621, - 2877, 3133, 3389, 3645, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 546, 547, 548, 549, 550, 551, 552, 557, 589, 2366, 2622, - 2878, 3134, 3390, 3646, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 558, 559, 560, 561, 562, 563, 564, 565, 590, 2367, 2623, - 2879, 3135, 3391, 3647, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 566, 567, 568, 569, 570, 571, 572, 573, 591, 2368, 2624, - 2880, 3136, 3392, 3648, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 574, 575, 576, 577, 578, 579, 580, 581, 592, 2369, 2625, - 2881, 3137, 3393, 3649, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 582, 583, 584, 585, 586, 587, 588, 593, 624, 2370, 2626, - 2882, 3138, 3394, 3650, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 594, 595, 596, 597, 598, 599, 600, 601, 625, 2371, 2627, - 2883, 3139, 3395, 3651, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 602, 603, 604, 605, 606, 607, 608, 609, 626, 2372, 2628, - 2884, 3140, 3396, 3652, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 610, 611, 612, 613, 614, 615, 616, 617, 627, 2373, 2629, - 2885, 3141, 3397, 3653, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 618, 619, 620, 621, 622, 623, 628, 629, 660, 2374, 2630, - 2886, 3142, 3398, 3654, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 630, 631, 632, 633, 634, 635, 636, 637, 661, 2375, 2631, - 2887, 3143, 3399, 3655, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 638, 639, 640, 641, 642, 643, 644, 645, 662, 2376, 2632, - 2888, 3144, 3400, 3656, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 646, 647, 648, 649, 650, 651, 652, 653, 663, 2377, 2633, - 2889, 3145, 3401, 3657, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 654, 655, 656, 657, 658, 659, 664, 665, 696, 2378, 2634, - 2890, 3146, 3402, 3658, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 666, 667, 668, 669, 670, 671, 672, 673, 697, 2379, 2635, - 2891, 3147, 3403, 3659, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 674, 675, 676, 677, 678, 679, 680, 681, 698, 2380, 2636, - 2892, 3148, 3404, 3660, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 682, 683, 684, 685, 686, 687, 688, 689, 699, 2381, 2637, - 2893, 3149, 3405, 3661, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 690, 691, 692, 693, 694, 695, 700, 701, 731, 2382, 2638, - 2894, 3150, 3406, 3662, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 702, 703, 704, 705, 706, 707, 708, 709, 732, 2383, 2639, - 2895, 3151, 3407, 3663, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 710, 711, 712, 713, 714, 715, 716, 717, 733, 2384, 2640, - 2896, 3152, 3408, 3664, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 718, 719, 720, 721, 722, 723, 724, 725, 734, 2385, 2641, - 2897, 3153, 3409, 3665, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 726, 727, 728, 729, 730, 735, 736, 737, 767, 2386, 2642, - 2898, 3154, 3410, 3666, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 738, 739, 740, 741, 742, 743, 744, 745, 768, 2387, 2643, - 2899, 3155, 3411, 3667, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 746, 747, 748, 749, 750, 751, 752, 753, 769, 2388, 2644, - 2900, 3156, 3412, 3668, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 754, 755, 756, 757, 758, 759, 760, 761, 770, 2389, 2645, - 2901, 3157, 3413, 3669, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 762, 763, 764, 765, 766, 771, 772, 773, 803, 2390, 2646, - 2902, 3158, 3414, 3670, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 774, 775, 776, 777, 778, 779, 780, 781, 804, 2391, 2647, - 2903, 3159, 3415, 3671, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 782, 783, 784, 785, 786, 787, 788, 789, 805, 2392, 2648, - 2904, 3160, 3416, 3672, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 790, 791, 792, 793, 794, 795, 796, 797, 806, 2393, 2649, - 2905, 3161, 3417, 3673, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 798, 799, 800, 801, 802, 807, 808, 809, 838, 2394, 2650, - 2906, 3162, 3418, 3674, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 810, 811, 812, 813, 814, 815, 816, 817, 839, 2395, 2651, - 2907, 3163, 3419, 3675, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 818, 819, 820, 821, 822, 823, 824, 825, 840, 2396, 2652, - 2908, 3164, 3420, 3676, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 826, 827, 828, 829, 830, 831, 832, 833, 841, 2397, 2653, - 2909, 3165, 3421, 3677, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 834, 835, 836, 837, 842, 843, 844, 845, 874, 2398, 2654, - 2910, 3166, 3422, 3678, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 846, 847, 848, 849, 850, 851, 852, 853, 875, 2399, 2655, - 2911, 3167, 3423, 3679, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 854, 855, 856, 857, 858, 859, 860, 861, 876, 2400, 2656, - 2912, 3168, 3424, 3680, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 862, 863, 864, 865, 866, 867, 868, 869, 877, 2401, 2657, - 2913, 3169, 3425, 3681, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 870, 871, 872, 873, 878, 879, 880, 881, 910, 2402, 2658, - 2914, 3170, 3426, 3682, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 882, 883, 884, 885, 886, 887, 888, 889, 911, 2403, 2659, - 2915, 3171, 3427, 3683, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 890, 891, 892, 893, 894, 895, 896, 897, 912, 2404, 2660, - 2916, 3172, 3428, 3684, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 898, 899, 900, 901, 902, 903, 904, 905, 913, 2405, 2661, - 2917, 3173, 3429, 3685, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 906, 907, 908, 909, 914, 915, 916, 917, 945, 2406, 2662, - 2918, 3174, 3430, 3686, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 918, 919, 920, 921, 922, 923, 924, 925, 946, 2407, 2663, - 2919, 3175, 3431, 3687, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 926, 927, 928, 929, 930, 931, 932, 933, 947, 2408, 2664, - 2920, 3176, 3432, 3688, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 934, 935, 936, 937, 938, 939, 940, 941, 948, 2409, 2665, - 2921, 3177, 3433, 3689, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 942, 943, 944, 949, 950, 951, 952, 953, 981, 2410, 2666, - 2922, 3178, 3434, 3690, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 954, 955, 956, 957, 958, 959, 960, 961, 982, 2411, 2667, - 2923, 3179, 3435, 3691, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 962, 963, 964, 965, 966, 967, 968, 969, 983, 2412, 2668, - 2924, 3180, 3436, 3692, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 970, 971, 972, 973, 974, 975, 976, 977, 984, 2413, 2669, - 2925, 3181, 3437, 3693, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 978, 979, 980, 985, 986, 987, 988, 989, 1016, 2414, 2670, - 2926, 3182, 3438, 3694, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 990, 991, 992, 993, 994, 995, 996, 997, 1017, 2415, 2671, - 2927, 3183, 3439, 3695, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [ 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1018, 2416, 2672, - 2928, 3184, 3440, 3696, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1019, 2417, 2673, - 2929, 3185, 3441, 3697, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1014, 1015, 1020, 1021, 1022, 1023, 1024, 1025, 1052, 2418, 2674, - 2930, 3186, 3442, 3698, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1053, 2419, 2675, - 2931, 3187, 3443, 3699, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1054, 2420, 2676, - 2932, 3188, 3444, 3700, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1055, 2421, 2677, - 2933, 3189, 3445, 3701, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1050, 1051, 1056, 1057, 1058, 1059, 1060, 1061, 1088, 2422, 2678, - 2934, 3190, 3446, 3702, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1089, 2423, 2679, - 2935, 3191, 3447, 3703, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1090, 2424, 2680, - 2936, 3192, 3448, 3704, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1091, 2425, 2681, - 2937, 3193, 3449, 3705, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1086, 1087, 1092, 1093, 1094, 1095, 1096, 1097, 1123, 2426, 2682, - 2938, 3194, 3450, 3706, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1124, 2427, 2683, - 2939, 3195, 3451, 3707, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1125, 2428, 2684, - 2940, 3196, 3452, 3708, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1126, 2429, 2685, - 2941, 3197, 3453, 3709, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1122, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1159, 2430, 2686, - 2942, 3198, 3454, 3710, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1160, 2431, 2687, - 2943, 3199, 3455, 3711, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1161, 2432, 2688, - 2944, 3200, 3456, 3712, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1162, 2433, 2689, - 2945, 3201, 3457, 3713, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1158, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1194, 2434, 2690, - 2946, 3202, 3458, 3714, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1195, 2435, 2691, - 2947, 3203, 3459, 3715, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1196, 2436, 2692, - 2948, 3204, 3460, 3716, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1229, 2437, 2693, - 2949, 3205, 3461, 3717, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1230, 2438, 2694, - 2950, 3206, 3462, 3718, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1231, 2439, 2695, - 2951, 3207, 3463, 3719, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1232, 2440, 2696, - 2952, 3208, 3464, 3720, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1264, 2441, 2697, - 2953, 3209, 3465, 3721, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1265, 2442, 2698, - 2954, 3210, 3466, 3722, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1266, 2443, 2699, - 2955, 3211, 3467, 3723, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1267, 2444, 2700, - 2956, 3212, 3468, 3724, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1257, 1258, 1259, 1260, 1261, 1262, 1263, 1268, 1300, 2445, 2701, - 2957, 3213, 3469, 3725, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1301, 2446, 2702, - 2958, 3214, 3470, 3726, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1302, 2447, 2703, - 2959, 3215, 3471, 3727, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1303, 2448, 2704, - 2960, 3216, 3472, 3728, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1293, 1294, 1295, 1296, 1297, 1298, 1299, 1304, 1335, 2449, 2705, - 2961, 3217, 3473, 3729, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1336, 2450, 2706, - 2962, 3218, 3474, 3730, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1337, 2451, 2707, - 2963, 3219, 3475, 3731, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1338, 2452, 2708, - 2964, 3220, 3476, 3732, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1329, 1330, 1331, 1332, 1333, 1334, 1339, 1340, 1370, 2453, 2709, - 2965, 3221, 3477, 3733, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1371, 2454, 2710, - 2966, 3222, 3478, 3734, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1372, 2455, 2711, - 2967, 3223, 3479, 3735, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1373, 2456, 2712, - 2968, 3224, 3480, 3736, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1365, 1366, 1367, 1368, 1369, 1374, 1375, 1376, 1406, 2457, 2713, - 2969, 3225, 3481, 3737, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1407, 2458, 2714, - 2970, 3226, 3482, 3738, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1408, 2459, 2715, - 2971, 3227, 3483, 3739, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1409, 2460, 2716, - 2972, 3228, 3484, 3740, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1401, 1402, 1403, 1404, 1405, 1410, 1411, 1412, 1441, 2461, 2717, - 2973, 3229, 3485, 3741, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1442, 2462, 2718, - 2974, 3230, 3486, 3742, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1421, 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1443, 2463, 2719, - 2975, 3231, 3487, 3743, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1444, 2464, 2720, - 2976, 3232, 3488, 3744, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1437, 1438, 1439, 1440, 1445, 1446, 1447, 1448, 1477, 2465, 2721, - 2977, 3233, 3489, 3745, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1449, 1450, 1451, 1452, 1453, 1454, 1455, 1456, 1478, 2466, 2722, - 2978, 3234, 3490, 3746, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1479, 2467, 2723, - 2979, 3235, 3491, 3747, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1465, 1466, 1467, 1468, 1469, 1470, 1471, 1472, 1480, 2468, 2724, - 2980, 3236, 3492, 3748, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1473, 1474, 1475, 1476, 1481, 1482, 1483, 1484, 1512, 2469, 2725, - 2981, 3237, 3493, 3749, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1513, 2470, 2726, - 2982, 3238, 3494, 3750, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1493, 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1514, 2471, 2727, - 2983, 3239, 3495, 3751, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1515, 2472, 2728, - 2984, 3240, 3496, 3752, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1509, 1510, 1511, 1516, 1517, 1518, 1519, 1520, 1547, 2473, 2729, - 2985, 3241, 3497, 3753, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1521, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1548, 2474, 2730, - 2986, 3242, 3498, 3754, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1529, 1530, 1531, 1532, 1533, 1534, 1535, 1536, 1549, 2475, 2731, - 2987, 3243, 3499, 3755, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1550, 2476, 2732, - 2988, 3244, 3500, 3756, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1545, 1546, 1551, 1552, 1553, 1554, 1555, 1556, 1583, 2477, 2733, - 2989, 3245, 3501, 3757, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1584, 2478, 2734, - 2990, 3246, 3502, 3758, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1565, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1585, 2479, 2735, - 2991, 3247, 3503, 3759, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1586, 2480, 2736, - 2992, 3248, 3504, 3760, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1581, 1582, 1587, 1588, 1589, 1590, 1591, 1592, 1618, 2481, 2737, - 2993, 3249, 3505, 3761, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1593, 1594, 1595, 1596, 1597, 1598, 1599, 1600, 1619, 2482, 2738, - 2994, 3250, 3506, 3762, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1620, 2483, 2739, - 2995, 3251, 3507, 3763, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1621, 2484, 2740, - 2996, 3252, 3508, 3764, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1617, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1653, 2485, 2741, - 2997, 3253, 3509, 3765, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1654, 2486, 2742, - 2998, 3254, 3510, 3766, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1655, 2487, 2743, - 2999, 3255, 3511, 3767, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1645, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1687, 2488, 2744, - 3000, 3256, 3512, 3768, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1688, 2489, 2745, - 3001, 3257, 3513, 3769, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1689, 2490, 2746, - 3002, 3258, 3514, 3770, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, 1690, 2491, 2747, - 3003, 3259, 3515, 3771, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1680, 1681, 1682, 1683, 1684, 1685, 1686, 1691, 1723, 2492, 2748, - 3004, 3260, 3516, 3772, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699, 1724, 2493, 2749, - 3005, 3261, 3517, 3773, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1725, 2494, 2750, - 3006, 3262, 3518, 3774, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1708, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1726, 2495, 2751, - 3007, 3263, 3519, 3775, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1716, 1717, 1718, 1719, 1720, 1721, 1722, 1727, 1758, 2496, 2752, - 3008, 3264, 3520, 3776, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1728, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1759, 2497, 2753, - 3009, 3265, 3521, 3777, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, 1760, 2498, 2754, - 3010, 3266, 3522, 3778, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1761, 2499, 2755, - 3011, 3267, 3523, 3779, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1752, 1753, 1754, 1755, 1756, 1757, 1762, 1763, 1793, 2500, 2756, - 3012, 3268, 3524, 3780, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1764, 1765, 1766, 1767, 1768, 1769, 1770, 1771, 1794, 2501, 2757, - 3013, 3269, 3525, 3781, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1795, 2502, 2758, - 3014, 3270, 3526, 3782, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1796, 2503, 2759, - 3015, 3271, 3527, 3783, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1788, 1789, 1790, 1791, 1792, 1797, 1798, 1799, 1828, 2504, 2760, - 3016, 3272, 3528, 3784, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1829, 2505, 2761, - 3017, 3273, 3529, 3785, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1830, 2506, 2762, - 3018, 3274, 3530, 3786, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1831, 2507, 2763, - 3019, 3275, 3531, 3787, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1824, 1825, 1826, 1827, 1832, 1833, 1834, 1835, 1864, 2508, 2764, - 3020, 3276, 3532, 3788, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1836, 1837, 1838, 1839, 1840, 1841, 1842, 1843, 1865, 2509, 2765, - 3021, 3277, 3533, 3789, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1866, 2510, 2766, - 3022, 3278, 3534, 3790, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1852, 1853, 1854, 1855, 1856, 1857, 1858, 1859, 1867, 2511, 2767, - 3023, 3279, 3535, 3791, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1860, 1861, 1862, 1863, 1868, 1869, 1870, 1871, 1899, 2512, 2768, - 3024, 3280, 3536, 3792, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, 1900, 2513, 2769, - 3025, 3281, 3537, 3793, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1901, 2514, 2770, - 3026, 3282, 3538, 3794, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, 1902, 2515, 2771, - 3027, 3283, 3539, 3795, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1896, 1897, 1898, 1903, 1904, 1905, 1906, 1907, 1934, 2516, 2772, - 3028, 3284, 3540, 3796, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1935, 2517, 2773, - 3029, 3285, 3541, 3797, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1936, 2518, 2774, - 3030, 3286, 3542, 3798, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1937, 2519, 2775, - 3031, 3287, 3543, 3799, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1932, 1933, 1938, 1939, 1940, 1941, 1942, 1943, 1969, 2520, 2776, - 3032, 3288, 3544, 3800, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1944, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1970, 2521, 2777, - 3033, 3289, 3545, 3801, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, 1971, 2522, 2778, - 3034, 3290, 3546, 3802, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1972, 2523, 2779, - 3035, 3291, 3547, 3803, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1968, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 2004, 2524, 2780, - 3036, 3292, 3548, 3804, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2005, 2525, 2781, - 3037, 3293, 3549, 3805, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 2006, 2526, 2782, - 3038, 3294, 3550, 3806, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2038, 2527, 2783, - 3039, 3295, 3551, 3807, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2039, 2528, 2784, - 3040, 3296, 3552, 3808, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2040, 2529, 2785, - 3041, 3297, 3553, 3809, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2041, 2530, 2786, - 3042, 3298, 3554, 3810, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2031, 2032, 2033, 2034, 2035, 2036, 2037, 2042, 2074, 2531, 2787, - 3043, 3299, 3555, 3811, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2075, 2532, 2788, - 3044, 3300, 3556, 3812, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2076, 2533, 2789, - 3045, 3301, 3557, 3813, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2077, 2534, 2790, - 3046, 3302, 3558, 3814, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2067, 2068, 2069, 2070, 2071, 2072, 2073, 2078, 2109, 2535, 2791, - 3047, 3303, 3559, 3815, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2079, 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2110, 2536, 2792, - 3048, 3304, 3560, 3816, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2087, 2088, 2089, 2090, 2091, 2092, 2093, 2094, 2111, 2537, 2793, - 3049, 3305, 3561, 3817, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2095, 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2112, 2538, 2794, - 3050, 3306, 3562, 3818, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2103, 2104, 2105, 2106, 2107, 2108, 2113, 2114, 2144, 2539, 2795, - 3051, 3307, 3563, 3819, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2145, 2540, 2796, - 3052, 3308, 3564, 3820, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2146, 2541, 2797, - 3053, 3309, 3565, 3821, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2147, 2542, 2798, - 3054, 3310, 3566, 3822, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2139, 2140, 2141, 2142, 2143, 2148, 2149, 2150, 2179, 2543, 2799, - 3055, 3311, 3567, 3823, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2151, 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2180, 2544, 2800, - 3056, 3312, 3568, 3824, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2181, 2545, 2801, - 3057, 3313, 3569, 3825, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2182, 2546, 2802, - 3058, 3314, 3570, 3826, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2175, 2176, 2177, 2178, 2183, 2184, 2185, 2186, 2214, 2547, 2803, - 3059, 3315, 3571, 3827, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2187, 2188, 2189, 2190, 2191, 2192, 2193, 2194, 2215, 2548, 2804, - 3060, 3316, 3572, 3828, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2216, 2549, 2805, - 3061, 3317, 3573, 3829, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2203, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2217, 2550, 2806, - 3062, 3318, 3574, 3830, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2211, 2212, 2213, 2218, 2219, 2220, 2221, 2222, 2249, 2551, 2807, - 3063, 3319, 3575, 3831, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2223, 2224, 2225, 2226, 2227, 2228, 2229, 2230, 2250, 2552, 2808, - 3064, 3320, 3576, 3832, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2251, 2553, 2809, - 3065, 3321, 3577, 3833, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2252, 2554, 2810, - 3066, 3322, 3578, 3834, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2247, 2248, 2253, 2254, 2255, 2256, 2257, 2258, 2284, 2555, 2811, - 3067, 3323, 3579, 3835, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2285, 2556, 2812, - 3068, 3324, 3580, 3836, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2267, 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2286, 2557, 2813, - 3069, 3325, 3581, 3837, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2287, 2558, 2814, - 3070, 3326, 3582, 3838, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2283, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2303, 2559, 2815, - 3071, 3327, 3583, 3839, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2304, 2560, 2816, - 3072, 3328, 3584, 3840, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] - - page_indices_no_dp = sum(page_indices_no_dp, [] ) - if dp_size == 1: - page_indices = jnp.array(page_indices_no_dp, dtype=jnp.int32) - elif dp_size == 2: - page_indices = jnp.array(page_indices_rank0 + page_indices_rank1, dtype=jnp.int32) - - attention_sink = None - sm_scale = head_dim ** -0.5 - - print(f"\nInput creation (dp_size={dp_size}):") - print(f" q: {q.shape}") - print(f" kv_cache: {kv_cache.shape}") - print(f" kv_lens: {kv_lens.shape} = {kv_lens}") - print(f" page_indices: {page_indices.shape}", page_indices) - print(f" cu_q_lens: {cu_q_lens.shape} = {cu_q_lens}") - print(f" distribution: {distribution.shape} = {distribution}") - - return { - 'q': q, - 'k': k, - 'v': v, - 'kv_cache': kv_cache, - 'kv_lens': kv_lens, - 'page_indices': page_indices, - 'cu_q_lens': cu_q_lens, - 'distribution': distribution, - 'attention_sink': attention_sink, - 'sm_scale': sm_scale, - } - -def run(inputs, dump_dir, dp_size=2, num_devices=8, dp=True): - - print("\n" + "="*80) - if dp: - print(f"Running WITH Data Parallelism (dp_size={dp_size}, num_devices={num_devices})") - else: - print(f"Running WITHOUT Data Parallelism (dp_size={dp_size}, num_devices={num_devices})") - print("="*80) - - print(f"Input shapes:") - for key, val in inputs.items(): - if isinstance(val, jax.Array): - print(f" {key}: {val.shape} {val.dtype}") - - devices = jax.devices() - dp_size = 2 - if dp: - device_array = np.array(devices[:8]).reshape(dp_size, -1) - else: - device_array = np.array(devices[:4]).reshape(1, -1) - mesh = Mesh( - device_array, - axis_names=(ShardingAxisName.ATTN_DATA, ShardingAxisName.ATTN_HEAD), - ) - print(f"Mesh: {mesh}") - - def _attention_wrapper(q, k, v, kv_cache, kv_lens, page_indices, cu_q_lens, distribution): - - attention_metadata = AttentionMetadata( - input_positions=None, - block_tables=page_indices, - seq_lens=kv_lens, - query_start_loc=cu_q_lens, - request_distribution=distribution, - ) - - batch_size, num_heads, head_dim = q.shape - num_kv_heads = k.shape[1] - - # Reshape q, k, v to match _jax_attn_func expectations - q_reshaped = q.reshape(batch_size, num_heads * head_dim) - k_reshaped = k.reshape(batch_size, num_kv_heads * head_dim) - v_reshaped = v.reshape(batch_size, num_kv_heads * head_dim) - - new_kv_cache, outputs = _jax_attn_func( - kv_cache=kv_cache, - q=q_reshaped, - k=k_reshaped, - v=v_reshaped, - sinks=None, - attention_metadata=attention_metadata, - mesh=mesh, - scale=inputs['sm_scale'], - head_size=head_dim, - num_heads=num_heads, - num_kv_heads=num_kv_heads, - q_scale=None, - k_scale=None, - v_scale=None, - sliding_window=None, - ) - - outputs = outputs.reshape(batch_size, num_heads, head_dim) - - return outputs, new_kv_cache - - attention_fn = jax.jit(_attention_wrapper) - - # Lower and dump HLO - print("\nLowering computation...") - lowered = attention_fn.lower( - inputs['q'], - inputs['k'], - inputs['v'], - inputs['kv_cache'], - inputs['kv_lens'], - inputs['page_indices'], - inputs['cu_q_lens'], - inputs['distribution'], - ) - - # Dump HLO - hlo_text = lowered.as_text() - if dp: - hlo_path = Path(dump_dir) / f"with_dp_{dp_size}_hlo.txt" - else: - hlo_path = Path(dump_dir) / f"no_dp_hlo.txt" - hlo_path.write_text(hlo_text) - print(f"✓ HLO dumped to: {hlo_path}") - - qkv_spec = P(ShardingAxisName.ATTN_DATA, ShardingAxisName.ATTN_HEAD, None) - kv_cache_spec = P(ShardingAxisName.ATTN_DATA, None, ShardingAxisName.ATTN_HEAD, None, None) - - inputs['q'] = jax.device_put(inputs['q'], NamedSharding(mesh, qkv_spec)) - inputs['k'] = jax.device_put(inputs['k'], NamedSharding(mesh, qkv_spec)) - inputs['v'] = jax.device_put(inputs['v'], NamedSharding(mesh, qkv_spec)) - inputs['kv_cache'] = jax.device_put(inputs['kv_cache'], NamedSharding(mesh, kv_cache_spec)) - inputs['kv_lens'] = jax.device_put(inputs['kv_lens'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) - inputs['page_indices'] = jax.device_put(inputs['page_indices'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) - inputs['cu_q_lens'] = jax.device_put(inputs['cu_q_lens'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) - inputs['distribution'] = jax.device_put(inputs['distribution'], NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) - - # Warm up - print("Warming up...") - for _ in range(3): - result = attention_fn( - inputs['q'], - inputs['k'], - inputs['v'], - inputs['kv_cache'], - inputs['kv_lens'], - inputs['page_indices'], - inputs['cu_q_lens'], - inputs['distribution'], - ) - jax.block_until_ready(result) - - # Benchmark - print("Benchmarking ...") - times = [] - dump_dir = f"gs://wenxindong-vm/trace/debug/gptoss/kernel/dp{dp}" - jax.profiler.start_trace(dump_dir) - for i in range(10): - start = time.time() - result = attention_fn( - inputs['q'], - inputs['k'], - inputs['v'], - inputs['kv_cache'], - inputs['kv_lens'], - inputs['page_indices'], - inputs['cu_q_lens'], - inputs['distribution'], - ) - jax.block_until_ready(result) - elapsed = time.time() - start - times.append(elapsed) - print(f" Run {i+1}: {elapsed*1000:.2f} ms") - jax.profiler.stop_trace() - avg_time = sum(times) / len(times) - print(f"\n✓ Average time: {avg_time*1000:.2f} ms") - - return result, avg_time - - -def main(): - parser = argparse.ArgumentParser(description="Reproduce attention DP issue") - parser.add_argument( - '--mode', - type=str, - choices=['no_dp', 'with_dp', 'both'], - default='both', - help='Which mode to run' - ) - parser.add_argument( - '--dump_dir', - type=str, - default='/tmp/attention_dp_repro', - help='Directory to dump HLO files' - ) - args = parser.parse_args() - - # Create dump directory - dump_dir = Path(args.dump_dir) - dump_dir.mkdir(parents=True, exist_ok=True) - print(f"HLO dump directory: {dump_dir}") - - # Note: We'll create DP-specific inputs when running with_dp mode - inputs_no_dp = None - inputs_with_dp = None - - print(f"\nJAX version: {jax.__version__}") - print(f"Available devices: {jax.devices()}") - print(f"Device count: {jax.device_count()}") - - results = {} - - # Run without DP - if args.mode in ['no_dp', 'both']: - print("\nCreating inputs for no_dp mode...") - inputs_no_dp = create_test_inputs(dp_size=1) - result, avg_time = run(inputs_no_dp, dump_dir, dp=False) - results['no_dp'] = {'result': result, 'time': avg_time} - - # Run with DP - if args.mode in ['with_dp', 'both']: - print(f"\nCreating inputs for with_dp mode (dp_size={2})...") - inputs_with_dp = create_test_inputs( dp_size=2) - result, avg_time = run(inputs_with_dp, dump_dir, dp=True) - results['with_dp'] = {'result': result, 'time': avg_time} - - # Compare results - if args.mode == 'both': - print("\n" + "="*80) - print("COMPARISON") - print("="*80) - - no_dp_time = results['no_dp']['time'] - with_dp_time = results['with_dp']['time'] - speedup = no_dp_time / with_dp_time - slowdown = with_dp_time / no_dp_time - - print(f"No DP time: {no_dp_time*1000:.2f} ms") - print(f"With DP time: {with_dp_time*1000:.2f} ms") - - if speedup > 1.0: - print(f"✓ DP is {speedup:.2f}x FASTER") - else: - print(f"✗ DP is {slowdown:.2f}x SLOWER") - - print("\n" + "="*80) - print(f"\n1. Compare HLO files in: {dump_dir}") - print(f" - no_dp_hlo.txt") - print(f" - with_dp_2_hlo.txt") - print(f"\n2. Or use XLA dump for more detail:") - print(f" XLA_FLAGS='--xla_dump_to={dump_dir}/xla_dump --xla_dump_hlo_as_text' python {__file__} --mode {args.mode}") - print() - - -if __name__ == '__main__': - main() - - -# ================================================================================ -# COMPARISON -# ================================================================================ -# No DP time: 2.63 ms -# With DP time: 2.62 ms -# ✓ DP is 1.00x FASTER \ No newline at end of file From 8970b8013aa4173594dffa6b720bd90acb38b5f7 Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Wed, 19 Nov 2025 23:49:52 +0000 Subject: [PATCH 3/9] only submit model dp --- tpu_inference/layers/common/sharding.py | 15 ++--- tpu_inference/layers/vllm/fused_moe.py | 67 ++++++++----------- .../layers/vllm/quantization/common.py | 21 ++---- .../layers/vllm/quantization/unquantized.py | 22 +++--- tpu_inference/layers/vllm/sharding.py | 10 ++- 5 files changed, 53 insertions(+), 82 deletions(-) diff --git a/tpu_inference/layers/common/sharding.py b/tpu_inference/layers/common/sharding.py index 930096156..127a74fe6 100644 --- a/tpu_inference/layers/common/sharding.py +++ b/tpu_inference/layers/common/sharding.py @@ -121,7 +121,6 @@ def from_vllm_config(cls, if enable_dp_attention: # Replicate attention layer when num_kv_heads < TP num_kv_heads = vllm_config.model_config.get_total_num_kv_heads() - kv_dtype = utils.get_jax_dtype_from_str_dtype( vllm_config.cache_config.cache_dtype) or jnp.bfloat16 packing = 4 // jnp.dtype(kv_dtype).itemsize @@ -129,11 +128,6 @@ def from_vllm_config(cls, # duplicate KV heads across devices, wasting kv cache memory. # Use attention DP instead to reduce per-device num_kv_heads and # eliminate this waste. - - # if head_dim is 64, multiply packing by 2 - if vllm_config.model_config.get_head_size() == 64: - packing *= 2 - num_kv_heads_per_device_in_kv_cache = (num_kv_heads * 2) / packing attn_dp = max( int(tensor_parallelism // num_kv_heads_per_device_in_kv_cache), @@ -172,10 +166,11 @@ def validate(cls, vllm_config, sharding_strategy): f"LoRA is not supported with data parallelism " f"(DP size: {total_dp_size}). Please disable LoRA or " f"set data parallelism to 1.") - # if not os.environ.get("NEW_MODEL_DESIGN", False): - # raise ValueError( - # "Must run DP with NEW_MODEL_DESIGN enabled. Please set the " - # "NEW_MODEL_DESIGN=True.") + if sharding_strategy.attention_data_parallelism > 1: + if not os.environ.get("NEW_MODEL_DESIGN", False): + raise ValueError( + "Must run Attention DP with NEW_MODEL_DESIGN enabled. Please set the " + "NEW_MODEL_DESIGN=True.") @property def total_dp_size(self) -> int: diff --git a/tpu_inference/layers/vllm/fused_moe.py b/tpu_inference/layers/vllm/fused_moe.py index 9f676f4db..7770acee2 100644 --- a/tpu_inference/layers/vllm/fused_moe.py +++ b/tpu_inference/layers/vllm/fused_moe.py @@ -123,8 +123,8 @@ def tensor_sharded_gmm_merged_column_parallel( gmm_result = shard_map( _gmm, mesh=mesh, - in_specs=(P(ShardingAxisName.MLP_DATA, None), P(None, ShardingAxisName.MLP_TENSOR, None), P(ShardingAxisName.MLP_DATA)), - out_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR)), + in_specs=(P("data", None), P(None, "model", None), P("data")), + out_specs=(P("data", "model")), check_rep=False, )(lhs, rhs, group_sizes) @@ -136,12 +136,12 @@ def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): gmm_result = shard_map( _add_bias, mesh=mesh, - in_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR), P(None, ShardingAxisName.MLP_TENSOR), P(ShardingAxisName.MLP_DATA)), - out_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR)), + in_specs=(P("data", "model"), P(None, "model"), P("data")), + out_specs=(P("data", "model")), check_rep=False, )(gmm_result, rhs_bias, group_sizes) - n_shards = mesh.shape['model'] * mesh.shape.get('attn_dp', 1) + n_shards = mesh.shape["model"] output_sizes = [intermediate_size, intermediate_size] return slice_sharded_tensor_for_concatenation(gmm_result, output_sizes, @@ -171,17 +171,16 @@ def tensor_sharded_gmm_row_parallel( def _gmm_all_reduce(lhs, rhs, group_sizes): r = _gmm(lhs, rhs, group_sizes) - return jax.lax.psum(r, axis_name=ShardingAxisName.MLP_TENSOR) + return jax.lax.psum(r, axis_name="model") gmm_result = shard_map( _gmm_all_reduce, mesh=mesh, - in_specs=(P(ShardingAxisName.MLP_DATA, ShardingAxisName.MLP_TENSOR), - P(None, None, ShardingAxisName.MLP_TENSOR), P(ShardingAxisName.MLP_DATA)), - out_specs=(P(ShardingAxisName.MLP_DATA)), + in_specs=(P("data", "model"), + P(None, None, "model"), P("data")), + out_specs=(P("data")), check_rep=False, )(lhs, rhs, group_sizes) - # jax.debug.print("gmm_result before bias {} {}", gmm_result.sum(), gmm_result.ravel()[:10]) if rhs_bias is not None: def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): rhs_bis = jnp.repeat(rhs_bias_local, group_sizes_global, 0, total_repeat_length=m//mesh.shape["data"]) @@ -190,8 +189,8 @@ def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): gmm_result = shard_map( _add_bias, mesh=mesh, - in_specs=(P(ShardingAxisName.MLP_DATA), P(), P(ShardingAxisName.MLP_DATA)), - out_specs=(P(ShardingAxisName.MLP_DATA)), + in_specs=(P("data"), P(), P("data")), + out_specs=(P("data")), check_rep=False, )(gmm_result, rhs_bias, group_sizes) @@ -215,7 +214,8 @@ def expert_sharded_gmm( num_experts_per_shard = num_experts // ep_size group_offset = jnp.arange(0, num_experts, num_experts_per_shard) group_offset = jax.lax.with_sharding_constraint( - group_offset, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) + group_offset, NamedSharding(mesh, P("model"))) + def _gmm(lhs, rhs, group_sizes, group_offset): # Group offset for this shard. `group_offset` is sharded, and in this # sharded function, it has only 1 element and `group_offset.shape` is @@ -254,9 +254,8 @@ def _gmm(lhs, rhs, group_sizes, group_offset): gmm_res = shard_map( _gmm, mesh=mesh, - in_specs=(P(ShardingAxisName.MLP_DATA, None), P(ShardingAxisName.EXPERT, None, - None), P(ShardingAxisName.MLP_DATA), P(ShardingAxisName.EXPERT)), - out_specs=(P(ShardingAxisName.EXPERT, None)), + in_specs=(P("data", None), P("model", None, None), P("data"), P("model")), + out_specs=(P("model", None)), check_rep=False, )(lhs, rhs, group_sizes, group_offset) @@ -275,11 +274,12 @@ def _gmm(lhs, rhs, group_sizes, group_offset): recv_sizes = send_sizes input_offsets = jax.lax.with_sharding_constraint( - input_offsets, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) + input_offsets, NamedSharding(mesh, P("model"))) send_sizes = jax.lax.with_sharding_constraint( - send_sizes, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) + send_sizes, NamedSharding(mesh, P("model"))) output_offsets = jax.lax.with_sharding_constraint( - output_offsets, NamedSharding(mesh, P(ShardingAxisName.EXPERT))) + output_offsets, NamedSharding(mesh, P("model"))) + def _ragged_all_to_all(operand, input_offsets, send_sizes, output_offsets, recv_sizes): output = jnp.zeros_like(operand) @@ -310,7 +310,8 @@ def _ragged_all_to_all(operand, input_offsets, send_sizes, output_offsets, send_sizes_of_shard, output_offsets_of_shard, recv_sizes_of_shard, - axis_name=ShardingAxisName.EXPERT) + axis_name="model") + # Use ragged_all_to_all to send the result from gmm for each expert to all # the shards. In the working example, the result would be: # A, A, A, A A, A, A, A A, A, A, A A, A, A, A @@ -331,8 +332,7 @@ def _ragged_all_to_all(operand, input_offsets, send_sizes, output_offsets, return shard_map( _ragged_all_to_all, mesh=mesh, - in_specs=(P(ShardingAxisName.EXPERT, None), P(ShardingAxisName.EXPERT), - P(ShardingAxisName.EXPERT), P(ShardingAxisName.EXPERT), P()), + in_specs=(P("model", None), P("model"), P("model"), P("model"), P()), out_specs=(P()), check_rep=False, )(gmm_res, input_offsets, send_sizes, output_offsets, recv_sizes) @@ -368,7 +368,7 @@ def fused_moe_func( hidden_size = hidden_states.shape[-1] num_tokens = hidden_states.size // hidden_size assert global_num_experts == w1.shape[0] - ep_size = mesh.shape['model'] * mesh.shape.get("attn_dp", 1) # only used if use_ep is True. + ep_size = mesh.shape["model"] # only used if use_ep is True. intermediate_size = w2.shape[-1] dtype = hidden_states.dtype assert (num_tokens * topk) % 16 == 0, ( @@ -398,13 +398,10 @@ def _process_tokens_locally(hidden_states_local, topk_indices_local): x, group_sizes, topk_argsort_revert_indices = shard_map( _process_tokens_locally, mesh=mesh, - in_specs=(P(ShardingAxisName.ATTN_DATA, None), P(ShardingAxisName.ATTN_DATA, None)), - out_specs=(P(ShardingAxisName.ATTN_DATA, None), P(ShardingAxisName.ATTN_DATA), P(ShardingAxisName.ATTN_DATA)), + in_specs=(P("data", None), P("data", None)), + out_specs=(P("data", None), P("data"), P("data")), check_rep=False, )(hidden_states, topk_indices) - - # jax.debug.print("hidden_state before gmm {} {}", x.sum(), x.ravel()[:10]) - # jax.debug.print("group_sizes {} {}", group_sizes.sum(), group_sizes) if use_ep: x = expert_sharded_gmm( x, @@ -426,11 +423,8 @@ def _process_tokens_locally(hidden_states_local, topk_indices_local): mesh=mesh, intermediate_size=intermediate_size, ) - # jax.debug.print("hidden_state after first gmm x1 {} {}", x1.sum(), x1.ravel()[:10]) - # jax.debug.print("hidden_state after first gmm x2 {} {}", x2.sum(), x2.ravel()[:10]) x = activation_fn(activation, x1, x2) - # jax.debug.print("hidden_state after activation {} {}", x.sum(), x.ravel()[:10]) if use_ep: x = expert_sharded_gmm( x, @@ -443,7 +437,7 @@ def _process_tokens_locally(hidden_states_local, topk_indices_local): ) else: x = jax.lax.with_sharding_constraint( - x, NamedSharding(mesh, P(ShardingAxisName.MLP_DATA, "model"))) + x, NamedSharding(mesh, P("data", "model"))) x = tensor_sharded_gmm_row_parallel( x, w2, @@ -452,7 +446,6 @@ def _process_tokens_locally(hidden_states_local, topk_indices_local): transpose_rhs=True, mesh=mesh, ) - # jax.debug.print("hidden_state after second gmm {} {}", x.sum(), x.ravel()[:10]) def _finalize_output(x_local, topk_argsort_revert_indices_local, topk_weights_local): x_local = x_local[topk_argsort_revert_indices_local].reshape(-1, topk, hidden_size) @@ -463,16 +456,14 @@ def _finalize_output(x_local, topk_argsort_revert_indices_local, topk_weights_lo x = shard_map( _finalize_output, mesh=mesh, - in_specs=(P(ShardingAxisName.ATTN_DATA, None), P(ShardingAxisName.ATTN_DATA), P(ShardingAxisName.ATTN_DATA, None)), - out_specs=(P(ShardingAxisName.ATTN_DATA, None)), + in_specs=(P("data", None), P("data"), P("data", None)), + out_specs=(P("data", None)), check_rep=False, )(x, topk_argsort_revert_indices, topk_weights) - # jax.debug.print("hidden_state after finalize output {} {}", x.sum(), x.ravel()[:10]) x = x.reshape(orig_shape) if reduce_results: - x = jax.lax.with_sharding_constraint(x, NamedSharding(mesh, P(ShardingAxisName.ATTN_DATA))) - # jax.debug.print("hidden_state after reducing result {} {}", x.sum(), x.ravel()[:10]) + x = jax.lax.with_sharding_constraint(x, NamedSharding(mesh, P("data"))) return x diff --git a/tpu_inference/layers/vllm/quantization/common.py b/tpu_inference/layers/vllm/quantization/common.py index 7872c4aaa..2b36a795e 100644 --- a/tpu_inference/layers/vllm/quantization/common.py +++ b/tpu_inference/layers/vllm/quantization/common.py @@ -11,7 +11,6 @@ ReplicatedLinear, RowParallelLinear) -from tpu_inference.layers.common.sharding import ShardingAxisName from tpu_inference.layers.vllm.linear_common import \ get_model_matmul_fusion_assignment from tpu_inference.utils import TPU_SECOND_LAST_MINOR @@ -35,23 +34,15 @@ def __init__(self, vllm_config: VllmConfig, mesh: Mesh, layer: LinearBase): self.enable_sequence_parallelism = vllm_config.compilation_config.pass_config.enable_sequence_parallelism self.input_sharding = None self.output_sharding = None - self.tp_size = self.mesh.shape['model'] * self.mesh.shape.get( - 'attn_dp', 1) if isinstance(layer, RowParallelLinear): - self.weight_sharding = P(None, ShardingAxisName.MLP_TENSOR) + self.weight_sharding = P(None, "model") if self.enable_sequence_parallelism: - self.output_sharding = P(ShardingAxisName.MLP_TENSOR, None) + self.output_sharding = P("model", None) elif isinstance(layer, ColumnParallelLinear): - if isinstance(layer, QKVParallelLinear): - self.input_sharding = P(ShardingAxisName.ATTN_DATA, None) - self.weight_sharding = P('model', None) - self.output_sharding = P(ShardingAxisName.ATTN_DATA, "model") - else: - self.weight_sharding = P(ShardingAxisName.MLP_TENSOR, None) - + self.weight_sharding = P("model", None) if self.enable_sequence_parallelism: - self.input_sharding = P(ShardingAxisName.MLP_TENSOR, None) + self.input_sharding = P("model", None) if isinstance(layer, MergedColumnParallelLinear) or isinstance( layer, QKVParallelLinear): @@ -81,7 +72,7 @@ def get_input_sharding(self, x: torchax.tensor.Tensor): if self.enable_sequence_parallelism: token_num = x.shape[0] # NOTE(chengjiyao): make sure the sharded token_num is larger than TPU_SECOND_LAST_MINOR - if token_num // self.tp_size >= TPU_SECOND_LAST_MINOR: + if token_num // self.mesh.shape["model"] >= TPU_SECOND_LAST_MINOR: return self.input_sharding else: return None @@ -91,7 +82,7 @@ def get_output_sharding(self, x: torchax.tensor.Tensor): if self.enable_sequence_parallelism: token_num = x.shape[0] # NOTE(chengjiyao): make sure the sharded token_num is larger than TPU_SECOND_LAST_MINOR - if token_num // self.tp_size >= TPU_SECOND_LAST_MINOR: + if token_num // self.mesh.shape["model"] >= TPU_SECOND_LAST_MINOR: return self.output_sharding else: return None diff --git a/tpu_inference/layers/vllm/quantization/unquantized.py b/tpu_inference/layers/vllm/quantization/unquantized.py index 674c9b139..983defbae 100644 --- a/tpu_inference/layers/vllm/quantization/unquantized.py +++ b/tpu_inference/layers/vllm/quantization/unquantized.py @@ -262,42 +262,38 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: w13_weight = jax.device_put( w13_weight, Format(Layout((0, 1, 2)), - NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None, None)))) + NamedSharding(self.mesh, P("model", None, None)))) w2_weight = jax.device_put( w2_weight, Format(Layout((0, 1, 2)), - NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None, None)))) + NamedSharding(self.mesh, P("model", None, None)))) if self.moe.has_bias: w13_bias = jax.device_put( w13_bias, Format(Layout((0, 1)), - NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None)))) + NamedSharding(self.mesh, P("model", None)))) w2_bias = jax.device_put( w2_bias, Format(Layout((0, 1)), - NamedSharding(self.mesh, P(ShardingAxisName.EXPERT, None)))) + NamedSharding(self.mesh, P("model", None)))) else: intermediate_size = w13_weight.shape[1] // 2 assert intermediate_size == w2_weight.shape[-1] output_sizes = [intermediate_size, intermediate_size] - n_shards = self.mesh.shape['model'] * self.mesh.shape.get( - "attn_dp", 1) + n_shards = self.mesh.shape["model"] assert intermediate_size % n_shards == 0 w13_weight = reorder_concatenated_tensor_for_sharding( w13_weight, output_sizes, n_shards, dim=1) w13_weight = jax.device_put( w13_weight, - Format( - Layout((0, 1, 2)), - NamedSharding( - self.mesh, - P(None, ShardingAxisName.MLP_TENSOR, None)))) + Format(Layout((0, 1, 2)), + NamedSharding(self.mesh, P(None, "model", None)))) w2_weight = jax.device_put( w2_weight, Format(Layout((0, 1, 2)), - NamedSharding(self.mesh, P(None, None,ShardingAxisName.MLP_TENSOR)))) + NamedSharding(self.mesh, P(None, None, "model")))) if self.moe.has_bias: w13_bias = reorder_concatenated_tensor_for_sharding( @@ -305,7 +301,7 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: w13_bias = jax.device_put( w13_bias, Format(Layout((0, 1)), - NamedSharding(self.mesh, P(None, ShardingAxisName.MLP_TENSOR)))) + NamedSharding(self.mesh, P(None, "model")))) w2_bias = jax.device_put( w2_bias, Format(Layout((0, 1)), diff --git a/tpu_inference/layers/vllm/sharding.py b/tpu_inference/layers/vllm/sharding.py index 043e6a64f..7dcf903ec 100644 --- a/tpu_inference/layers/vllm/sharding.py +++ b/tpu_inference/layers/vllm/sharding.py @@ -110,8 +110,7 @@ def _shard_tensor_to_tpu_replicated(tensor: torch.Tensor, def _shard_vocab_parallel_embedding(layer: VocabParallelEmbedding, mesh: Mesh) -> None: weight = _convert_to_torchax_and_shard( - layer.weight, NamedSharding(mesh, P(ShardingAxisName.MLP_TENSOR, - None))) + layer.weight, NamedSharding(mesh, P('model', None))) layer.weight = Parameter(weight, requires_grad=False) @@ -120,12 +119,11 @@ def _shard_lm_head(layer: ParallelLMHead, mesh: Mesh): # if that config is set, then we should not create new weights but reuse the # weight from VocabParallelEmbedding weight = _convert_to_torchax_and_shard( - layer.weight, NamedSharding(mesh, P(ShardingAxisName.MLP_TENSOR, - None))) + layer.weight, NamedSharding(mesh, P('model', None))) layer.weight = Parameter(weight, requires_grad=False) if layer.bias is not None: - bias = _convert_to_torchax_and_shard( - layer.bias, NamedSharding(mesh, P(ShardingAxisName.MLP_TENSOR))) + bias = _convert_to_torchax_and_shard(layer.bias, + NamedSharding(mesh, P('model'))) layer.bias = Parameter(bias, requires_grad=False) From 270e511d9f287d8f470c84ab2f95fb8051260cda Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Wed, 19 Nov 2025 23:51:03 +0000 Subject: [PATCH 4/9] wip --- tpu_inference/layers/vllm/fused_moe.py | 4 +++- tpu_inference/layers/vllm/quantization/unquantized.py | 2 -- tpu_inference/layers/vllm/sharding.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tpu_inference/layers/vllm/fused_moe.py b/tpu_inference/layers/vllm/fused_moe.py index 7770acee2..24d76bb87 100644 --- a/tpu_inference/layers/vllm/fused_moe.py +++ b/tpu_inference/layers/vllm/fused_moe.py @@ -8,7 +8,7 @@ from tpu_inference.layers.vllm.linear_common import \ slice_sharded_tensor_for_concatenation -from tpu_inference.layers.common.sharding import ShardingAxisName + P = PartitionSpec @@ -374,6 +374,7 @@ def fused_moe_func( assert (num_tokens * topk) % 16 == 0, ( "The kernel requires num_tokens * topk to be a multiple of " f"16 but got {num_tokens}*{topk}={num_tokens*topk}") + hidden_states = hidden_states.reshape(num_tokens, hidden_size) gating_output = gating_output.reshape(num_tokens, global_num_experts) @@ -425,6 +426,7 @@ def _process_tokens_locally(hidden_states_local, topk_indices_local): ) x = activation_fn(activation, x1, x2) + if use_ep: x = expert_sharded_gmm( x, diff --git a/tpu_inference/layers/vllm/quantization/unquantized.py b/tpu_inference/layers/vllm/quantization/unquantized.py index 983defbae..8f2a622cd 100644 --- a/tpu_inference/layers/vllm/quantization/unquantized.py +++ b/tpu_inference/layers/vllm/quantization/unquantized.py @@ -25,8 +25,6 @@ from tpu_inference.kernels.fused_moe.v1.kernel import fused_ep_moe from tpu_inference.layers.common.quant_methods import (UNQUANTIZED, get_tpu_quant_method) -from tpu_inference.layers.common.sharding import ShardingAxisName - from tpu_inference.layers.vllm.fused_moe import fused_moe_func_padded from tpu_inference.layers.vllm.linear_common import ( reorder_concatenated_tensor_for_sharding, diff --git a/tpu_inference/layers/vllm/sharding.py b/tpu_inference/layers/vllm/sharding.py index 7dcf903ec..b9fd4fdd9 100644 --- a/tpu_inference/layers/vllm/sharding.py +++ b/tpu_inference/layers/vllm/sharding.py @@ -19,7 +19,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding) -from tpu_inference.layers.common.sharding import ShardingAxisName from tpu_inference import envs from tpu_inference.logger import init_logger From a9d51546602f030e2f611f57392c7a8dd6df11c3 Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Wed, 19 Nov 2025 23:54:05 +0000 Subject: [PATCH 5/9] wip --- tpu_inference/layers/vllm/fused_moe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tpu_inference/layers/vllm/fused_moe.py b/tpu_inference/layers/vllm/fused_moe.py index 24d76bb87..674c05e8e 100644 --- a/tpu_inference/layers/vllm/fused_moe.py +++ b/tpu_inference/layers/vllm/fused_moe.py @@ -209,7 +209,7 @@ def expert_sharded_gmm( # adapted from https://github.com/pytorch/xla/blob/1d409399474197c484894be90b75d9855393dda5/torch_xla/experimental/custom_kernel.py#L1401 m, k, g = lhs.shape[0], lhs.shape[1], rhs.shape[0] n = rhs.shape[1] if transpose_rhs else rhs.shape[2] - tm, tk, tn = _get_tiling_size_for_gmm_kernel(m//mesh.shape["data"], k, n, g) + tm, tk, tn = _get_tiling_size_for_gmm_kernel(m, k, n, g) num_experts_per_shard = num_experts // ep_size group_offset = jnp.arange(0, num_experts, num_experts_per_shard) @@ -254,7 +254,7 @@ def _gmm(lhs, rhs, group_sizes, group_offset): gmm_res = shard_map( _gmm, mesh=mesh, - in_specs=(P("data", None), P("model", None, None), P("data"), P("model")), + in_specs=(P(), P("model", None, None), P(), P("model")), out_specs=(P("model", None)), check_rep=False, )(lhs, rhs, group_sizes, group_offset) From b10487a010e8269d03ed91ffc15f9a1a8b2d69cf Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Thu, 20 Nov 2025 05:31:47 +0000 Subject: [PATCH 6/9] formatting --- tpu_inference/layers/vllm/fused_moe.py | 47 ++++++++++++++++---------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/tpu_inference/layers/vllm/fused_moe.py b/tpu_inference/layers/vllm/fused_moe.py index 674c05e8e..b19128475 100644 --- a/tpu_inference/layers/vllm/fused_moe.py +++ b/tpu_inference/layers/vllm/fused_moe.py @@ -110,7 +110,8 @@ def tensor_sharded_gmm_merged_column_parallel( # adapted from https://github.com/pytorch/xla/blob/1d409399474197c484894be90b75d9855393dda5/torch_xla/experimental/custom_kernel.py#L1401 m, k, g = lhs.shape[0], lhs.shape[1], rhs.shape[0] n = rhs.shape[1] if transpose_rhs else rhs.shape[2] - tm, tk, tn = _get_tiling_size_for_gmm_kernel(m//mesh.shape["data"], k, n, g) + tm, tk, tn = _get_tiling_size_for_gmm_kernel(m // mesh.shape["data"], k, n, + g) _gmm = functools.partial( gmm, @@ -129,10 +130,14 @@ def tensor_sharded_gmm_merged_column_parallel( )(lhs, rhs, group_sizes) if rhs_bias is not None: + def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): - rhs_bis = jnp.repeat(rhs_bias_local, group_sizes_global, 0, total_repeat_length=m//mesh.shape["data"]) + rhs_bis = jnp.repeat(rhs_bias_local, + group_sizes_global, + 0, + total_repeat_length=m // mesh.shape["data"]) return (gmm_result_local + rhs_bis).astype(gmm_result_local.dtype) - + gmm_result = shard_map( _add_bias, mesh=mesh, @@ -159,7 +164,8 @@ def tensor_sharded_gmm_row_parallel( # adapted from https://github.com/pytorch/xla/blob/1d409399474197c484894be90b75d9855393dda5/torch_xla/experimental/custom_kernel.py#L1401 m, k, g = lhs.shape[0], lhs.shape[1], rhs.shape[0] n = rhs.shape[1] if transpose_rhs else rhs.shape[2] - tm, tk, tn = _get_tiling_size_for_gmm_kernel(m//mesh.shape["data"], k, n, g) + tm, tk, tn = _get_tiling_size_for_gmm_kernel(m // mesh.shape["data"], k, n, + g) _gmm = functools.partial( gmm, @@ -176,16 +182,19 @@ def _gmm_all_reduce(lhs, rhs, group_sizes): gmm_result = shard_map( _gmm_all_reduce, mesh=mesh, - in_specs=(P("data", "model"), - P(None, None, "model"), P("data")), - out_specs=(P("data")), - check_rep=False, + in_specs=(P("data", "model"), P(None, None, "model"), P("data")), + out_specs=(P("data")), + check_rep=False, )(lhs, rhs, group_sizes) if rhs_bias is not None: + def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): - rhs_bis = jnp.repeat(rhs_bias_local, group_sizes_global, 0, total_repeat_length=m//mesh.shape["data"]) + rhs_bis = jnp.repeat(rhs_bias_local, + group_sizes_global, + 0, + total_repeat_length=m // mesh.shape["data"]) return (gmm_result_local + rhs_bis).astype(gmm_result_local.dtype) - + gmm_result = shard_map( _add_bias, mesh=mesh, @@ -389,13 +398,15 @@ def _process_tokens_locally(hidden_states_local, topk_indices_local): topk_indices_flat = topk_indices_local.flatten() topk_argsort_indices = jnp.argsort(topk_indices_flat) topk_argsort_revert_indices = jnp.argsort(topk_argsort_indices) - token_indices = jnp.arange(num_tokens_local, dtype=jnp.int32).repeat(topk) + token_indices = jnp.arange(num_tokens_local, + dtype=jnp.int32).repeat(topk) token_indices_sorted = token_indices[topk_argsort_indices] - group_sizes_local = jnp.bincount(topk_indices_flat, length=global_num_experts) - + group_sizes_local = jnp.bincount(topk_indices_flat, + length=global_num_experts) + x = hidden_states_local[token_indices_sorted] return x, group_sizes_local, topk_argsort_revert_indices - + x, group_sizes, topk_argsort_revert_indices = shard_map( _process_tokens_locally, mesh=mesh, @@ -449,12 +460,14 @@ def _process_tokens_locally(hidden_states_local, topk_indices_local): mesh=mesh, ) - def _finalize_output(x_local, topk_argsort_revert_indices_local, topk_weights_local): - x_local = x_local[topk_argsort_revert_indices_local].reshape(-1, topk, hidden_size) + def _finalize_output(x_local, topk_argsort_revert_indices_local, + topk_weights_local): + x_local = x_local[topk_argsort_revert_indices_local].reshape( + -1, topk, hidden_size) x_local = x_local * jnp.expand_dims(topk_weights_local, axis=-1) x_local = x_local.sum(axis=-2) return x_local - + x = shard_map( _finalize_output, mesh=mesh, From a670581d34512db90fe8efa95cd454c75175ee9d Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Thu, 20 Nov 2025 23:13:22 +0000 Subject: [PATCH 7/9] wip --- tests/e2e/test_data_parallel.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_data_parallel.py b/tests/e2e/test_data_parallel.py index 9d794df29..ec1ad8b37 100644 --- a/tests/e2e/test_data_parallel.py +++ b/tests/e2e/test_data_parallel.py @@ -14,7 +14,6 @@ def setup_new_model_design(): """Automatically set NEW_MODEL_DESIGN=True for all tests.""" os.environ['NEW_MODEL_DESIGN'] = 'True' - @pytest.fixture def test_prompts(): """Simple test prompts for data parallelism testing.""" @@ -81,9 +80,11 @@ def _run_inference_with_config(model_name: str, time.sleep(5) +@pytest.mark.parametrize("model_impl_type", ["vllm", "flax_nnx"]) def test_model_data_parallelism( test_prompts: list, sampling_params: SamplingParams, + model_impl_type: str, ): """ Test model-wise data parallelism where data=2 in the mesh axis. @@ -95,6 +96,7 @@ def test_model_data_parallelism( """ # Use Llama 1B for this test test_model = "meta-llama/Llama-3.2-1B-Instruct" + os.environ['MODEL_IMPL_TYPE'] = model_impl_type # Test with data parallelism enabled outputs = _run_inference_with_config( @@ -103,6 +105,7 @@ def test_model_data_parallelism( sampling_params=sampling_params, tensor_parallel_size=1, data_parallel_size=2, + async_scheduling=True, ) # Verify we got outputs for all prompts @@ -175,7 +178,7 @@ def test_data_parallelism_correctness( """ os.environ['SKIP_JAX_PRECOMPILE'] = '1' os.environ['VLLM_XLA_CHECK_RECOMPILATION'] = '0' - model_name = "Qwen/Qwen2.5-1.5B-Instruct" + model_name = "meta-llama/Llama-3.2-1B-Instruct" # Use a smaller subset of prompts for correctness testing small_prompts = test_prompts[:10] From 9087e9c576943e42a0dcaba39aa7abb24aea89cf Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Thu, 20 Nov 2025 23:13:47 +0000 Subject: [PATCH 8/9] formatting --- tests/e2e/test_data_parallel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/test_data_parallel.py b/tests/e2e/test_data_parallel.py index ec1ad8b37..9567731c3 100644 --- a/tests/e2e/test_data_parallel.py +++ b/tests/e2e/test_data_parallel.py @@ -14,6 +14,7 @@ def setup_new_model_design(): """Automatically set NEW_MODEL_DESIGN=True for all tests.""" os.environ['NEW_MODEL_DESIGN'] = 'True' + @pytest.fixture def test_prompts(): """Simple test prompts for data parallelism testing.""" From e1f54cb56141ddbd4bae189977d30f734e549a9e Mon Sep 17 00:00:00 2001 From: wenxindongwork Date: Sun, 23 Nov 2025 02:05:50 +0000 Subject: [PATCH 9/9] wip --- tpu_inference/layers/vllm/fused_moe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tpu_inference/layers/vllm/fused_moe.py b/tpu_inference/layers/vllm/fused_moe.py index b19128475..c455ba936 100644 --- a/tpu_inference/layers/vllm/fused_moe.py +++ b/tpu_inference/layers/vllm/fused_moe.py @@ -143,7 +143,6 @@ def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): mesh=mesh, in_specs=(P("data", "model"), P(None, "model"), P("data")), out_specs=(P("data", "model")), - check_rep=False, )(gmm_result, rhs_bias, group_sizes) n_shards = mesh.shape["model"] @@ -200,7 +199,6 @@ def _add_bias(gmm_result_local, rhs_bias_local, group_sizes_global): mesh=mesh, in_specs=(P("data"), P(), P("data")), out_specs=(P("data")), - check_rep=False, )(gmm_result, rhs_bias, group_sizes) return gmm_result