From 12f6865bf166eb34b5d5876a55e439382ad37648 Mon Sep 17 00:00:00 2001 From: Jiahao Li Date: Sun, 9 Jul 2023 20:02:17 +0800 Subject: [PATCH] Move test data out of source file (#45) --- chatglm_test.cpp | 727 ++++++++++--------------------------- tests/data/glm2_block.data | Bin 0 -> 14272 bytes tests/data/glm_block.data | Bin 0 -> 52352 bytes tests/data/layer_norm.data | Bin 0 -> 2048 bytes tests/data/linear.data | Bin 0 -> 2496 bytes tests/data/rms_norm.data | Bin 0 -> 1792 bytes tests/test_convert.py | 212 ++++++----- 7 files changed, 290 insertions(+), 649 deletions(-) create mode 100644 tests/data/glm2_block.data create mode 100644 tests/data/glm_block.data create mode 100644 tests/data/layer_norm.data create mode 100644 tests/data/linear.data create mode 100644 tests/data/rms_norm.data diff --git a/chatglm_test.cpp b/chatglm_test.cpp index 3900eb5..1d800d7 100644 --- a/chatglm_test.cpp +++ b/chatglm_test.cpp @@ -6,22 +6,19 @@ namespace chatglm { namespace fs = std::filesystem; -void expect_all_close(float *a, float *b, size_t n, float atol = 2e-4) { - for (size_t i = 0; i < n; i++) { - EXPECT_LT(std::abs(a[i] - b[i]), atol); +static inline void expect_all_close(ggml_tensor *a, ggml_tensor *b, float atol = 1e-5) { + ASSERT_EQ(a->type, b->type); + ASSERT_EQ(a->type, GGML_TYPE_F32); + ASSERT_EQ(ggml_nelements(a), ggml_nelements(b)); + int64_t numel = ggml_nelements(a); + for (int64_t i = 0; i < numel; i++) { + EXPECT_LT(std::abs(((float *)a->data)[i] - ((float *)b->data)[i]), atol); } } -bool has_shape(ggml_tensor *tensor, const std::vector &shape) { - if (tensor->n_dims != (int)shape.size()) { - return false; - } - for (int i = 0; i < tensor->n_dims; i++) { - if (tensor->ne[tensor->n_dims - 1 - i] != shape[i]) { - return false; - } - } - return true; +static inline char *map_tensor_data(char *ptr, ggml_tensor *tensor) { + tensor->data = ptr; + return ptr + ggml_nbytes(tensor); } class ChatGLMTest : public ::testing::Test { @@ -38,6 +35,11 @@ class ChatGLMTest : public ::testing::Test { ctx.gctx = GGMLContext({1024 * 1024, nullptr, false}); ctx.scratch = {0, scratch_buf.size(), scratch_buf.data()}; + + reset_cgraph(); + } + + void reset_cgraph() { ctx.gf = {}; ctx.gf.n_threads = 1; } @@ -54,616 +56,263 @@ TEST_F(ChatGLMTest, Embedding) { memcpy(x->data, x_data, sizeof(x_data)); Embedding model(&ictx, 4, 3); memcpy(model.weight->data, w_data, sizeof(w_data)); - ggml_tensor *y = model.forward(&ctx, x); + ggml_tensor *ref = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 3, 5); + ref->data = y_data; - ggml_build_forward_expand(&ctx.gf, y); + ggml_tensor *out = model.forward(&ctx, x); + + ggml_build_forward_expand(&ctx.gf, out); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref, out); } TEST_F(ChatGLMTest, Linear) { - float w_data[]{-1.1258, -1.1524, -0.2506, -0.4339, 0.8487, 0.6920, -0.3160, -2.1152, - 0.4681, -0.1577, 1.4437, 0.2660, 0.1665, 0.8744, -0.1435, -0.1116, - 0.9318, 1.2590, 2.0050, 0.0537, 0.6181, -0.4128, -0.8411, -2.3160}; - float b_data[]{0.3704, 1.4565, 0.9398, 0.7748, 0.1919, 1.2638, -1.2904, -0.7911}; - float x_data[]{-0.0209, -0.7185, 0.5186, -1.3125, 0.1920, 0.5428}; - float y_data[]{1.0919, 1.2147, 2.7089, -0.1211, -0.5142, 1.2496, -1.0504, -1.3794, - 1.4908, 2.5645, 1.2025, 1.4034, 0.0634, 2.2725, -3.5762, -1.6679}; - - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 3, 2); - memcpy(x->data, x_data, sizeof(x_data)); + fs::path test_path = fs::path(__FILE__).parent_path() / "tests/data/linear.data"; + MappedFile mapped_file(test_path.string()); + char *ptr = mapped_file.data; + + ggml_tensor *w = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 32, 16); + ptr = map_tensor_data(ptr, w); + ggml_tensor *b = ggml_new_tensor_1d(ctx.gctx.get(), GGML_TYPE_F32, 16); + ptr = map_tensor_data(ptr, b); + ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 32, 2); + ptr = map_tensor_data(ptr, x); + ggml_tensor *ref = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 16, 2); + ptr = map_tensor_data(ptr, ref); + ASSERT_EQ(ptr, mapped_file.data + mapped_file.size); // fp32 { ictx.dtype = GGML_TYPE_F32; - Linear model(&ictx, 3, 8); - memcpy(model.weight->data, w_data, sizeof(w_data)); - memcpy(model.bias->data, b_data, sizeof(b_data)); + Linear model(&ictx, 32, 16); + model.weight->data = w->data; + model.bias->data = b->data; - ggml_tensor *y = model.forward(&ctx, x); + ggml_tensor *out = model.forward(&ctx, x); - ggml_build_forward_expand(&ctx.gf, y); + ggml_build_forward_expand(&ctx.gf, out); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref, out); } // fp16 { + reset_cgraph(); + ictx.dtype = GGML_TYPE_F16; - Linear model(&ictx, 3, 8); - ggml_fp32_to_fp16_row(w_data, (ggml_fp16_t *)model.weight->data, ggml_nelements(model.weight)); - memcpy(model.bias->data, b_data, sizeof(b_data)); + Linear model(&ictx, 32, 16); + ggml_fp32_to_fp16_row((float *)w->data, (ggml_fp16_t *)model.weight->data, ggml_nelements(model.weight)); + model.bias->data = b->data; - ggml_tensor *y = model.forward(&ctx, x); + ggml_tensor *out = model.forward(&ctx, x); - ggml_build_forward_expand(&ctx.gf, y); + ggml_build_forward_expand(&ctx.gf, out); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - EXPECT_EQ(y->type, GGML_TYPE_F32); - expect_all_close((float *)y->data, y_data, ggml_nelements(y), 5e-3); + EXPECT_EQ(out->type, GGML_TYPE_F32); + expect_all_close(ref, out, 5e-3); } } TEST_F(ChatGLMTest, LayerNorm) { - float w_data[]{1.5410, -0.2934, -2.1788, 0.5684, -1.0845, -1.3986, 0.4033, 0.8380, -0.7193}; - float b_data[]{-0.4033, -0.5966, 0.1820, -0.8567, 1.1006, -1.0712, 0.1227, -0.5663, 0.3731}; - float x_data[]{0.4397, 0.1124, 0.5433, -0.3952, 0.2055, -0.4503, -0.5731, -0.5554, 0.5943, - 1.5419, 1.8197, -0.5515, -1.3253, 0.1886, -0.0691, -0.4949, -1.4959, -0.1938}; - float y_data[]{1.1039, -0.6742, -2.4414, -1.3358, 0.5938, 0.2759, -0.3738, -1.5655, -0.5730, - 1.9137, -1.1141, 1.1753, -1.5275, 0.8437, -1.0652, -0.0398, -1.6891, 0.4602}; - - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 9, 2); - memcpy(x->data, x_data, sizeof(x_data)); - LayerNorm model(&ictx, 9); - memcpy(model.weight->data, w_data, sizeof(w_data)); - memcpy(model.bias->data, b_data, sizeof(b_data)); - ggml_tensor *y = model.forward(&ctx, x); + fs::path test_path = fs::path(__FILE__).parent_path() / "tests/data/layer_norm.data"; + MappedFile mapped_file(test_path.string()); + char *ptr = mapped_file.data; - ggml_build_forward_expand(&ctx.gf, y); + LayerNorm model(&ictx, 64); + ptr = map_tensor_data(ptr, model.weight); + ptr = map_tensor_data(ptr, model.bias); + + ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 64, 3); + ptr = map_tensor_data(ptr, x); + x = ggml_dup(ctx.gctx.get(), x); + + ggml_tensor *ref = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 64, 3); + ptr = map_tensor_data(ptr, ref); + + ASSERT_EQ(ptr, mapped_file.data + mapped_file.size); + + ggml_tensor *out = model.forward(&ctx, x); + ggml_build_forward_expand(&ctx.gf, out); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref, out); } TEST_F(ChatGLMTest, RMSNorm) { - float w_data[]{0.4963, 0.7682, 0.0885, 0.1320, 0.3074, 0.6341, 0.4901}; - float x_data[]{0.1642, 0.3058, 0.2100, 0.9056, 0.6035, 0.8110, -0.0451, - 0.8797, 1.0482, -0.0445, -0.7229, 2.8663, -0.5655, 0.1604}; - float y_data[]{0.1521, 0.4385, 0.0347, 0.2232, 0.3464, 0.9599, -0.0412, - 0.3489, 0.6436, -0.0031, -0.0763, 0.7043, -0.2866, 0.0628}; + fs::path test_path = fs::path(__FILE__).parent_path() / "tests/data/rms_norm.data"; + MappedFile mapped_file(test_path.string()); + char *ptr = mapped_file.data; - RMSNorm model(&ictx, 7); - memcpy(model.weight->data, w_data, sizeof(w_data)); + RMSNorm model(&ictx, 64); + ptr = map_tensor_data(ptr, model.weight); - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 7, 2); - memcpy(x->data, x_data, sizeof(x_data)); - ggml_tensor *y = model.forward(&ctx, x); + ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 64, 3); + ptr = map_tensor_data(ptr, x); + x = ggml_dup(ctx.gctx.get(), x); + + ggml_tensor *ref = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 64, 3); + ptr = map_tensor_data(ptr, ref); - ggml_build_forward_expand(&ctx.gf, y); + ASSERT_EQ(ptr, mapped_file.data + mapped_file.size); + + ggml_tensor *out = model.forward(&ctx, x); + + ggml_build_forward_expand(&ctx.gf, out); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref, out); } -TEST_F(ChatGLMTest, GLMSelfAttention) { - float query_key_value_weight_data[]{ - -1.8717e-03, 1.3411e-01, -2.0576e-01, -1.8398e-01, -9.6289e-02, 6.7039e-02, -4.9533e-03, 1.9822e-01, - -2.2186e-02, 6.6153e-02, -7.5553e-02, -4.9141e-02, -2.3884e-01, -1.6557e-01, -1.0306e-01, 9.2609e-03, - 9.8834e-02, 1.5001e-01, -1.6949e-01, -1.0887e-01, 9.0804e-02, 2.0760e-01, -5.1450e-02, 1.8708e-01, - -4.0296e-02, 2.6454e-02, 2.2637e-01, -2.3192e-01, -1.5738e-01, -6.3291e-02, -9.7450e-02, 2.1600e-01, - -1.6204e-01, -1.1508e-01, -1.7466e-01, -2.3414e-01, -1.4594e-01, 2.1490e-01, 1.1155e-01, 1.2117e-01, - 1.3148e-02, -1.2817e-01, 4.2296e-02, -2.3342e-01, -1.8064e-01, -1.2888e-01, 1.5773e-01, 1.4658e-01, - -1.1087e-01, -9.0206e-03, 1.5989e-01, 2.4853e-01, 9.9221e-02, 3.3773e-02, 1.6762e-01, -1.4720e-01, - 4.6586e-02, -1.9383e-01, -1.7327e-01, -1.2915e-01, 1.1312e-01, 1.0054e-01, -1.4809e-01, 7.5527e-02, - 1.3724e-01, -3.1554e-02, 9.5454e-03, 5.7926e-02, 1.5509e-01, 2.4005e-01, -1.9266e-01, -9.1617e-02, - 9.8252e-02, 2.0714e-01, 2.1755e-01, 2.2059e-01, 4.9754e-02, -2.1740e-01, 2.2998e-02, -1.5640e-01, - -2.3299e-01, 2.2212e-01, 1.9009e-01, -2.4938e-01, 4.6793e-02, -4.2115e-02, -4.1140e-02, -1.1444e-01, - 9.6139e-02, -1.4808e-01, 9.1648e-02, 1.2643e-01, 1.7897e-01, 9.3478e-02, -2.4743e-01, -1.6217e-01, - 1.2483e-01, 5.2325e-02, -1.9502e-01, -1.4395e-01, 2.3519e-01, 1.6845e-01, -1.0901e-01, -6.2921e-02, - -2.3815e-01, -4.4935e-03, -1.8826e-01, -1.9284e-01, -1.3775e-02, 3.7536e-02, -1.0238e-01, 1.4834e-01, - -1.5213e-01, 2.2684e-01, 1.7132e-01, -2.1082e-01, -6.2221e-02, 1.1281e-02, 3.6475e-02, 5.9294e-02, - 9.8107e-02, 1.4975e-02, -1.2198e-01, 1.1830e-01, -2.3981e-01, -1.4818e-01, -6.2582e-02, -1.2178e-01, - -8.7458e-02, -2.0491e-01, -5.3179e-02, 5.3439e-02, -1.6287e-01, -1.2830e-02, 1.7896e-01, -2.5700e-02, - 6.9481e-03, -2.1567e-02, 5.0595e-02, 1.5896e-01, 2.3681e-01, 1.5876e-01, 2.3735e-01, -1.8080e-02, - -2.2458e-01, -1.1852e-01, 1.7023e-01, -1.6206e-03, -1.2426e-01, -1.9158e-01, -2.3396e-01, -2.1100e-01, - -5.0709e-02, 1.3710e-01, 1.3516e-01, -2.4111e-01, 1.5595e-01, -1.9563e-01, -5.2853e-02, -1.0137e-01, - -4.8154e-02, -4.9086e-02, -2.2434e-01, -2.1586e-01, -3.9120e-02, 3.2330e-03, -1.1357e-01, 9.4175e-02, - -2.2501e-01, -1.6872e-02, 2.1985e-01, -1.0197e-01, 2.2575e-01, 9.0538e-02, -2.2562e-01, 1.5817e-01, - -2.8849e-02, -1.1160e-01, 1.9991e-01, -2.0202e-01, 2.6826e-02, -5.2342e-02, 1.7853e-01, 6.9786e-02, - 1.2013e-01, 8.8290e-02, -6.0119e-02, -5.2576e-02, -2.0602e-01, 1.3546e-01, 1.9849e-01, 1.7106e-01, - -1.7634e-01, 1.1150e-02, -1.7623e-01, -1.3762e-01, -1.4568e-01, 8.5436e-02, -1.4898e-01, -5.4543e-03, - 1.0517e-02, 1.6116e-01, -1.8898e-01, -1.7163e-01, -1.4517e-01, 1.7498e-01, -8.9866e-02, 2.1087e-01, - 9.0402e-02, 3.1657e-02, -1.8610e-03, -4.9420e-02, 3.1367e-02, -5.7086e-02, -1.7566e-03, 3.1898e-02, - -1.9555e-01, -1.3103e-01, 2.0187e-01, -2.0289e-01, -1.7952e-02, 2.4731e-01, 9.0309e-02, 7.0783e-03, - -2.1665e-01, 1.2384e-01, -1.7807e-01, -7.0966e-02, -8.3879e-02, -3.7022e-02, 2.7346e-03, 2.0620e-01, - 3.1210e-02, 2.2392e-01, 1.5293e-01, -1.5805e-01, 1.1213e-01, -1.7672e-01, -1.0596e-01, 7.3531e-02, - 8.2548e-02, 1.8756e-01, -8.0479e-02, 4.0022e-04, 1.2871e-01, -2.4177e-01, 1.8075e-01, -2.0673e-01, - 3.4456e-03, -4.2504e-02, -1.3167e-01, 3.3043e-02, 2.0673e-01, -7.3080e-02, -1.4842e-01, -9.2460e-02, - -2.4779e-01, 1.1285e-01, -1.2007e-01, -1.6684e-01, -1.4403e-01, 1.4374e-01, 1.3239e-01, 1.9188e-01, - 9.0681e-02, -8.3490e-02, -6.9862e-02, 7.3857e-02, 2.0551e-01, 6.7973e-02, -1.1829e-01, -1.1752e-01, - -2.3637e-01, 5.4012e-02, -1.4030e-01, -2.2289e-01, 2.1922e-01, -1.6235e-01, -2.8440e-02, 7.1623e-02, - 7.9646e-03, -1.6822e-01, -2.0208e-01, 1.9927e-01, 4.0710e-02, 2.0741e-01, -8.3810e-02, 7.3639e-02, - -5.7169e-02, -1.1118e-02, -1.5226e-01, 8.4550e-02, 7.9041e-02, -5.1571e-03, -5.6226e-02, -1.5411e-01, - 1.7289e-01, -1.8611e-01, 1.0242e-01, -8.4063e-02, -1.2062e-01, 4.4911e-02, -1.2986e-01, 5.7601e-02, - 4.9095e-02, -1.8562e-01, 4.1625e-02, 1.0648e-01, 9.8958e-02, -3.1470e-02, -2.0495e-01, -3.8539e-02, - 8.6827e-02, -9.1219e-02, 9.4895e-02, 1.6649e-01, -1.3054e-01, 2.4655e-03, 1.0337e-01, 1.9594e-02, - 2.0883e-02, 3.1216e-02, -1.9654e-01, 1.9647e-02, 1.7312e-01, 2.2528e-01, 1.4694e-01, 3.3501e-02, - 1.1675e-01, -1.2162e-01, -2.0717e-01, -2.1498e-01, 2.4940e-01, 1.5870e-01, -1.7281e-01, 9.7811e-02, - 1.8879e-01, 2.4990e-01, 2.1860e-01, 1.9369e-01, -5.7315e-02, -8.7735e-02, 2.0526e-01, 1.4009e-01, - -1.5044e-01, 2.2475e-01, 1.2079e-01, 1.3628e-01, -1.5669e-01, 7.1725e-02, -8.7641e-02, 1.9534e-01, - -4.4985e-02, 9.7328e-02, 4.4405e-02, 1.0637e-01, -8.4957e-02, 1.2189e-01, -1.7462e-01, 5.6464e-02, - -1.6915e-01, -2.4663e-01, -2.0076e-01, 1.9737e-01, 1.3526e-01, 2.3455e-01, 2.0028e-01, -2.2326e-01, - -1.7061e-01, -4.0396e-02, -1.6236e-01, 1.7360e-01, -1.8900e-01, -1.2198e-01, -2.4152e-01, -1.4194e-01, - 2.0562e-01, 2.0469e-01, 1.7896e-01, 1.9303e-01, 2.2230e-01, -6.4016e-02, 1.1000e-01, 2.2273e-01, - 8.2705e-02, 2.4992e-01, 1.2967e-01, 1.5541e-01, -8.7498e-02, 1.1996e-01, 2.8727e-02, -5.9704e-02, - -1.4093e-01, -1.4028e-01, -1.9237e-01, 1.6783e-01, 1.7773e-01, -2.8453e-02, -1.4467e-01, 1.9323e-01, - 1.5987e-01, 1.8584e-02, -1.1803e-01, 2.2976e-01, 1.0224e-01, -1.8979e-01, 2.3927e-01, 1.8984e-01, - -9.1123e-02, 1.4054e-01, -1.4205e-01, -3.9179e-02, 2.1228e-01, 1.0330e-02, -1.7680e-01, -8.3558e-02, - -6.7861e-02, -4.8232e-02, 2.3925e-02, 2.3121e-01, 1.3385e-02, -1.5436e-01, 1.2814e-02, 1.1987e-01, - 1.2401e-01, -2.2848e-01, -4.4736e-02, -1.8579e-01, -1.0667e-01, 9.0073e-02, -1.7753e-01, 9.2932e-02, - 2.1219e-01, 1.6397e-02, -1.6662e-01, -8.9571e-02, 5.4591e-02, -1.9058e-01, 1.2420e-01, -2.2697e-01, - -2.4032e-01, -2.4292e-01, -5.0716e-02, 1.6811e-01, -2.3662e-01, 2.0780e-01, -1.0001e-01, 7.3221e-02, - 1.1401e-02, -2.2543e-01, 2.0733e-01, 1.3461e-01, 2.4850e-01, 1.2630e-01, -1.6502e-01, 2.0865e-01, - 1.3436e-02, 1.1855e-01, -2.0046e-01, -7.1907e-02, -2.4547e-01, -9.7373e-02, 5.3933e-02, -1.9629e-01, - 7.9691e-02, 1.3420e-01, 3.4827e-02, -1.6727e-01, -1.9383e-01, -7.7129e-02, 1.0974e-01, 2.4660e-01, - 1.4376e-01, -2.8152e-02, 8.7654e-02, -2.4527e-01, -2.1353e-01, 1.1665e-01, -1.4160e-01, 1.2027e-01, - -1.7648e-01, -1.2383e-01, -2.0592e-01, 1.3046e-01, -2.5474e-02, 1.9240e-01, 1.5472e-01, 1.3834e-01, - 8.0390e-03, -7.7294e-02, -5.4358e-02, 3.3225e-02, 1.2393e-01, -1.7515e-01, 2.0982e-01, -2.7183e-02, - -2.0949e-01, -1.3526e-01, 2.2120e-01, 2.2863e-01, -2.3157e-01, 1.7632e-01, 1.2529e-01, 1.4798e-01, - 2.1163e-01, -1.3474e-01, 7.8944e-02, 1.0231e-01, -7.3873e-02, 8.3663e-02, -7.1928e-02, 1.5457e-01, - -6.9364e-02, -9.3199e-02, 6.2937e-02, 8.8673e-02, -1.2214e-01, 2.2100e-02, 1.4488e-01, -2.4874e-02, - 7.6083e-02, -6.0297e-02, 8.7625e-02, -1.8110e-01, -1.4701e-01, -1.2690e-01, 2.2975e-01, -6.7271e-02, - -6.8259e-04, -1.2112e-01, 2.4957e-01, 2.4417e-01, -1.8855e-01, -2.0267e-01, -1.8950e-01, -1.2057e-03, - -6.3726e-02, -1.6364e-01, -8.9668e-02, 4.7233e-02, -1.3062e-01, 5.5396e-02, -5.7329e-02, -1.2114e-01, - 3.4346e-02, 2.0556e-01, -1.6902e-01, 1.1609e-02, -9.2192e-02, 2.4533e-01, -2.3719e-01, -2.3967e-01, - 2.4635e-01, -1.5817e-01, 4.7931e-02, -2.1577e-02, -5.2664e-02, -5.5837e-02, 1.5886e-01, 1.1949e-02, - -2.4340e-01, -1.4759e-01, -8.5225e-02, 1.2580e-01, -1.6178e-01, 2.3573e-01, -5.5682e-02, -4.4890e-02, - 1.9589e-01, 1.2565e-01, 2.1203e-01, 1.4461e-01, -7.5838e-02, -1.6587e-01, -1.8601e-02, 2.0691e-01, - -8.3905e-02, -2.3184e-01, 1.0248e-01, 2.4337e-01, -7.1171e-02, -2.0701e-01, -2.2676e-01, 6.2650e-02, - -1.8930e-02, -1.2625e-01, 5.0533e-02, 9.4940e-02, 1.9883e-01, 1.9410e-01, -3.7421e-02, -2.2044e-01, - -2.2591e-01, 2.3342e-01, 1.1051e-01, 1.0898e-01, -2.1631e-01, 2.3150e-01, 2.3684e-01, 2.2572e-01, - -2.1090e-01, -9.4331e-02, -1.7194e-01, 2.3674e-01, -1.0742e-01, -1.1414e-01, 1.3098e-01, -1.1565e-01, - -1.2313e-01, -2.1868e-02, -2.4030e-02, -1.9474e-01, 2.0842e-01, -1.1028e-01, 8.8679e-02, 2.1745e-01, - 1.2608e-01, 3.5395e-02, 2.1271e-01, 3.3607e-02, -1.1566e-01, 2.3650e-01, 5.9170e-02, -2.4392e-01, - -7.1173e-02, -1.7029e-01, 2.1919e-01, -4.1286e-02, -2.2788e-01, -1.5716e-02, 1.5700e-01, 6.4950e-02, - 7.9051e-02, 2.3218e-02, 9.3205e-02, -6.0913e-02, -9.9450e-02, -2.3368e-01, -1.8833e-01, 1.0835e-01, - -1.4803e-01, 3.5867e-02, 7.9798e-02, 1.7701e-02, -1.6209e-01, 2.3908e-01, -1.4537e-01, 2.0563e-01, - -1.9888e-01, -6.0137e-02, 1.3599e-01, -1.0215e-01, 2.1001e-01, -1.7204e-01, -2.0994e-01, -1.1273e-01, - 4.0413e-02, 2.3019e-01, -1.1935e-01, 8.9407e-02, -6.2677e-02, -5.4216e-02, 1.8383e-01, -1.9375e-01, - 2.6553e-02, 2.3510e-01, -3.4353e-02, 1.9411e-01, -7.6989e-02, 2.0124e-01, -2.4183e-01, -3.6031e-02, - -4.3912e-02, 8.1021e-02, 9.8082e-02, 1.9195e-01, -3.7246e-02, -9.9138e-03, 1.7120e-01, -6.7643e-02, - 2.1916e-01, -1.6645e-01, -2.7052e-02, -1.3428e-02, 1.1155e-01, 1.7092e-01, -3.9626e-02, -2.0712e-01, - 1.2387e-01, 7.4783e-02, 1.0042e-01, -1.5422e-01, 1.6090e-01, 2.3679e-01, 2.1693e-02, -2.3351e-01, - 1.7550e-01, -1.8536e-01, 5.7468e-02, 3.6329e-02, -1.1701e-01, 8.7049e-02, -2.2361e-01, 5.6940e-02, - -1.5849e-01, -2.7033e-02, 3.2157e-02, 2.1299e-01, -1.1928e-01, 1.6016e-01, -3.1758e-02, -1.1873e-01, - -2.1770e-01, -2.2938e-01, 2.4415e-01, -6.2350e-02, 1.2487e-02, 6.7778e-02, 1.6993e-01, 2.1337e-01, - 2.0275e-01, -1.8522e-01, -4.0054e-02, -1.4793e-01, -1.4284e-01, 5.9302e-02, 2.3466e-01, -2.0028e-01, - 1.5130e-01, -1.2962e-01, -4.8694e-02, 1.9844e-01, -5.6543e-02, 2.2764e-02, -1.7476e-01, 2.1281e-01, - -3.2317e-02, -1.8285e-01, 7.3203e-02, -1.7775e-01, -1.9838e-01, 1.5230e-02, 1.9821e-01, -7.0746e-02, - 1.1767e-01, 2.1483e-01, 1.6582e-01, -1.3114e-01, -2.7405e-02, -7.8671e-02, -2.0103e-01, 1.0329e-04, - 1.8811e-01, 2.1063e-01, 2.3325e-02, 5.6781e-02, -1.0823e-01, 1.8871e-01, -1.0403e-01, -1.7366e-01, - 3.8498e-02, 1.4985e-01, -2.2540e-01, 2.2599e-01, 8.9933e-02, -1.7515e-01, -5.3852e-02, 2.1689e-01}; - float query_key_value_bias_data[]{ - -1.9181e-01, -7.3069e-02, 8.1995e-02, -2.1902e-01, 1.3705e-01, 1.3014e-01, 1.5505e-01, -1.5939e-01, - 2.4900e-01, -1.4819e-01, 2.4959e-01, -2.3992e-01, -2.2274e-01, 1.5355e-01, 2.6128e-02, 1.4422e-02, - -1.3844e-01, -1.0487e-01, -7.3097e-02, -2.4354e-01, 1.2992e-02, 4.4215e-02, -2.1166e-04, 8.0733e-02, - 2.3721e-01, 6.6471e-02, -9.1518e-02, -1.0289e-01, -1.5995e-01, -1.7330e-01, -4.0262e-02, -4.4216e-02, - 1.1122e-01, -1.0686e-01, 1.9930e-01, -1.7542e-01, 7.1055e-04, 2.2473e-01, 2.4860e-01, -1.4482e-01, - 4.4532e-02, 2.9531e-02, -1.1721e-01, -8.6374e-02, 6.7713e-02, -1.7384e-01, 4.1249e-02, 1.0818e-01}; - float dense_weight_data[]{ - -0.0985, 0.2077, -0.0165, 0.1134, 0.2476, -0.0764, 0.1359, -0.0715, -0.0365, -0.0424, -0.0016, -0.0944, - 0.0586, 0.0094, 0.1585, -0.0506, 0.0251, -0.0930, -0.2094, 0.1012, 0.0320, -0.1001, -0.0845, 0.0654, - -0.0452, 0.1634, 0.0142, 0.0944, 0.1089, -0.0613, 0.1082, 0.1845, 0.0115, 0.0489, 0.0091, 0.1731, - -0.1055, -0.1329, 0.1089, -0.2176, 0.0040, -0.1150, 0.1650, -0.2328, 0.1516, 0.2473, 0.0506, -0.0167, - 0.2461, -0.1068, -0.0200, -0.1095, -0.0345, 0.0543, 0.0334, -0.0472, -0.2500, 0.0128, -0.0074, 0.0376, - 0.1916, 0.2430, -0.1483, -0.0156, -0.1002, -0.2304, -0.1819, 0.2346, 0.1458, 0.1343, 0.1389, -0.1986, - 0.0263, 0.2327, -0.1395, 0.2224, 0.0657, 0.1759, -0.1071, 0.1153, -0.2216, -0.0159, 0.0834, 0.0750, - 0.2092, 0.2457, 0.2274, 0.1678, 0.1758, -0.0323, -0.1940, -0.0928, -0.0281, -0.1351, 0.1278, 0.0878, - 0.0403, 0.0604, 0.2196, 0.0911, -0.2192, -0.1815, 0.1102, 0.0341, 0.1219, -0.2497, -0.2307, 0.1533, - 0.1599, -0.2262, 0.0949, -0.1952, 0.1893, 0.0784, 0.2472, -0.2465, -0.2407, 0.1690, 0.2220, 0.1532, - 0.1662, -0.2106, 0.1810, -0.2360, 0.0559, -0.1638, -0.0993, 0.0443, -0.0983, -0.0396, -0.2370, -0.2357, - 0.0968, -0.1058, -0.1521, -0.0986, -0.2044, -0.1999, 0.1496, 0.1271, 0.0333, -0.1366, 0.0832, 0.0112, - -0.1787, 0.0538, 0.2276, -0.2459, -0.2486, 0.0320, 0.1883, -0.1521, 0.0550, 0.1757, 0.0771, 0.0541, - 0.2494, 0.1629, -0.1406, 0.0216, 0.1277, 0.2205, -0.2487, 0.1354, -0.0726, -0.2448, 0.0555, -0.1361, - 0.0354, -0.1623, -0.1881, -0.0212, -0.0840, 0.1462, 0.0216, 0.1951, 0.0469, -0.0804, 0.1693, 0.0137, - 0.0469, -0.0804, 0.0009, -0.0481, -0.0213, -0.1950, -0.0015, -0.2075, -0.0538, 0.1449, -0.1738, -0.1685, - -0.0610, -0.0685, 0.0423, 0.0415, 0.1268, -0.1722, -0.0176, 0.1398, 0.2162, -0.0182, -0.1447, 0.0719, - 0.1424, -0.1562, -0.1451, 0.1105, -0.0175, -0.2361, -0.1441, 0.1014, -0.0848, 0.1726, 0.1976, 0.0364, - -0.0198, -0.0794, -0.0126, 0.0455, -0.1910, -0.0597, -0.2080, 0.1534, -0.1592, 0.2284, -0.0644, -0.1432, - 0.1201, 0.0373, 0.1731, 0.1044, -0.2408, 0.1581, -0.0471, -0.1105, 0.1588, 0.1823, -0.2197, -0.0226, - 0.2053, 0.0968, 0.2106, -0.0857, -0.1379, 0.2150, 0.1042, 0.2400, -0.1044, -0.1605, -0.0293, -0.2354, - 0.0980, 0.1844, 0.0600, -0.0247}; - float dense_bias_data[]{0.1240, -0.1587, 0.2446, -0.2486, -0.2395, -0.0591, 0.2042, 0.0250, - 0.0960, -0.1833, 0.0912, -0.0279, 0.1002, 0.1766, 0.1087, -0.0213}; - - GLMSelfAttention model(&ictx, 16, 2, 16); - memcpy(model.query_key_value.weight->data, query_key_value_weight_data, sizeof(query_key_value_weight_data)); - memcpy(model.query_key_value.bias->data, query_key_value_bias_data, sizeof(query_key_value_bias_data)); - memcpy(model.dense.weight->data, dense_weight_data, sizeof(dense_weight_data)); - memcpy(model.dense.bias->data, dense_bias_data, sizeof(dense_bias_data)); +TEST_F(ChatGLMTest, GLMBlock) { + fs::path test_path = fs::path(__FILE__).parent_path() / "tests/data/glm_block.data"; + MappedFile mapped_file(test_path.string()); + char *ptr = mapped_file.data; + + constexpr int hidden_size = 32; + constexpr int num_attention_heads = 8; + constexpr int num_hidden_layers = 28; + constexpr int max_length = 16; + constexpr int seq_len = 4; + GLMBlock model(&ictx, hidden_size, num_attention_heads, num_hidden_layers, max_length); + + ptr = map_tensor_data(ptr, model.input_layernorm.weight); + ptr = map_tensor_data(ptr, model.input_layernorm.bias); + ptr = map_tensor_data(ptr, model.attention.query_key_value.weight); + ptr = map_tensor_data(ptr, model.attention.query_key_value.bias); + ptr = map_tensor_data(ptr, model.attention.dense.weight); + ptr = map_tensor_data(ptr, model.attention.dense.bias); + ptr = map_tensor_data(ptr, model.post_attention_layernorm.weight); + ptr = map_tensor_data(ptr, model.post_attention_layernorm.bias); + ptr = map_tensor_data(ptr, model.mlp.dense_h_to_4h.weight); + ptr = map_tensor_data(ptr, model.mlp.dense_h_to_4h.bias); + ptr = map_tensor_data(ptr, model.mlp.dense_4h_to_h.weight); + ptr = map_tensor_data(ptr, model.mlp.dense_4h_to_h.bias); + + ggml_tensor *x1 = ggml_new_tensor_2d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size, seq_len); + ptr = map_tensor_data(ptr, x1); + x1 = ggml_dup(ictx.gctx.get(), x1); + + ggml_tensor *ref_y1 = ggml_new_tensor_2d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size, seq_len); + ptr = map_tensor_data(ptr, ref_y1); + + ggml_tensor *x2 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, x2); + x2 = ggml_dup(ictx.gctx.get(), x2); + + ggml_tensor *ref_y2 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, ref_y2); + + ggml_tensor *x3 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, x3); + x3 = ggml_dup(ictx.gctx.get(), x3); + + ggml_tensor *ref_y3 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, ref_y3); + + ASSERT_EQ(ptr, mapped_file.data + mapped_file.size); // self attention { - float x_data[]{-1.1230, 0.6210, -0.8764, -1.7740, -0.8104, -1.4760, 0.6670, 0.6688, -0.0752, 0.1643, - 0.0275, 0.5813, 0.2350, 1.0717, -0.5229, -0.7213, -0.2058, -0.8088, 0.4647, 0.3915, - -1.3406, 0.7596, 0.7288, -0.0667, 0.5559, -1.4076, 0.1106, 0.0352, -0.5646, 0.9159, - 0.2990, 0.1771, -0.3439, 0.2291, 0.4366, 0.2702, -0.2002, 1.0184, -0.4169, -0.0308, - -0.9846, 0.2024, 0.4156, -1.2037, -0.1333, -0.2934, 0.4649, 0.5762, -0.0578, 1.1715, - -1.5185, 0.0475, -0.0103, 1.4353, 0.2518, 0.7408, -1.0982, 0.1548, -1.3989, 0.0408, - -1.1174, 0.7029, -0.6675, 0.1254}; - float y_data[]{ - 1.6484e-01, -1.9052e-01, 2.8198e-01, -1.7810e-01, -4.1363e-01, -3.2469e-02, 3.9480e-01, 3.2204e-01, - 1.3954e-01, -4.9466e-02, 5.7194e-02, -5.4678e-02, 1.5006e-01, -2.2709e-04, 1.1909e-01, 1.2991e-01, - 1.4828e-01, -1.7710e-01, 3.0013e-01, -1.6764e-01, -3.2587e-01, -2.7857e-02, 3.3097e-01, 2.3543e-01, - 1.5086e-01, -3.0025e-02, -1.9906e-02, -3.6032e-02, 1.2470e-01, 4.0979e-02, 1.1201e-01, 8.7082e-02, - 1.2724e-01, -1.7805e-01, 2.5121e-01, -1.9400e-01, -3.5495e-01, -3.3908e-02, 2.7882e-01, 2.1870e-01, - 1.6772e-01, -6.0175e-02, -3.4353e-02, -5.1714e-02, 9.2530e-02, 4.2405e-02, 1.1574e-01, 7.0774e-02, - 1.5804e-01, -1.7552e-01, 2.2854e-01, -1.2723e-01, -2.5867e-01, -3.2931e-02, 2.1397e-01, 2.2805e-01, - 1.5255e-01, 1.1970e-02, 4.6676e-02, -9.0893e-02, 1.3114e-01, 5.0670e-02, 2.1247e-01, 4.2616e-02}; - - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 16, 4); - memcpy(x->data, x_data, sizeof(x_data)); - ggml_tensor *y = model.forward(&ctx, x, 0, 4); - - ggml_build_forward_expand(&ctx.gf, y); + ggml_tensor *out_y1 = model.forward(&ctx, x1, 0, seq_len); + + ggml_build_forward_expand(&ctx.gf, out_y1); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref_y1, out_y1, 5e-4); } // cross attention { - ctx.gf = {}; - ctx.gf.n_threads = 1; - - float x_data[]{0.4047, -0.6549, 0.0521, 0.3401, -0.2124, 1.5629, -0.9072, -1.5662, - 0.0485, 0.9935, 2.1180, -0.1827, -0.7768, 1.7764, -0.5033, 0.0566}; - float y_data[]{0.1011, -0.1058, 0.1434, -0.1980, -0.1304, 0.0632, 0.2109, 0.0619, - 0.0893, 0.0146, -0.1291, -0.1430, 0.1655, 0.0916, 0.1488, -0.0282}; - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 16, 1); - memcpy(x->data, x_data, sizeof(x_data)); + ggml_tensor *out_y2 = model.forward(&ctx, x2, seq_len, seq_len); - ggml_tensor *y = model.forward(&ctx, x, 4, 4); - - ggml_build_forward_expand(&ctx.gf, y); + ggml_build_forward_expand(&ctx.gf, out_y2); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref_y2, out_y2, 5e-4); } { - ctx.gf = {}; - ctx.gf.n_threads = 1; - - float x_data[]{-0.5481, -0.6917, -0.6559, 0.6949, -0.4671, -0.1680, 0.7585, 1.1881, - -0.6305, -0.0654, 0.6188, 2.0020, 0.2952, 0.5314, -0.5227, 0.0995}; - float y_data[]{0.1479, -0.1224, 0.2174, -0.1259, -0.1662, 0.0400, 0.2235, 0.1599, - 0.0633, 0.0484, -0.0541, -0.1475, 0.1395, 0.0448, 0.2216, 0.0838}; - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 16, 1); - memcpy(x->data, x_data, sizeof(x_data)); + ggml_tensor *out_y3 = model.forward(&ctx, x3, seq_len + 1, seq_len); - ggml_tensor *y = model.forward(&ctx, x, 5, 4); - - ggml_build_forward_expand(&ctx.gf, y); + ggml_build_forward_expand(&ctx.gf, out_y3); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref_y3, out_y3, 5e-4); } } -TEST_F(ChatGLMTest, GLMBlock) { - float x_data[]{-1.0159, -1.5296, -1.2180, 0.8683, -0.3850, 0.0964, -0.3182, 1.8511, -0.1479, -1.0800, -0.1396, - -1.2574, 1.5435, 0.9686, -1.5552, 0.7989, 1.1896, -0.1795, -1.8232, -0.0514, -1.6361, -2.5106, - -1.2383, -0.9296, -0.0585, 0.7729, 0.4689, 0.2599, 0.7576, -2.0418, -0.5524, 0.6270}; - float query_key_value_weight_data[]{ - -0.0026, 0.1897, -0.2910, -0.2602, -0.1362, 0.0948, -0.0070, 0.2803, -0.0314, 0.0936, -0.1068, -0.0695, - -0.3378, -0.2342, -0.1457, 0.0131, 0.1398, 0.2121, -0.2397, -0.1540, 0.1284, 0.2936, -0.0728, 0.2646, - -0.0570, 0.0374, 0.3201, -0.3280, -0.2226, -0.0895, -0.1378, 0.3055, -0.2292, -0.1628, -0.2470, -0.3311, - -0.2064, 0.3039, 0.1578, 0.1714, 0.0186, -0.1813, 0.0598, -0.3301, -0.2555, -0.1823, 0.2231, 0.2073, - -0.1568, -0.0128, 0.2261, 0.3515, 0.1403, 0.0478, 0.2371, -0.2082, 0.0659, -0.2741, -0.2450, -0.1826, - 0.1600, 0.1422, -0.2094, 0.1068, 0.1941, -0.0446, 0.0135, 0.0819, 0.2193, 0.3395, -0.2725, -0.1296, - 0.1389, 0.2929, 0.3077, 0.3120, 0.0704, -0.3074, 0.0325, -0.2212, -0.3295, 0.3141, 0.2688, -0.3527, - 0.0662, -0.0596, -0.0582, -0.1618, 0.1360, -0.2094, 0.1296, 0.1788, 0.2531, 0.1322, -0.3499, -0.2293, - 0.1765, 0.0740, -0.2758, -0.2036, 0.3326, 0.2382, -0.1542, -0.0890, -0.3368, -0.0064, -0.2662, -0.2727, - -0.0195, 0.0531, -0.1448, 0.2098, -0.2152, 0.3208, 0.2423, -0.2981, -0.0880, 0.0160, 0.0516, 0.0839, - 0.1387, 0.0212, -0.1725, 0.1673, -0.3391, -0.2096, -0.0885, -0.1722, -0.1237, -0.2898, -0.0752, 0.0756, - -0.2303, -0.0181, 0.2531, -0.0363, 0.0098, -0.0305, 0.0716, 0.2248, 0.3349, 0.2245, 0.3357, -0.0256, - -0.3176, -0.1676, 0.2407, -0.0023, -0.1757, -0.2709, -0.3309, -0.2984, -0.0717, 0.1939, 0.1911, -0.3410, - 0.2205, -0.2767, -0.0747, -0.1434, -0.0681, -0.0694, -0.3173, -0.3053, -0.0553, 0.0046, -0.1606, 0.1332, - -0.3182, -0.0239, 0.3109, -0.1442, 0.3193, 0.1280, -0.3191, 0.2237, -0.0408, -0.1578, 0.2827, -0.2857, - 0.0379, -0.0740, 0.2525, 0.0987, 0.1699, 0.1249, -0.0850, -0.0744, -0.2914, 0.1916, 0.2807, 0.2419}; - float query_key_value_bias_data[]{-0.2494, 0.0158, -0.2492, -0.1946, -0.2060, 0.1208, -0.2107, -0.0077, - 0.0149, 0.2279, -0.2673, -0.2427, -0.2053, 0.2475, -0.1271, 0.2982, - 0.1278, 0.0448, -0.0026, -0.0699, 0.0444, -0.0807, -0.0025, 0.0451}; - float dense_weight_data[]{-0.2766, -0.1853, 0.2855, -0.2869, -0.0254, 0.3497, 0.1277, 0.0100, -0.3064, 0.1751, - -0.2518, -0.1004, -0.1186, -0.0524, 0.0039, 0.2916, 0.0441, 0.3167, 0.2163, -0.2235, - 0.1586, -0.2499, -0.1498, 0.1040, 0.1167, 0.2652, -0.1138, 0.0006, 0.1820, -0.3419, - 0.2556, -0.2924, 0.0049, -0.0601, -0.1862, 0.0467, 0.2924, -0.1034, -0.2099, -0.1308, - -0.3504, 0.1596, -0.1698, -0.2359, -0.2037, 0.2033, 0.1872, 0.2714, 0.1282, -0.1181, - -0.0988, 0.1045, 0.2906, 0.0961, -0.1673, -0.1662, -0.3343, 0.0764, -0.1984, -0.3152, - 0.3100, -0.2296, -0.0402, 0.1013}; - float dense_bias_data[]{0.0113, -0.2379, -0.2858, 0.2818, 0.0576, 0.2933, -0.1185, 0.1041}; - - float dense_h_to_4h_weight_data[]{ - -0.0808, -0.0157, -0.2153, 0.1196, 0.1118, -0.0073, -0.0795, -0.2179, 0.2445, -0.2632, 0.1448, -0.1189, - -0.1706, 0.0635, -0.1837, 0.0815, 0.0694, -0.2625, 0.0589, 0.1506, 0.1399, -0.0445, -0.2898, -0.0545, - 0.1228, -0.1290, 0.1342, 0.2355, -0.1846, 0.0035, 0.1462, 0.0277, 0.0295, 0.0441, -0.2779, 0.0278, - 0.2448, 0.3186, 0.2078, 0.0474, 0.1651, -0.1720, -0.2930, -0.3040, 0.3527, 0.2244, -0.2444, 0.1383, - 0.2670, 0.3534, 0.3091, 0.2739, -0.0811, -0.1241, 0.2903, 0.1981, -0.2128, 0.3179, 0.1708, 0.1927, - -0.2216, 0.1014, -0.1239, 0.2763, -0.0636, 0.1376, 0.0628, 0.1504, -0.1201, 0.1724, -0.2469, 0.0799, - -0.2392, -0.3488, -0.2839, 0.2791, 0.1913, 0.3317, 0.2832, -0.3157, -0.2413, -0.0571, -0.2296, 0.2455, - -0.2673, -0.1725, -0.3416, -0.2007, 0.2908, 0.2895, 0.2531, 0.2730, 0.3144, -0.0905, 0.1556, 0.3150, - 0.1170, 0.3534, 0.1834, 0.2198, -0.1237, 0.1696, 0.0406, -0.0844, -0.1993, -0.1984, -0.2721, 0.2374, - 0.2514, -0.0402, -0.2046, 0.2733, 0.2261, 0.0263, -0.1669, 0.3249, 0.1446, -0.2684, 0.3384, 0.2685, - -0.1289, 0.1988, -0.2009, -0.0554, 0.3002, 0.0146, -0.2500, -0.1182, -0.0960, -0.0682, 0.0338, 0.3270, - 0.0189, -0.2183, 0.0181, 0.1695, 0.1754, -0.3231, -0.0633, -0.2627, -0.1509, 0.1274, -0.2511, 0.1314, - 0.3001, 0.0232, -0.2356, -0.1267, 0.0772, -0.2695, 0.1756, -0.3210, -0.3399, -0.3435, -0.0717, 0.2377, - -0.3346, 0.2939, -0.1414, 0.1036, 0.0161, -0.3188, 0.2932, 0.1904, 0.3514, 0.1786, -0.2334, 0.2951, - 0.0190, 0.1677, -0.2835, -0.1017, -0.3471, -0.1377, 0.0763, -0.2776, 0.1127, 0.1898, 0.0493, -0.2366, - -0.2741, -0.1091, 0.1552, 0.3487, 0.2033, -0.0398, 0.1240, -0.3469, -0.3020, 0.1650, -0.2003, 0.1701, - -0.2496, -0.1751, -0.2912, 0.1845, -0.0360, 0.2721, 0.2188, 0.1956, 0.0114, -0.1093, -0.0769, 0.0470, - 0.1753, -0.2477, 0.2967, -0.0384, -0.2963, -0.1913, 0.3128, 0.3233, -0.3275, 0.2494, 0.1772, 0.2093, - 0.2993, -0.1905, 0.1116, 0.1447, -0.1045, 0.1183, -0.1017, 0.2186, -0.0981, -0.1318, 0.0890, 0.1254, - -0.1727, 0.0313, 0.2049, -0.0352, 0.1076, -0.0853, 0.1239, -0.2561, -0.2079, -0.1795, 0.3249, -0.0951, - -0.0010, -0.1713, 0.3529, 0.3453, -0.2666, -0.2866, -0.2680, -0.0017, -0.0901, -0.2314, -0.1268, 0.0668, - -0.1847, 0.0783, -0.0811, -0.1713}; - float dense_h_to_4h_bias_data[]{0.0486, 0.2907, -0.2390, 0.0164, -0.1304, 0.3469, -0.3354, -0.3389, - 0.3484, -0.2237, 0.0678, -0.0305, -0.0745, -0.0790, 0.2247, 0.0169, - -0.3442, -0.2087, -0.1205, 0.1779, -0.2288, 0.3334, -0.0787, -0.0635, - 0.2770, 0.1777, 0.2999, 0.2045, -0.1073, -0.2346, -0.0263, 0.2926}; - float dense_4h_to_h_weight_data[]{ - -5.9330e-02, -1.6394e-01, 7.2466e-02, 1.7209e-01, -5.0325e-02, -1.4638e-01, -1.6035e-01, 4.4300e-02, - -1.3385e-02, -8.9270e-02, 3.5732e-02, 6.7133e-02, 1.4059e-01, 1.3725e-01, -2.6461e-02, -1.5588e-01, - -1.5974e-01, 1.6505e-01, 7.8145e-02, 7.7058e-02, -1.5295e-01, 1.6370e-01, 1.6747e-01, 1.5960e-01, - -1.4913e-01, -6.6702e-02, -1.2158e-01, 1.6740e-01, -7.5957e-02, -8.0708e-02, 9.2616e-02, -8.1776e-02, - -8.7066e-02, -1.5463e-02, -1.6992e-02, -1.3771e-01, 1.4737e-01, -7.7980e-02, 6.2705e-02, 1.5376e-01, - 8.9151e-02, 2.5028e-02, 1.5041e-01, 2.3763e-02, -8.1787e-02, 1.6723e-01, 4.1840e-02, -1.7248e-01, - -5.0327e-02, -1.2041e-01, 1.5499e-01, -2.9194e-02, -1.6114e-01, -1.1113e-02, 1.1102e-01, 4.5927e-02, - 5.5898e-02, 1.6418e-02, 6.5906e-02, -4.3072e-02, -7.0322e-02, -1.6523e-01, -1.3317e-01, 7.6615e-02, - -1.0467e-01, 2.5362e-02, 5.6426e-02, 1.2516e-02, -1.1461e-01, 1.6906e-01, -1.0280e-01, 1.4540e-01, - -1.4063e-01, -4.2523e-02, 9.6161e-02, -7.2228e-02, 1.4850e-01, -1.2165e-01, -1.4845e-01, -7.9712e-02, - 2.8576e-02, 1.6277e-01, -8.4393e-02, 6.3220e-02, -4.4319e-02, -3.8336e-02, 1.2998e-01, -1.3700e-01, - 1.8776e-02, 1.6624e-01, -2.4291e-02, 1.3726e-01, -5.4440e-02, 1.4230e-01, -1.7100e-01, -2.5478e-02, - -3.1050e-02, 5.7291e-02, 6.9354e-02, 1.3573e-01, -2.6337e-02, -7.0101e-03, 1.2106e-01, -4.7831e-02, - 1.5497e-01, -1.1770e-01, -1.9129e-02, -9.4948e-03, 7.8880e-02, 1.2086e-01, -2.8020e-02, -1.4646e-01, - 8.7591e-02, 5.2880e-02, 7.1011e-02, -1.0905e-01, 1.1377e-01, 1.6743e-01, 1.5339e-02, -1.6512e-01, - 1.2410e-01, -1.3107e-01, 4.0636e-02, 2.5689e-02, -8.2738e-02, 6.1553e-02, -1.5811e-01, 4.0263e-02, - -1.1207e-01, -1.9116e-02, 2.2739e-02, 1.5060e-01, -8.4344e-02, 1.1325e-01, -2.2457e-02, -8.3956e-02, - -1.5394e-01, -1.6220e-01, 1.7264e-01, -4.4088e-02, 8.8298e-03, 4.7926e-02, 1.2016e-01, 1.5087e-01, - 1.4337e-01, -1.3097e-01, -2.8323e-02, -1.0461e-01, -1.0100e-01, 4.1933e-02, 1.6593e-01, -1.4162e-01, - 1.0699e-01, -9.1653e-02, -3.4432e-02, 1.4032e-01, -3.9982e-02, 1.6097e-02, -1.2357e-01, 1.5048e-01, - -2.2852e-02, -1.2929e-01, 5.1762e-02, -1.2569e-01, -1.4027e-01, 1.0769e-02, 1.4016e-01, -5.0025e-02, - 8.3203e-02, 1.5191e-01, 1.1725e-01, -9.2730e-02, -1.9378e-02, -5.5629e-02, -1.4215e-01, 7.3040e-05, - 1.3301e-01, 1.4894e-01, 1.6493e-02, 4.0150e-02, -7.6530e-02, 1.3344e-01, -7.3558e-02, -1.2280e-01, - 2.7222e-02, 1.0596e-01, -1.5938e-01, 1.5980e-01, 6.3592e-02, -1.2385e-01, -3.8079e-02, 1.5337e-01, - -1.3563e-01, -5.1668e-02, 5.7979e-02, -1.5487e-01, 9.6909e-02, 9.2021e-02, 1.0964e-01, -1.1270e-01, - 1.7607e-01, -1.0479e-01, 1.7648e-01, -1.6965e-01, -1.5750e-01, 1.0858e-01, 1.8475e-02, 1.0198e-02, - -9.7892e-02, -7.4153e-02, -5.1687e-02, -1.7221e-01, 9.1869e-03, 3.1265e-02, -1.4966e-04, 5.7087e-02, - 1.6773e-01, 4.7002e-02, -6.4713e-02, -7.2752e-02, -1.1310e-01, -1.2254e-01, -2.8470e-02, -3.1266e-02, - 7.8641e-02, -7.5561e-02, 1.4093e-01, -1.2404e-01, 5.0243e-04, 1.5891e-01, 1.7578e-01, -1.0240e-01, - 3.1489e-02, 2.0881e-02, -8.2882e-02, -6.1075e-02, 4.7881e-02, -1.2292e-01, 2.9167e-02, 7.6496e-02, - -6.9662e-02, 1.4684e-01, -1.1634e-02, 8.0206e-02, 1.7506e-01, -5.4035e-02, 9.6062e-02, -5.0563e-02, - -2.5822e-02, -2.9959e-02, -1.0982e-03, -6.6781e-02, 4.1434e-02, 6.6610e-03, 1.1206e-01, -3.5781e-02}; - float dense_4h_to_h_bias_data[]{0.0177, -0.0658, -0.1480, 0.0715, 0.0226, -0.0708, -0.0598, 0.0462}; - float y_data[]{-6.1799, -9.0367, -7.6170, 7.9863, -0.7231, 2.1669, -0.6415, 14.4362, -1.0757, -7.2829, -0.6062, - -7.4960, 11.8125, 7.5786, -9.6781, 7.1442, 14.2660, 4.8352, -7.2181, 6.3933, -4.4885, -10.8612, - -2.3747, 0.0401, -0.9771, 6.1194, 2.9300, 2.2652, 6.4303, -17.4357, -5.0949, 5.3586}; - - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, 8, 4); - memcpy(x->data, x_data, sizeof(x_data)); - - GLMBlock model(&ictx, 8, 2, 28, 16); - ggml_set_f32(model.input_layernorm.weight, 1); - ggml_set_f32(model.input_layernorm.bias, 0); - memcpy(model.attention.query_key_value.weight->data, query_key_value_weight_data, - sizeof(query_key_value_weight_data)); - memcpy(model.attention.query_key_value.bias->data, query_key_value_bias_data, sizeof(query_key_value_bias_data)); - memcpy(model.attention.dense.weight->data, dense_weight_data, sizeof(dense_weight_data)); - memcpy(model.attention.dense.bias->data, dense_bias_data, sizeof(dense_bias_data)); - ggml_set_f32(model.post_attention_layernorm.weight, 1); - ggml_set_f32(model.post_attention_layernorm.bias, 0); - memcpy(model.mlp.dense_h_to_4h.weight->data, dense_h_to_4h_weight_data, sizeof(dense_h_to_4h_weight_data)); - memcpy(model.mlp.dense_h_to_4h.bias->data, dense_h_to_4h_bias_data, sizeof(dense_h_to_4h_bias_data)); - memcpy(model.mlp.dense_4h_to_h.weight->data, dense_4h_to_h_weight_data, sizeof(dense_4h_to_h_weight_data)); - memcpy(model.mlp.dense_4h_to_h.bias->data, dense_4h_to_h_bias_data, sizeof(dense_4h_to_h_bias_data)); - - ggml_tensor *y = model.forward(&ctx, x, 0, 4); - - ggml_build_forward_expand(&ctx.gf, y); - ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - - expect_all_close((float *)y->data, y_data, ggml_nelements(y), 5e-4); -} - TEST_F(ChatGLMTest, GLM2Block) { - float input_layernorm_weight[]{0.4692, 0.1864, 0.3191, 0.8249, 0.2995, 0.8105, 0.3017, 0.3836, - 0.5106, 0.0412, 0.4950, 0.4496, 0.4551, 0.4000, 0.8942, 0.8690}; - float attn_qkv_weight[]{ - -0.0019, 0.1341, -0.2058, -0.1840, -0.0963, 0.0670, -0.0050, 0.1982, -0.0222, 0.0662, -0.0756, -0.0491, - -0.2388, -0.1656, -0.1031, 0.0093, 0.0988, 0.1500, -0.1695, -0.1089, 0.0908, 0.2076, -0.0515, 0.1871, - -0.0403, 0.0265, 0.2264, -0.2319, -0.1574, -0.0633, -0.0974, 0.2160, -0.1620, -0.1151, -0.1747, -0.2341, - -0.1459, 0.2149, 0.1116, 0.1212, 0.0131, -0.1282, 0.0423, -0.2334, -0.1806, -0.1289, 0.1577, 0.1466, - -0.1109, -0.0090, 0.1599, 0.2485, 0.0992, 0.0338, 0.1676, -0.1472, 0.0466, -0.1938, -0.1733, -0.1291, - 0.1131, 0.1005, -0.1481, 0.0755, 0.1372, -0.0316, 0.0095, 0.0579, 0.1551, 0.2400, -0.1927, -0.0916, - 0.0983, 0.2071, 0.2176, 0.2206, 0.0498, -0.2174, 0.0230, -0.1564, -0.2330, 0.2221, 0.1901, -0.2494, - 0.0468, -0.0421, -0.0411, -0.1144, 0.0961, -0.1481, 0.0916, 0.1264, 0.1790, 0.0935, -0.2474, -0.1622, - 0.1248, 0.0523, -0.1950, -0.1440, 0.2352, 0.1685, -0.1090, -0.0629, -0.2381, -0.0045, -0.1883, -0.1928, - -0.0138, 0.0375, -0.1024, 0.1483, -0.1521, 0.2268, 0.1713, -0.2108, -0.0622, 0.0113, 0.0365, 0.0593, - 0.0981, 0.0150, -0.1220, 0.1183, -0.2398, -0.1482, -0.0626, -0.1218, -0.0875, -0.2049, -0.0532, 0.0534, - -0.1629, -0.0128, 0.1790, -0.0257, 0.0069, -0.0216, 0.0506, 0.1590, 0.2368, 0.1588, 0.2374, -0.0181, - -0.2246, -0.1185, 0.1702, -0.0016, -0.1243, -0.1916, -0.2340, -0.2110, -0.0507, 0.1371, 0.1352, -0.2411, - 0.1559, -0.1956, -0.0529, -0.1014, -0.0482, -0.0491, -0.2243, -0.2159, -0.0391, 0.0032, -0.1136, 0.0942, - -0.2250, -0.0169, 0.2199, -0.1020, 0.2258, 0.0905, -0.2256, 0.1582, -0.0288, -0.1116, 0.1999, -0.2020, - 0.0268, -0.0523, 0.1785, 0.0698, 0.1201, 0.0883, -0.0601, -0.0526, -0.2060, 0.1355, 0.1985, 0.1711, - -0.1763, 0.0111, -0.1762, -0.1376, -0.1457, 0.0854, -0.1490, -0.0055, 0.0105, 0.1612, -0.1890, -0.1716, - -0.1452, 0.1750, -0.0899, 0.2109, 0.0904, 0.0317, -0.0019, -0.0494, 0.0314, -0.0571, -0.0018, 0.0319, - -0.1956, -0.1310, 0.2019, -0.2029, -0.0180, 0.2473, 0.0903, 0.0071, -0.2167, 0.1238, -0.1781, -0.0710, - -0.0839, -0.0370, 0.0027, 0.2062, 0.0312, 0.2239, 0.1529, -0.1581, 0.1121, -0.1767, -0.1060, 0.0735, - 0.0825, 0.1876, -0.0805, 0.0004, 0.1287, -0.2418, 0.1807, -0.2067, 0.0034, -0.0425, -0.1317, 0.0330, - 0.2067, -0.0731, -0.1484, -0.0925, -0.2478, 0.1128, -0.1201, -0.1668, -0.1440, 0.1437, 0.1324, 0.1919, - 0.0907, -0.0835, -0.0699, 0.0739, 0.2055, 0.0680, -0.1183, -0.1175, -0.2364, 0.0540, -0.1403, -0.2229, - 0.2192, -0.1624, -0.0284, 0.0716, 0.0080, -0.1682, -0.2021, 0.1993, 0.0407, 0.2074, -0.0838, 0.0736, - -0.0572, -0.0111, -0.1523, 0.0846, 0.0790, -0.0052, -0.0562, -0.1541, 0.1729, -0.1861, 0.1024, -0.0841, - -0.1206, 0.0449, -0.1299, 0.0576, 0.0491, -0.1856, 0.0416, 0.1065, 0.0990, -0.0315, -0.2049, -0.0385, - 0.0868, -0.0912, 0.0949, 0.1665, -0.1305, 0.0025, 0.1034, 0.0196, 0.0209, 0.0312, -0.1965, 0.0196, - 0.1731, 0.2253, 0.1469, 0.0335, 0.1168, -0.1216, -0.2072, -0.2150, 0.2494, 0.1587, -0.1728, 0.0978, - 0.1888, 0.2499, 0.2186, 0.1937, -0.0573, -0.0877, 0.2053, 0.1401, -0.1504, 0.2248, 0.1208, 0.1363, - -0.1567, 0.0717, -0.0876, 0.1953, -0.0450, 0.0973, 0.0444, 0.1064, -0.0850, 0.1219, -0.1746, 0.0565, - -0.1691, -0.2466, -0.2008, 0.1974, 0.1353, 0.2346, 0.2003, -0.2233, -0.1706, -0.0404, -0.1624, 0.1736, - -0.1890, -0.1220, -0.2415, -0.1419, 0.2056, 0.2047, 0.1790, 0.1930, 0.2223, -0.0640, 0.1100, 0.2227, - 0.0827, 0.2499, 0.1297, 0.1554, -0.0875, 0.1200, 0.0287, -0.0597, -0.1409, -0.1403, -0.1924, 0.1678, - 0.1777, -0.0285, -0.1447, 0.1932, 0.1599, 0.0186, -0.1180, 0.2298, 0.1022, -0.1898, 0.2393, 0.1898, - -0.0911, 0.1405, -0.1420, -0.0392, 0.2123, 0.0103, -0.1768, -0.0836, -0.0679, -0.0482, 0.0239, 0.2312, - 0.0134, -0.1544, 0.0128, 0.1199, 0.1240, -0.2285, -0.0447, -0.1858, -0.1067, 0.0901, -0.1775, 0.0929, - 0.2122, 0.0164, -0.1666, -0.0896, 0.0546, -0.1906, 0.1242, -0.2270, -0.2403, -0.2429, -0.0507, 0.1681, - -0.2366, 0.2078, -0.1000, 0.0732, 0.0114, -0.2254, 0.2073, 0.1346, 0.2485, 0.1263, -0.1650, 0.2086, - 0.0134, 0.1186, -0.2005, -0.0719, -0.2455, -0.0974, 0.0539, -0.1963, 0.0797, 0.1342, 0.0348, -0.1673, - -0.1938, -0.0771, 0.1097, 0.2466, 0.1438, -0.0282, 0.0877, -0.2453, -0.2135, 0.1167, -0.1416, 0.1203, - -0.1765, -0.1238, -0.2059, 0.1305, -0.0255, 0.1924, 0.1547, 0.1383, 0.0080, -0.0773, -0.0544, 0.0332, - 0.1239, -0.1751, 0.2098, -0.0272, -0.2095, -0.1353, 0.2212, 0.2286, -0.2316, 0.1763, 0.1253, 0.1480, - 0.2116, -0.1347, 0.0789, 0.1023, -0.0739, 0.0837, -0.0719, 0.1546}; - float attn_qkv_bias[]{-0.0694, -0.0932, 0.0629, 0.0887, -0.1221, 0.0221, 0.1449, -0.0249, - 0.0761, -0.0603, 0.0876, -0.1811, -0.1470, -0.1269, 0.2298, -0.0673, - -0.0007, -0.1211, 0.2496, 0.2442, -0.1885, -0.2027, -0.1895, -0.0012, - -0.0637, -0.1636, -0.0897, 0.0472, -0.1306, 0.0554, -0.0573, -0.1211}; - float attn_dense_weight[]{ - 3.4346e-02, 2.0556e-01, -1.6902e-01, 1.1609e-02, -9.2192e-02, 2.4533e-01, -2.3719e-01, -2.3967e-01, - 2.4635e-01, -1.5817e-01, 4.7931e-02, -2.1577e-02, -5.2664e-02, -5.5837e-02, 1.5886e-01, 1.1949e-02, - -2.4340e-01, -1.4759e-01, -8.5225e-02, 1.2580e-01, -1.6178e-01, 2.3573e-01, -5.5682e-02, -4.4890e-02, - 1.9589e-01, 1.2565e-01, 2.1203e-01, 1.4461e-01, -7.5838e-02, -1.6587e-01, -1.8601e-02, 2.0691e-01, - -8.3905e-02, -2.3184e-01, 1.0248e-01, 2.4337e-01, -7.1171e-02, -2.0701e-01, -2.2676e-01, 6.2650e-02, - -1.8930e-02, -1.2625e-01, 5.0533e-02, 9.4940e-02, 1.9883e-01, 1.9410e-01, -3.7421e-02, -2.2044e-01, - -2.2591e-01, 2.3342e-01, 1.1051e-01, 1.0898e-01, -2.1631e-01, 2.3150e-01, 2.3684e-01, 2.2572e-01, - -2.1090e-01, -9.4331e-02, -1.7194e-01, 2.3674e-01, -1.0742e-01, -1.1414e-01, 1.3098e-01, -1.1565e-01, - -1.2313e-01, -2.1868e-02, -2.4030e-02, -1.9474e-01, 2.0842e-01, -1.1028e-01, 8.8679e-02, 2.1745e-01, - 1.2608e-01, 3.5395e-02, 2.1271e-01, 3.3607e-02, -1.1566e-01, 2.3650e-01, 5.9170e-02, -2.4392e-01, - -7.1173e-02, -1.7029e-01, 2.1919e-01, -4.1286e-02, -2.2788e-01, -1.5716e-02, 1.5700e-01, 6.4950e-02, - 7.9051e-02, 2.3218e-02, 9.3205e-02, -6.0913e-02, -9.9450e-02, -2.3368e-01, -1.8833e-01, 1.0835e-01, - -1.4803e-01, 3.5867e-02, 7.9798e-02, 1.7701e-02, -1.6209e-01, 2.3908e-01, -1.4537e-01, 2.0563e-01, - -1.9888e-01, -6.0137e-02, 1.3599e-01, -1.0215e-01, 2.1001e-01, -1.7204e-01, -2.0994e-01, -1.1273e-01, - 4.0413e-02, 2.3019e-01, -1.1935e-01, 8.9407e-02, -6.2677e-02, -5.4216e-02, 1.8383e-01, -1.9375e-01, - 2.6553e-02, 2.3510e-01, -3.4353e-02, 1.9411e-01, -7.6989e-02, 2.0124e-01, -2.4183e-01, -3.6031e-02, - -4.3912e-02, 8.1021e-02, 9.8082e-02, 1.9195e-01, -3.7246e-02, -9.9138e-03, 1.7120e-01, -6.7643e-02, - 2.1916e-01, -1.6645e-01, -2.7052e-02, -1.3428e-02, 1.1155e-01, 1.7092e-01, -3.9626e-02, -2.0712e-01, - 1.2387e-01, 7.4783e-02, 1.0042e-01, -1.5422e-01, 1.6090e-01, 2.3679e-01, 2.1693e-02, -2.3351e-01, - 1.7550e-01, -1.8536e-01, 5.7468e-02, 3.6329e-02, -1.1701e-01, 8.7049e-02, -2.2361e-01, 5.6940e-02, - -1.5849e-01, -2.7033e-02, 3.2157e-02, 2.1299e-01, -1.1928e-01, 1.6016e-01, -3.1758e-02, -1.1873e-01, - -2.1770e-01, -2.2938e-01, 2.4415e-01, -6.2350e-02, 1.2487e-02, 6.7778e-02, 1.6993e-01, 2.1337e-01, - 2.0275e-01, -1.8522e-01, -4.0054e-02, -1.4793e-01, -1.4284e-01, 5.9302e-02, 2.3466e-01, -2.0028e-01, - 1.5130e-01, -1.2962e-01, -4.8694e-02, 1.9844e-01, -5.6543e-02, 2.2764e-02, -1.7476e-01, 2.1281e-01, - -3.2317e-02, -1.8285e-01, 7.3203e-02, -1.7775e-01, -1.9838e-01, 1.5230e-02, 1.9821e-01, -7.0746e-02, - 1.1767e-01, 2.1483e-01, 1.6582e-01, -1.3114e-01, -2.7405e-02, -7.8671e-02, -2.0103e-01, 1.0329e-04, - 1.8811e-01, 2.1063e-01, 2.3325e-02, 5.6781e-02, -1.0823e-01, 1.8871e-01, -1.0403e-01, -1.7366e-01, - 3.8498e-02, 1.4985e-01, -2.2540e-01, 2.2599e-01, 8.9933e-02, -1.7515e-01, -5.3852e-02, 2.1689e-01, - -1.9181e-01, -7.3069e-02, 8.1995e-02, -2.1902e-01, 1.3705e-01, 1.3014e-01, 1.5505e-01, -1.5939e-01, - 2.4900e-01, -1.4819e-01, 2.4959e-01, -2.3992e-01, -2.2274e-01, 1.5355e-01, 2.6128e-02, 1.4422e-02, - -1.3844e-01, -1.0487e-01, -7.3097e-02, -2.4354e-01, 1.2992e-02, 4.4215e-02, -2.1166e-04, 8.0733e-02, - 2.3721e-01, 6.6471e-02, -9.1518e-02, -1.0289e-01, -1.5995e-01, -1.7330e-01, -4.0262e-02, -4.4216e-02}; - float post_attention_layernorm_weight[]{0.1611, 0.7323, 0.1078, 0.0743, 0.6528, 0.5073, 0.2667, 0.0177, - 0.3064, 0.6670, 0.0372, 0.0143, 0.5634, 0.1398, 0.0620, 0.3074}; - float mlp_dense_h_to_4h_weight[]{ - 0.1112, -0.1069, 0.1993, -0.1754, 0.0007, 0.2247, 0.2486, -0.1448, 0.0445, 0.0295, -0.1172, -0.0864, - 0.0677, -0.1738, 0.0412, 0.1082, -0.0985, 0.2077, -0.0165, 0.1134, 0.2476, -0.0764, 0.1359, -0.0715, - -0.0365, -0.0424, -0.0016, -0.0944, 0.0586, 0.0094, 0.1585, -0.0506, 0.0251, -0.0930, -0.2094, 0.1012, - 0.0320, -0.1001, -0.0845, 0.0654, -0.0452, 0.1634, 0.0142, 0.0944, 0.1089, -0.0613, 0.1082, 0.1845, - 0.0115, 0.0489, 0.0091, 0.1731, -0.1055, -0.1329, 0.1089, -0.2176, 0.0040, -0.1150, 0.1650, -0.2328, - 0.1516, 0.2473, 0.0506, -0.0167, 0.2461, -0.1068, -0.0200, -0.1095, -0.0345, 0.0543, 0.0334, -0.0472, - -0.2500, 0.0128, -0.0074, 0.0376, 0.1916, 0.2430, -0.1483, -0.0156, -0.1002, -0.2304, -0.1819, 0.2346, - 0.1458, 0.1343, 0.1389, -0.1986, 0.0263, 0.2327, -0.1395, 0.2224, 0.0657, 0.1759, -0.1071, 0.1153, - -0.2216, -0.0159, 0.0834, 0.0750, 0.2092, 0.2457, 0.2274, 0.1678, 0.1758, -0.0323, -0.1940, -0.0928, - -0.0281, -0.1351, 0.1278, 0.0878, 0.0403, 0.0604, 0.2196, 0.0911, -0.2192, -0.1815, 0.1102, 0.0341, - 0.1219, -0.2497, -0.2307, 0.1533, 0.1599, -0.2262, 0.0949, -0.1952, 0.1893, 0.0784, 0.2472, -0.2465, - -0.2407, 0.1690, 0.2220, 0.1532, 0.1662, -0.2106, 0.1810, -0.2360, 0.0559, -0.1638, -0.0993, 0.0443, - -0.0983, -0.0396, -0.2370, -0.2357, 0.0968, -0.1058, -0.1521, -0.0986, -0.2044, -0.1999, 0.1496, 0.1271, - 0.0333, -0.1366, 0.0832, 0.0112, -0.1787, 0.0538, 0.2276, -0.2459, -0.2486, 0.0320, 0.1883, -0.1521, - 0.0550, 0.1757, 0.0771, 0.0541, 0.2494, 0.1629, -0.1406, 0.0216, 0.1277, 0.2205, -0.2487, 0.1354, - -0.0726, -0.2448, 0.0555, -0.1361, 0.0354, -0.1623, -0.1881, -0.0212, -0.0840, 0.1462, 0.0216, 0.1951}; - float mlp_dense_4h_to_h_weight[]{ - 0.0765, -0.1313, 0.2765, 0.0223, 0.0766, -0.1313, 0.0015, -0.0785, -0.0347, -0.3184, -0.0025, -0.3389, - -0.0878, 0.2366, -0.2839, -0.2752, -0.0997, -0.1119, 0.0691, 0.0678, 0.2070, -0.2812, -0.0288, 0.2283, - 0.3531, -0.0298, -0.2362, 0.1174, 0.2325, -0.2551, -0.2369, 0.1804, -0.0286, -0.3856, -0.2354, 0.1656, - -0.1385, 0.2818, 0.3227, 0.0594, -0.0323, -0.1296, -0.0206, 0.0743, -0.3118, -0.0976, -0.3396, 0.2506, - -0.2600, 0.3730, -0.1052, -0.2339, 0.1961, 0.0609, 0.2827, 0.1704, -0.3933, 0.2582, -0.0769, -0.1804, - 0.2593, 0.2977, -0.3588, -0.0369, 0.3353, 0.1581, 0.3439, -0.1399, -0.2252, 0.3511, 0.1702, 0.3919, - -0.1705, -0.2621, -0.0479, -0.3844, 0.1600, 0.3011, 0.0980, -0.0403, 0.2024, -0.2591, 0.3994, -0.4059, - -0.3911, -0.0965, 0.3335, 0.0409, 0.1568, -0.2992, 0.1489, -0.0456, 0.1636, 0.2883, 0.1775, -0.0347}; + fs::path test_path = fs::path(__FILE__).parent_path() / "tests/data/glm2_block.data"; + MappedFile mapped_file(test_path.string()); + char *ptr = mapped_file.data; constexpr int seq_len = 3; - constexpr int hidden_size = 16; - constexpr int num_attention_heads = 4; + constexpr int hidden_size = 32; + constexpr int num_attention_heads = 8; constexpr int num_kv_heads = 2; constexpr int ffn_hidden_size = 6; constexpr int max_length = 8; GLM2Block model(&ictx, hidden_size, num_attention_heads, num_kv_heads, ffn_hidden_size, max_length); - memcpy(model.input_layernorm.weight->data, input_layernorm_weight, sizeof(input_layernorm_weight)); - memcpy(model.attention.query_key_value.weight->data, attn_qkv_weight, sizeof(attn_qkv_weight)); - memcpy(model.attention.query_key_value.bias->data, attn_qkv_bias, sizeof(attn_qkv_bias)); - memcpy(model.attention.dense.weight->data, attn_dense_weight, sizeof(attn_dense_weight)); - memcpy(model.post_attention_layernorm.weight->data, post_attention_layernorm_weight, - sizeof(post_attention_layernorm_weight)); - memcpy(model.mlp.dense_h_to_4h.weight->data, mlp_dense_h_to_4h_weight, sizeof(mlp_dense_h_to_4h_weight)); - memcpy(model.mlp.dense_4h_to_h.weight->data, mlp_dense_4h_to_h_weight, sizeof(mlp_dense_4h_to_h_weight)); + ptr = map_tensor_data(ptr, model.input_layernorm.weight); + ptr = map_tensor_data(ptr, model.attention.query_key_value.weight); + ptr = map_tensor_data(ptr, model.attention.query_key_value.bias); + ptr = map_tensor_data(ptr, model.attention.dense.weight); + ptr = map_tensor_data(ptr, model.post_attention_layernorm.weight); + ptr = map_tensor_data(ptr, model.mlp.dense_h_to_4h.weight); + ptr = map_tensor_data(ptr, model.mlp.dense_4h_to_h.weight); + + ggml_tensor *x1 = ggml_new_tensor_2d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size, seq_len); + ptr = map_tensor_data(ptr, x1); + x1 = ggml_dup(ictx.gctx.get(), x1); + + ggml_tensor *ref_y1 = ggml_new_tensor_2d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size, seq_len); + ptr = map_tensor_data(ptr, ref_y1); + + ggml_tensor *x2 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, x2); + x2 = ggml_dup(ictx.gctx.get(), x2); + + ggml_tensor *ref_y2 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, ref_y2); + + ggml_tensor *x3 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, x3); + x3 = ggml_dup(ictx.gctx.get(), x3); + + ggml_tensor *ref_y3 = ggml_new_tensor_1d(ictx.gctx.get(), GGML_TYPE_F32, hidden_size); + ptr = map_tensor_data(ptr, ref_y3); + + ASSERT_EQ(ptr, mapped_file.data + mapped_file.size); // self attention { - float x_data[]{-0.3439, 0.2291, 0.4366, 0.2702, -0.2002, 1.0184, -0.4169, -0.0308, -0.9846, 0.2024, - 0.4156, -1.2037, -0.1333, -0.2934, 0.4649, 0.5762, -0.0578, 1.1715, -1.5185, 0.0475, - -0.0103, 1.4353, 0.2518, 0.7408, -1.0982, 0.1548, -1.3989, 0.0408, -1.1174, 0.7029, - -0.6675, 0.1254, 0.4047, -0.6549, 0.0521, 0.3401, -0.2124, 1.5629, -0.9072, -1.5662, - 0.0485, 0.9935, 2.1180, -0.1827, -0.7768, 1.7764, -0.5033, 0.0566}; - float y_data[]{-0.3964, 0.2281, 0.6048, -0.3200, -0.0622, 0.9805, -0.7734, -0.1910, -1.1877, 0.1297, - 0.4010, -1.0275, -0.0449, -0.5022, 0.5744, 0.6155, -0.1103, 1.1213, -1.3659, -0.3552, - 0.1080, 1.3184, 0.0149, 0.6601, -1.1811, 0.0690, -1.4537, 0.0565, -1.1463, 0.5818, - -0.6814, 0.0777, 0.2416, -0.6671, 0.2555, -0.0109, -0.1029, 1.5266, -1.0859, -1.6493, - -0.0626, 0.8874, 2.0539, -0.1481, -0.8140, 1.6529, -0.5594, 0.0603}; - - ggml_tensor *x = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, hidden_size, seq_len); - memcpy(x->data, x_data, sizeof(x_data)); - - ggml_tensor *y = model.forward(&ctx, x, 0); - ggml_build_forward_expand(&ctx.gf, y); + ggml_tensor *out_y1 = model.forward(&ctx, x1, 0); + ggml_build_forward_expand(&ctx.gf, out_y1); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y->data, y_data, ggml_nelements(y)); + expect_all_close(ref_y1, out_y1, 5e-5); } // cross attention { - ctx.gf = {}; - ctx.gf.n_threads = 1; + reset_cgraph(); - float x2_data[]{-0.5481, -0.6917, -0.6559, 0.6949, -0.4671, -0.1680, 0.7585, 1.1881, - -0.6305, -0.0654, 0.6188, 2.0020, 0.2952, 0.5314, -0.5227, 0.0995}; - float y2_data[]{-0.6584, -0.6923, -0.5611, 0.5060, -0.4226, -0.2116, 0.5962, 1.1427, - -0.7516, -0.1508, 0.6161, 2.0421, 0.2349, 0.4791, -0.6062, 0.0516}; - - ggml_tensor *x2 = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, hidden_size, 1); - memcpy(x2->data, x2_data, sizeof(x2_data)); - - ggml_tensor *y2 = model.forward(&ctx, x2, seq_len); - ggml_build_forward_expand(&ctx.gf, y2); + ggml_tensor *out_y2 = model.forward(&ctx, x2, seq_len); + ggml_build_forward_expand(&ctx.gf, out_y2); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y2->data, y2_data, ggml_nelements(y2)); + expect_all_close(ref_y2, out_y2, 5e-5); } { - ctx.gf = {}; - ctx.gf.n_threads = 1; - - float x3_data[]{-0.1169, 0.3728, -1.0211, 1.0795, 1.1469, -0.1733, 0.0637, -1.2699, - -0.6212, -0.2381, 0.0892, 1.8008, -2.0627, 0.3222, -1.1390, 1.2418}; - float y3_data[]{-0.2959, 0.3463, -0.9365, 0.9941, 1.1817, -0.1974, -0.0675, -1.3237, - -0.7128, -0.3565, 0.0678, 1.8228, -2.1314, 0.3017, -1.2696, 1.1742}; - - ggml_tensor *x3 = ggml_new_tensor_2d(ctx.gctx.get(), GGML_TYPE_F32, hidden_size, 1); - memcpy(x3->data, x3_data, sizeof(x3_data)); + reset_cgraph(); - ggml_tensor *y3 = model.forward(&ctx, x3, seq_len + 1); - ggml_build_forward_expand(&ctx.gf, y3); + ggml_tensor *out_y3 = model.forward(&ctx, x3, seq_len + 1); + ggml_build_forward_expand(&ctx.gf, out_y3); ggml_graph_compute(ctx.gctx.get(), &ctx.gf); - expect_all_close((float *)y3->data, y3_data, ggml_nelements(y3)); + expect_all_close(ref_y3, out_y3, 5e-5); } } diff --git a/tests/data/glm2_block.data b/tests/data/glm2_block.data new file mode 100644 index 0000000000000000000000000000000000000000..0c18db538e72d31f9a44b39f47abcc0dbfd69be7 GIT binary patch literal 14272 zcmWNXcRW^q6vyqo_nz4^RPx;OeJCy3d+OKF9yEw{R#HYtRw3C%8R5C-`$VNdB}Gvv zlG2c@BKi6E{&Vl^+}Amu_xYTU_bCA`@6sOb(38ictzCfoD?6I_eZEU7GbeFfGh!GS ztxeq1((Ad}Q>JpI8a8r67ZnjS)8#4ziE;M~=#t-YsYFKaIuR(jOMFgw!`2>k5?a(v z{N$R+sa^&SteL`bbe+f zhVA(X8)m(Np{O+cCG>(<9MKI*gN~%OFbc9V?lHgc7c8}EM4#_s;JsRgp8Ga}zeIVc zP=5`^F2}&SLrqxDUkeZTH8As08T;I-m6P|t3~naAhlTYvWba97s&gY4yXQv2{ZUoI zgz6I43Voukr2zAG>QZy(5XSRF2WA$V(@Tv8r1qL5*?Q55ehU?cnt^_H{h}Z2!;0(Z zJv{>I_c!6D@?aQKy#rR41WBz|7|gP8q<$?u=>MPuy3VUZ;}lsceWV_zd0LY4U;S|H zZ7OnMDlYJ`rb^6Wp8paFx;=d-gRm((ZA%;bjv17qOM{wB zb;9jMLQK@WmrN+n3dWTp!Q;3MW0_Tia*Olfbfyipjc-DY8B;J}Z4vq|JdQG6Dzr#c z3rlOXK<({U6uPFuPKg&JAKT*C4P$AX(02i>?KvfQ?(hjbj#v@nY)+bH{u_)?i zOf+?apliJ`O_1+`9_?(lF*O};-Blo;<|xrmTa%ehiRzf+E=WA|J-~G626U92goXOH zC^AO_?^vr+6RZ1BTp>?p+!=Key=*ln^D-$ z>HO~t$TdmO84B%a@Q7a|#rdx?B|5?P&RB|N7kIq1eU?wN5U57@mE{6NP0@VE60*wE8 z096h(vE#mDE*loA6353SjGM6$De!BBCmkPg>VFw<AnJe&gUsZtn3Yn2e7=&InutzIe8wj>Co>V5IzDi0LtSpjWNRmkr)cMzVR zk9l@^*lS`%1NZBIpH(d$*=k5VczPsH=PO3gO!lnrB9MwG=L{zIqxVuJ8aYpa)YX4w z5^U5+03N_Eb5?*yoC&UfD!^_$`3|Ra9D+OFx5D+m`*`JVjIhJ19Sb*%z|tx)8oM|S zH~|Myw%{h-QWk<43M`htGKKERa*)4Wj3(>$G2b1Nz-wU?lz5b}=&eF6p`D!-coa@+ z2IKra;c!E%4+NwT8%}+6X)lh4T){0&>knT>?qvznxKNJfrEXz%7mt9yo-t{E{0h?K zs+oJrx$suM2&V|C(WqQ)YIn(r9DUIRZ(T=F=lTV(E0!Qy{Z7PVt{!mNWvPX{)!i{Mh@aeU?@OOAL5;q+hUAvZ+;93_*W zLf#LhLW&@@{V%QxH-?7(0aS~3pk7{zwDz+cnfm@7Ja4oh4{2&8(+@7U1jy+ZnQZy# z7ij%On#hE7!y3<3jQ5W`ys*KPY?F~bqA;HO4DG(@A0(sBq<5n)1HjA@s?8X4-XhLRtSMC@}J9QgeM zw=1g?^;~UcZec!znkdqUnTydFC)3YD7mzyXz$8OCy88Jxn7z6h>hLks!}*1>Yh-Bs zmW>$r^$MJibRdsx>tTcPH8!#RIBZFnOk0z+sLqcZ*st&u-&x-Tqu6m=x3n0~YN=BD z?<_2fie+UMOB1f#dw9M45v03Ua(1+B#KHZ#`1SBzW`Tb(2xN|7?>BVf@}1 zLj;>7BuoE%RwLuZS21h%DM&n^ON8RfAX@w>6Dq1jg7d4Hu9MjqOsvTGMk^RewWmSG z3$ZJC2xCh1h>>|O`$NElu2Y)I?%mJ{mvtFv8Q2PWCqkL?e?no#9((*C5{+{lE$A9A z33~B!3QYED!qMvI%sIX7@L6#IGq17-Ep=^3KxhLV?o=Tu0t$50+7=al-Uo?qvA9J+ zgJc>1!RNvnRND0#&eG^%iVIB1o~8oK3pfozXQ$vEFD z0KIy<;n&a|c1mA97N+ac0yR^r_}HFYk9Wdnc?!(Vh3?Rn|4MtFR(HrTrF;mhezhLKZcPZfyEJf>)=1VFMKeuvjh~a`;JTvq*u+O8&!}rNS=H%8kff0ViT` z=LP1D2*7Ijd`5xhB3K4usGd9A=a1sSpFbFWZVFiCM`H1MHF6@EumdwP0cSdr=4BgD zlf4gq+KX^r_)9i@_aNK;E2`4-{#k4;vw&;vEgYQ_U2rv03DcZom{32Y{%tdJzeI|>?zbZm zPnu9X(TSZbAA`M(rgT@iFs<*hAXC*}g58m7Sn2o}zSPUp5Y@|!W1u+Pd9(!s(p=Gl%%xsW}-Sv~nzjOzxxBbD5-t$?TV^1I{R);*DwT!)T^aYxm zEC#TC210fncw4Oj8PPO&^uvQ4{%uLeX+1{1xCmXB4kLE{0WU3iQkvVq{3qtko__6( z3lAx=YIVX8KTC?5$lBn6i)wV(u?-xeKH+gM6XKHXf>zs~d9I+GArn$${@xWg_qS*3_E&Lk~ zUyN<(f2CWnx#bSdNYNlgGcG}^stENw`h_`ZCdzZT8;AaPpJD54AG9o#BUkPT()?cn zbk5d|xb56L4x=>-JYL@5yjWn1MImo-@i#%-fiY1I4m+khAEdy)Ho>vM0cKngk$Ek z@8e_qlV(E83k9jv+H3f=PL(toDBxi4bI|)@MGXIC!Y3IcdToMbRW#1Q>;*DpezXH^ z&S`_cRx0H0PILM;D1i|>t3^z6gz5hHy&y4i6@+U53P&e)muV`*XQ4I`I8(x6gE}d{ zWlq~S+{1=eEsRxPg&&>FNMm&#)T@30&6z{+)>)iNS7x)T{a-TdvI*(7u7oPNXpCO5 z7Pt7T6I?z6Zyp;(k-$NaHy?yS=`Gk-rGWAmMX98fI{iE8EQsbNVS}~;dETc$TdW?j z>r4VUmlZ3)?6eX|v>pT}udA@zIt%B{7Nk@93~744JYyYWNBqu8;>vgZyrKwUdThQI zJPPfCIY%7m=CA)yOI3`-?w{bOi%#sK{eoa*`;OfvJBD7S7WAs_572@F_WhwyoYLij zZ~Pu${^6B~TeRtbpEwz8jD;idWvFNDkFk3)AVnb*XZDhbo|psapE~i&rYW#=%4Ehz zQ;%5wJA)3!F1U4An;14I({a}&Ov+AMa(P(>l5;j>N}2(Axx<`{*qvizs*K5eekzW& z_~Vk^vv6rp17^iPgkmvk@;1E#t0vtA3)cf6EWihc{Co)78-~q_@-(4CfrK1*hH`%0 zu;YL=y?b_qZH*grxe`6XEYjV@%(vJIXGl7`bNOVLW8_GLXY`<=p&t9f;T(Q@rALYf zwaA;nRG#rzAPU`T$48-pWd6Z9?3!biWa!W{IB-CZm|Dw|RQIpA_2)A9PyZ77W|xAL z(;2pu7sy=MGJ+b94C%=gXW{isb9f(jl#%Wlgan%eoF9A&(!C#|m$(|tTWEsG5o54+ zz8;z8tVqIC7Rl`lA|O02hvh6;#PqNSZ!E^4V8iHr*12{1oi3V$qwXrZUKJ0 zEQAS_ZuneGpYrC9vjbThz+Na8H+>l5nHFrQ%ye?1mFI=YptBMYe?0`2kN2XPJ;M7$ z3mR_Vh^p?&Xi<6+cgS4^W;he8-(<6URhHoRm<6>r?PcHY(4>yqFG1>L6*R_Lkeen4 zF?FsoBlTJqyl*=Z{ANNTym)v`T#H7U8q@YwH?i44gKV%p2vc)x$$M)t;yq}_8QSz7 zhws^w|E4IC5ljJgf+KvB~YgjTsML%Vb{8el4h9&3R!HOqzVX}X2Rb4DTVV=Zj+c0dd1D0E2agsS7ZB#LiB zBYkyn+160@gh~+J3fCgHJ~y)3ftqB$y0?p`YASo$<|SrV`(Wvkop5tf8rW++<7FPT zCU$Wnc&s!;fspMuh+wy?~~fgBn*3`@?2 zv59}3K=V>P8faIJDNDgFyU`wA=Py$3*Uu6MO}*S zzi|x)7leV?^-^~5hAKVrEE*n@Z!p<&j6IT>2U{nHLe|}V7@ZO@SholNCLrePsgn8-K8%SEv1Y%O84HU2`U`Pm3)Y>a^AcWZ(3xEHpYT40uXIqRw{NXpKHl)t<970cJl(2~F>pwp*H`r~AX zfj}P<^!X21IhxYmT|%_L)|R@zQ6xWeUR2VOEVK|h3>_h1IBx$L-L6GIXRHXMqzh7M zzaPl*b0I!clH7i{o2h(Lig#Yzgr4ZXOhlj_mAuk}Z;tALz36+mab*WSlPd((Y436B zbQ|(}RDg_L6^CPr>5#8}43=MO!~@b#Djihafb2I$-%BEQyf(I>RNocbj z0Yz)rGC0BBj~wY_p{wl4HhK89QkM?c6v3{MaNOH>24kW!T04vomv&?lTbyXwkiuHUb>+{;W?Mo z$KojNA`i0{8WM%YZ%~4_5q2L_q=TC}87Bi((xN*6IU?F*XyhKarR`_m$^t~sNkF%3 zO)_hCDYP9Ip#mW~WLb|DZxLUBU12pFeWkdlJEXv@q(xY;*_;X-uf$F@1=PJQL53|w zNo>y>tl8bhlN?=)T{o)mc=kV*#~j7P^-tjF`QizNZ(}MK)ZysgO;~(Nj&!lNaFV$# z+xEel488P+(b?sA(yJLgdyZom-HgZ5QekAo0HgCY$lJ^s?BBf^bDjsmu8Rs}pM?$S zc-+l+S`Kg?I%S}6R2_`8yg;?31XDfLX{4wvEuVPrs{8It;{pZpO-g|nj3V6chz8v+ z9Q2)g7g+V1@F6Dp3Ib!}kh6OV?B4JILN(-xTbdmFcJPCk2wAG-JOg7F1~L3Yg$(z_ zNen;eNbfITa9`3nc-f{+IR3`;Typ{3xo1UtwQXp4MF)PH&1XkkJm7hrB>m|=Pw27D-s2(9xR5>s*_1daxD53pM)}v zi@34;HVFC}us!x-#KQP7Jjz!i^)i)kKdqUwc3hj}Wxv8ntR<=Gv?n6ptVy>krq+fIR*RA@5Ry515jTx4|?og;&G+(%sL-QDz;}inyE)( z$qhZy{mY1!RW)L!>14`JoeFvrJkN9A1?>T&=+@JXUkA#ud}%ZDe1$17j7T#KZ4r=VnpguSaF1%R?)1xnM0D-iWdo)3jtU*fvLja=ip;vhc#zidy`Y zy_p@0mL)~v0nGmMqtN7DP`Sp!3#|7%=E!qXIRi7S=^Ix)I^%&4v(2s;7CQFB{FS4e zeaV+WXm}n36i}0iLAOIQAS@ zCJV3mqkdBsCWQRM5#5`ZdS8|<{!@>+pC-}xpNgm+69bobP1Jw?>57QcA*iHR2%V2~ z$i3fuwq|M>R;o$UYb6sbBew*O8JWV*;c9TPea$M0SHiT}hIIM$8E}$kN21LfX+<9o zweEMrqg@?r!027rO=mF9A38Wozpq2>EfQo)!z1{wOb|Q;TN&FaJfIgo0QXEB z9?ZSM`u1Icy#0l!In#oOj}PNx`D~agE=08hcfiuNzaSHPlW{kRVQ)_V1SKPabiUgF zy3Ez0Gg{iPVbvv$vt=gEUTg@ZNA7VhXRDF)(G*;*IvZUN_JV@iG>ltlL$=pO;o^H^ zm0cIC(D!o=+$ui@yxO}s6!immlFsZ}t!Wr|YbP&ua1{74lZlCC5xmZW88q2;a_2!vQ)h2=ohUEAcXNB`#py+Fw}FuSW5Z0%?3-#_AQP@rJBRFnZNr$aa2?OZHvF zJqu;2<1`IAxzvEneI!j4e>`M;5|$vO1cOSx8(g?AMX$yfk@6)@)Ql6w$o=+Zm(-cz zwL3%L)1*P%GFfI3J&6`^V#MWeC%l#s0$u6Z_}{bv&dyba#9KO*!6G?)K0}jU5voQR z6e5WuwY+=nu58iJdH6p02YO`BWVd8_qV`^d|CT6|f@PghJt#)TYYgetuv4gTET8$z z*QADf+`(hfcU)tf3}$DP$g&nKDqW~fPJPdTCp|wf#NCFhf2c#}@cMD8MhP0_DY5mU zLh$Oua-0-5v6EY5vsYG0lC+J|I}Q$%>cd{YGF6qJVnX#6vjJxamfk= zqT>(^5klHDZ-p2kB92tg+ni03>cBr0+VsZJC?-9;5WC(85#uJ8NkChuL#Zg@z3&l0$u>R9V`A4Ch?H8CTPB zpX@_6(sL!KT8Lp#o)hiO`Uh6?9Z6wM88jJACc~>|!SsR6>>67I>O57CI?eop3U{XA zk{1o2urVH0N8j@rPZ{$ZbG~8vvlL#zBU>`O9irEYYyZ% zFNEJ<1v^Vla>i1h!|!=|Sf=N>xe`nh>*E1&#*g(&l$RF1kX-nz*iA7nxrDftP14dOI{Cz zq#T5FtqGs+Ml8!(aMY759KOvzy-eH@}!Fn%ZlZpLVGQ=uGA&rd#`|@!67`gaSs+BEQI45 zWN`YREHrMnqr9Qp=pPsgJ&)wrPMMon(xpp3X?z07cts3)JDG}Iw*f#I{8&uI1 zp)Fr+$*W^-E`Qi&Jh`GCv|g&=x9zhTy+jrxmTFQnpPx83!Nq%IgsIFf2lAt4#3e5A z99F4DfzL4ms$g4zyGO&o=9wn#^(g_fg9n-2y=v4o%aN{mvlVLf^I_YBU&$yM&<{EF z=-89U)L;XrW%YEBlMsM)_3G?xIX5R@ajl8=1zkwfc*Hm?E@3}#I-u3J7+3iZf|k4$yOz82#2l6m?jV9^m_hO1?6})P`&dQi=2d62r zL_EnIV73~K6IhFZW%BTF;^w=t>L;^cdo^nQkVoC%)1XtNNOkvr#uS6=pepG|6HWg> zM4U9WjBw%EWU4^y+epYQd5`ZuR5Mwf$)G$_o4lPaLEro<#^b$9VgFzSZ^q&Q>{_cq zkMj)Z*orZfHc7&n+eXp9PLDdPZe}``yn=XeRH4952Ns7tSh=V)RtT&z+9cOt6Lh_V=~RPPP45*76-s^^BrM^iOEtwF~JM7xtk4a)4=Vn67-9S-bi* z2;Y2(BzGz7Y$LQdIBZFqE^+YUyCR`A%(mefSt!By%9@!VM>TC**R4Jw$;oSZTnRF zUf4`-u-#sg*|v)7zIr8h-A@JX4vi3S8ksfWD;sm?x><8W*ZOn4Ox(EVx7u>=yAP9~ zIjgw!FEWUNo)cF}DU>Wu86$VDdva5GF5F!gYKV39UlR9V3sLvra7PVRas%b9xu*^V zGX+N#39o1ccXeDgJ7%s*FLEcm!zz|}S!_?2KXV1=DpPuRqONX44Hr;rdeA|M++CtV ze6Ir%`3F4q*p_*_Ru?Z`fSvnv>N59-jgcztHx zzD(AvUYFe5v=37!ayP*`lFYK43>ADG>@>ljE-NqnVE=Zkgb*cFGApf?m?yMo2kQ^#cYMeLNlQ|Ly4~0;{+eh zv_ge)7W2Y3n7x`eanCo?quaUHas9PxIKD%N{86=|U#~_nf7aO1f}@lt{3{eJms?<6 zxgD_xQFalFE633?7YLclLX1&)MZjAv(rVm=UI9s<9`_XHH59|#zhUJSw-o;WqKv+JiZJccni`9A4NKRa|ALBH{#Lb_SDb7 zl-&5Vl4CsM7asADqdp=UbU~#a{qoG1T)g@RiaPf&j&ro=^v!_2GL}^1Yyeccxnkol zEfo5;38iT+u*3J@PO2WPUR}!8>O`?`_j~Ys>%^#5hCNZ8l!Aj94KQc$2@0Q2cF}D2 zhjmj-skLGj_(Vz39MvbxnRt2J9rhev{IDb0w#Hzx{W|ZSS20?QhC-*?O>B&ahmSoe z=o32#HqBDRWV{suH9xyt|CkM%`zO(vPnF0ceFkesgW#CUAWMx7;&PL9$)p5)Qn-it9N-w6x;` z<9FZ*>wiavxhI(lk>d%#^WBPnzRqTX1gya^*^gu2r2xlrF5=<;R7hrc9Q$@|1)Jb+ zL{+8l{58n*16g?3?lvwzrHGSf9%e;VO0lQO z2EB%}cqhCw;QJ9X@kim@ixoOTBNg2B2^yuzSr+#3*!ch(b7nk!5zL|hojHaT8+o*x8Vnb6tu-_T)M z3Un4pkfVMsjH7QkOi>8Of(CgIid_c!&e^yptR0A%70v$_h95;5ao5)-8N5KPw;T_U^b>*w??M;6cL~0>bS%GT2n#lbY^h5G&{pqr5 zaj2c44Bs<)G0Ea9)cf;s7rD~kPvVHi*Zt&MKsnW&Wrl$lWQa@NW0LyyA+BC_iX2_fWACgqpnqPT zB$hKOX@*`QaeY`%q-K9aJ7W{F%%`4|EB=L!qsBz|v?hs=u%eY}bx{4xi?$!TO~Q;F z=>;P`pu;cRsIY`AI{l90)ap(C+tz_679Xa0vkgeK`))cfHJ>^M8xrlkN2!dFE-krV zkA(r>8LP(%^!gz+x^q$kE;)LUtX?w$J7>XsHu^*MYpXYL;q^Gzi)s)C4oK`5Jg@zF(cIvRJDQ*!g38tsUDLb#5nC{Fley39(2o>7E=Px{FgwlHc;_VC+L0Ce_D!3y2aSse1ZAzM2jl zR^({8$WV`~F~msrCp7FYrDI!JDseuF@ZI7t>HSpdbVHadUYdzR-{+7uVz1zVz!aJp zmQB|#+)Qp*9;J)>oiQq7D+qcVq^rueOmKEA4PEmT{9il3-OZXL()cOeYhT4Q)OnKO z7a{b&kIz8+(ssW8^lt9gOPjcT#k=_mGbp~w{Ede5Y`CGtN4T!@WVlA6mgLxnZthz{ zLlS?%nm7yhAtZ)ees z`^&j9@$<+jemK9;U(eMf^Do_dE0oS%T#Ilhm~M8zN}apS_-XEWRL$Th*FD6Vqa`-R zFS-}Vm6MI+uJ|FrXL6jlX{yR}^tmSg)SYxvU8luY=}n}&_3zRq{VH;BfdbVIh~vhs zPoBpR7=rY#BwO9M18LXC;wgkJH@Ev>(Gun@}-l7x(rXQU27!ze(c!6h60A$TcXy ziht^$FiC32gjdfgX*#!npK$sIm!A>JHJ>+&Z}rz5|5h*P;b zp3hcCa5*QJl8pYNWX^Xz?tNb~QfL2(db|2_t?vvle#4baP-QvM9MI-2IpOG9=_|+Q zx!2Rz+w!=tBb`Xc{xE($QE&}fK1PRxp3}UpWV)668bjjV(fk5?eoSK$l`@Ft2BclW zh_ODtRK^yrxI_Y1O|>{p?pP}{R`a~p5nmhSL4qZQ@ zzzyu$&F{Er$KSE(0;?DCkVLlC)1jgQvOdO*`?>ZK_gKbVS~{yA^G22Fn6wA~Q+pB( zY@AM#ns?Cclc(_)Cm-i8fLGK-~Vufh>M2t^A>F&JI?*&I{e*D80n*YPzvRa4^6%MewhdV zl3WY~THNMmEl=WKDi(4*t-FNl)q0T%o-g7`E`80d={?PDGS22#>2XQht1$kd_J3T} z3I(cpJOSQK3*rVX>tjN7Z0YNLmHa(Yy`;*)jBiu_mfD!M^LM@KC12Z4@VD4+B47TD za{Y`}kk4m>_#ugBxd%L^-QAqz!RM$H!b7;jcb*!}_tF-3Ep^<--M2K2+N2k74-M3F ziwuLgj(79zW>)Z-~a1N|H$p~6vryK`&Wr- zPZy@G`=c;-nj5T&+`@G^D@!_pPqB$fGhuzxTGU)FgH?m(G^TbCuR#JXjBn-a(v1ce zP8)RWAHlH9RLndX3}cH`NS~lG1|1S5I=eox4oni0-fuv3zZ%opfga3T=HkD>T?!mo7&NUHv<@0TLiu5RH4N|no1sSK&J)MNa@fp z+<2RcGr3u>9rifV2Sn7Fn8{k)E&l#dcE z5Z1urIt@^H`xympD6tL+{NzJNJiC4(jT83n2x}Rs055Dmg2!PqVm;6cW$(<$tiUC# zLaIL9cs&;rgGZst?ozqka3QANm!NaDo6@O|)M#Rd0xpWGgA8sp6vSInua++0+>)jZ zit70L=|%h)D2dI}WKe3qJ`G=}MctFT+3-eXj%;8C+HDMn=&$;;CcvH5x0yoh^WsoA zz<{W02Scy70Zo+cg#pb>wk0(k?>>+tALlC2kDHU3jY+DQHH)8kOz{At(c92nauU3z zSfbEeHN0=GObyLyps-AqxZNLvXB#H5(VU;T?l_5`7d%9l(MB}(4}|sf6@INe#&ydd z=Jfpc31nKus9V%8Y?$7NCfg?9qrn-JFHj>#2F^g$hWpICqj}~02C~?Cx0q>NX^0-r zua~>ml`?vlrRm8^#6M;V29M4_5q}2fY^N5DSzQPJA+8iJ#H5sT_XETG5nF0kWY$neNS-!s&MyWu?|Un*?+wMu zopw}u_YJ7*yp8Lx{6O=I4Jd5V0KH`uaCcCGbSw!5i3UHsxXJ^C23A1FGbQq?V-^T5 z%*7n592_(>qe1($Ai(S?9^R}+J-9j~N9!|2(F``+ZxKjDm2yUshjGhN1$tq=9I0>k z%p_W~PM%bl! z0WT)_fo%j_f zlMO(L&;m&9{EMq144`rNC#odaQXe0A`m|q$IKHoj7cHiwR7jr0W$95t`7zX;{KQ_> z`HX+J`~jUeV)Xir0=(R1Nipgf`gSW39Y-aS6wgnzY9&do{I(12oNX5Bf)q+^eMC!J;P-~-K^pys(*D0oj2-(8Aij_xofwep%wa6~6o98h8&CEkYNghF_@-;nyp z@}o$Y2o(*wg17gd0j+acB*cU|8_?b1UGnA)M?u*d?ZD_y1d8GDQFjY^6u70r<=B%!QdVI_baE4HNjTCM0 z-GD)#ufn+tw&amz1FToP!6tPc2j4^++McXIwZ3J+KDlT3&ioGO$4%n8rG*%-p-ip6 zval>Vj+I(0NoLBthgZuVLHew6&i0NCII>S0KOcI)cm@^%U&aIueo-ZGD3!I%SEaMM zt8syuE|EQFNbckrQ{UB-;Jd#V^cJ?lFGihw*Qv(MH#bADNj%hztibri2jJGS#dy7? z6C^&)!<}PcXfods?(8@P62Rbto^-rlg{h9&-Zd>pj(1Zi4NGajtD zkFD#Bh<~daE|8fh8{3wTCB^on@u@W_3l*b|6$(__QJn5RzM*_1b~;A<`hq*(oW>vZ zVzAE?aigCuv`ZMF%ierU{8`61zg8lPYJQ^VaS=xCOep4Fw;_qtjLwd*pm`f?Nxw`l z#x_cT#xLF+3RlC+rF!^E`XembC&A1h*0hk@f^n?@C~Z{&3G)Z=N&YnQbG9CO2nk?g zt2DVWS`0;D>SXDzm(X%+9d_G&$FnC^vB58uNwuIq+IJe$q3!XI=wwaC^t^Cuk1`do zol0d6IzXM*I@Y481i!6QfZAgb&^jhdJR+q?clB%bYw8qQdC`E%37L`*i8Lr{NQG(F zw5WfYF?o2*jBMLyK*EdEs8n(|x=#MW`MbF<6OGFf;HZ|yn-Ne8rvKtc(`h(2S&gf&P=ZWYseG803H8JXN*2IHyLG}yok zdy_{owpfSgn+&qw_zdYf1xI$z`X0Ef&A^+Vn<3{!7<2A-7`W}W#x9{4oNH%F*Z7Fh zi~T8J1%-VqfCWS5A%G zGWd-z1l6dd%MF~ZHoz3-8Ij$sd6;wLGzf${;BFCZ%+e4h3G=0>V?z>ikTZ>(_*chv zos%Mxe|@l|M4kv}TG4AI4eYr0KQ=9UCuE9Wfb|m+#HsQXJ5(aI}bv>SEHUULZ>zR~um0yI>|qaRL{qYLH$#F1vh- z5Zx!V9HtlZ(e4|vv}M0FqZ=wq{(W4Co?COE`n)EYr~4fie?ru(xXN0+^~J_$dAisl zY5E0wG4lJ28Zp#9%lICg3Vdt|YG;U%@|%|skN?3N{GE&iUmK*@8_<&L1|*s}15<8I z;DcY97$l`lHg!wT)bwSTcXSV=Ueck`hwEVVAr_h<-hKzJo*S9wI`xX$ujg5CBQ zS;kzRh{0VA2Qkqbp#O^i`X+pVRWehVwUyyuf~g>$V}te?DR4Gkm|U1A%xt_T4ZDIG zIp4C!p}p-g_8oYDhpSS-DoUGf!5N(Rleci%$TwgjOsI6oTi~kNk%%)2Q1HuzEw-{H zP4};$pXzqDJ>n`B^*v;3e6*owXC)I~C5AB*_B6s%i7C01%awiG#?%x^kXOT2B5HBtYs3-W1a#xR}6$ z;(zc+H5vK(Z$bWHU8>yY#4Zl;gE+%Aa5PAjxOdqQ{`{$=PktD5BmTjMH|6YuASo)o zH4HDrU1y~;s&T$t8Bntmkdpn0N$IzQ#X?g^W_vK$Qyt2`Rgi|nUW3DHYZ>d90!V*s zPcvkU>4ZQB#K-l4S*-!h=jk!IHPX~uMV$CL_`GQDblebIXeIPYk1993Qiv%FdJ)X@&3pCAf_Y0T=n6n+Zr_CaMM)cB5g~R z`0eSTs4}K@kr?^A;2j>^G6e+B{DT6r0ZQ*_L2R-;DS!JJo?m^6^{a!JYd=dNEMp4M z*RZElvIYcaEJaB-8#+{`3k%e1@#^$v%zo+5_-VQrWhxJ`fs8-?N1LFtc-DA?iQD;|$-Q(Rsd}{IVz?N< z{5c3%b>lsiMr4H3;L$e^cI?+QI!PPw!pn=$d+88j&u{S2kR`?0jm&={TiDaDwxHKR z1y-eA5E5ofP(x`8JaAEkj@fm9ZS+Sx?qf)tGo8_F+Y4Ou<2G=6UNHvu#-Ya`5#%;m z(j9+4z{RyLm7a~mFytIJ5D$(qmQ-v_{tQYSq*^uUcc2sq{ z1+7yz!!_-SWXog>qaqOnOo0%s6%B!njp?kab}aPePr`&xD$Lms!1C>|WnJZZkQqLM z&a*zl6pL6ieDxkyL`S3G4HLAC4&WTuHlpt9`pRp@V%e8!XITCUIW#VjA)<-Gqr3-TmYlxlC;WFoYnI8DjJQCcq~H zOZs22FSfn8k8UYyq`>VGv?~kI1xG(ICyj-y1v%#b3Hs__MB^HFf#l>IvaXZGMwTl8Nv$v*yzJA2WIg0eeav_{8Crz?* zb%^&RZTkB6IM?8GGYlnM1L-+RWd7|m?n0Mg>^nIK!wU}MwnJRlw%-=RKPG`|gFf+H zYC`7XRO)hc5I+_n}a_9#Yr5#T5;T zKE1)yvPx=^FvnAhER39K`A@F=Vo z<{q}Cn?CyT;x z&Y-PGM6Q(vp(bI@5|u7DVNYMvlt}o&#=YZ zAm*y?II2C;qbFB{!z*_acprb1k?b9TM2kdR7;*~Iw>(52Q5Bf)Wr)d96R>un4sn{U zO&-q|VBQ~Z0pWIgn(Nby4cqx4Zu)6w5p5Z&Owuq=H5=cqwxe-ck<7jNQLy=b5oQSG z;k+3}v_V3FruMGnXi5KuwA72ZnXeYB^eVBjJc#|+XGWLc6nfdlmK@K{!w;7QFtOYf zUx-Yh+=Y|u&s*!kS|AQLc8zk4^46DU*xS?cbAn`Kx&je>H44)n??Gd0g!f6NG(y)7 zm1ilUY4J(iE^`@}u?(zwoyqP|T7r`krqtSKkbS#do!V(OfyBv5Xo)i=cMSb8b)F(4 z@k$!D+_NY6#gIh#aPfwy2EAZpKs#66!8Thpvfk1k9J4IRdvg)8WyF{>y74`ZRa=w) z9OTJ3rU1*+5q?)>h|cq1X6NQXSgnx=?c@O{ywagQ^bw94w86JF6)ZTaP4`@#N_IvB zaZidUkoIHJG-|-tObtaEoXM1^)4FB{R8nIp2F5GwrDCDjkYO0Pb;q6ssr_G9sq9dKuA8dz&Q=Vlx=Csy&} zcVhbJakUOJD#PGkZVE~oS;8_iTXOK{Ay^WAmQDI= z59*g1P*+2Z9#IgZFFx?RiOWh*DJ~7K-X!53IUDM%Qv(-wKSRBmAFR|PRq8sk9?eA) zK`YyY_VQ}P@xCZ|np=ibaxS5V=K?INyaH#AeMLjNSSI3a9HhD|W#Ov;D5*-&eYbDG zi04@_zFEwU+*YP1p2xsr@&#-bOt6PDa$vJf7*sxsK?S+_Y>bUPQGdABIZI29D$ZEO z#%dLT?z-LhHxV&cN0~H)R=|YlC~N#nkufz*V%0QFpuhbYM^^bQ8W@X!>FGpHzAJ|r z9*e*&VKZRQMg@A-Z#5KK)xd)d*VwF7DJpu%r2JD%@C=3hT9hlsMK5WT}8aJbWz?q%$+s(V;mF9{Pxp zeOv?Zk~xdJj-=uP`={`Gl?Y63R3k+z!@;kKcTVl{^huaJQCW2h#hP4EH$T!j^|2_5 zI?KWwFFhi+_%(`gH^8oA@^oZl4`Z*ZOx|e!ge)OVGCE!ju4((&x6%MHa}&`uQ=QD7 zQw$x)g(zRB7Fjl6#$8mw$F4A&gMJb-QF~O5SxE~pZ<7h-J6?`GDsrfOPmGLB6DDy3 zud!}d2Uq;hV(h(LiN`blv0UaTCV4-Bqvr~F8oq-m_pHZ1e>Y;`DH+ns-o>dVmTX6t zIT>vVggS#{Cl~$w@ZdB z{rL}XzAJ@C7mI;!ViK};Il!*@%xY*^%IJQRke$uU)hGFB*)?Y!jn*#hhenXXC8q3QQ$V z%=;r7;X!gSOhOzq>uV5&9S%5rZV&z`J^&4M^I^cM36CqBW7h2#ry{$Tqp|7*EV`{j z`iAsrNo5OW7}?N@R7cR^>3Qy~ozQvY54sNY;pd;FSh}>0d9lKX7{r(owqY&X;AKPO zo%W#`uLe{EH1O{7NT{M_-iSCjXKSS*A}s%qGr^1 z-v|WlxXAV#)uODEEGc_dg@G2Qpf7~yjVOvRtxFbzWm_~THi(f{Qy*+Edx}Guo7jmM zX;L71gxPoQ546t8D_>*k1LnIQb7W_xa(=p*)7LIK)U9?uv(>5)yzGWy;mSXpy~&qB zU{@l1bJwCNyq#{&S}7WCIhW({N zPHDY`iJ|{+T>B2D)=1OEzZ)?7<5ZgPT^?0qW8v~n-udr4T^4mZ6cu#xq34knss2^L z);X47xr!vcQN+_SGE3l?z7c#Ms{&`sSFF5fIXKPHqswo)!AY(ai7~dLWk0y6QPT&H zc6PHz{yf0p_IKcIgFu{x(+pc#fWd?Blxd`9~SVpGnNiqpy#__=9zf(&%Vm~ z{kRG_`|?rU-IRzoKfd(WGp zUw;#e;IB<$VdOQ<%mbH7~-LRevFK`U_mL_ag50lBRY} zYSgA!m&|)4N#(yiWcMd7K}ZP!rCe7yUn4=U#p;vNCHB;q6V1r{@?)3O8{>`pqp-hK zjksp8%p!UcP2)v~^PwJiB_#mbl5_Asr=Of1tMteg$y5dlWblQXI=w1Tg;FR$lE$BM zt2)t! zk40Z`jX^RPpHU#o-e^$Cd{uJlYZg2i_=cggEQt3*EjpJwjE-tWsGp<2HV6wq^NHm+ zHJ-PVn`W|CSBaCf4U*&~rx>DK8yVXHJGgUBlEz-S2<9gCwE0W|D$FOC`BjLpZ?m|E zKV{%*{t0Hay$ao5G!y&6=fS3zGueB}m)ZBT+M(yPI1PMOg7>FekyXla&~uvarf zhet8^qihiv_07b47ZNaYSp_3K)e0_Go?%zK{)x?_Z`e&1&rtjvg~{$gT(UxrXxYX< zlz=A9Ss_A*kR8?8V!~dL=*Hh=n)LS3XeK=(AA4U55JgWJD&?X=N_M^hi*q_8NNhQ_ zd}_zkumQgcv?KXhCD5v8L&jFmh8aIMv1=^l=yXRNYVZCVD8~-}Gk<*?AzM^OQNY`4h})M&_iu9hjw^X2q*p82bkyF!*;Al0(dB`-@8$yicFf zRWbOl;2jw7K3id{Pk*Mp1oiqB6!jD$t5Y|^hDrx$ekMz{-ZrML(LzM(OTRCtxb&MwG7XAd1x`An0nId+9r4AZ3@mNx8c@0m>azaOx4Lpf&)E``V` zF7Rg8Gh7q<9!;L;(VxS`<>|BjgURonY+|t$3DlLq{qP07y?-#L`WIkBSRE_7)rxGg zzs0T8)gb~~Dw%@(FkEwC-BQyxkFv-JOOE?VCc>-e_4VUQlb z>adaXX7vn^5#xh(4XW%tnF?5&Xhk+w^0=*IJeKIcgSegNos3LNWBd45LUzHS1xI6SaaL7G8U1v<3L`v{OumMeRy#^1SzQN9UAK1cI8*#6}R@Rl(rsJYU zv_3r<%+xd3bDQ7bt(Z9Ohv|lNyPXSs`j!jRFIbVX!-~{5DW9D@QUlq&hQ#9uA6@(= z2}F-1fTQ~x*n0Ul=2-BPj)GKl=FMZTT3ML4Mx6{-3}VWHN@#Ao$x0b-1A7N)B6?*O zz#J7C&$kwXN@U?7FZ10{`JM6HR)y-{WKlchG-wsbQ|*2In4)_Vl*R36lF@I7ikGC* zqMW%F8A=fM_5x%Vy~p=mRm`oPWKeY1ByVSk(bxY9@%Z3U*f&zfbzA%sd)KPb<6K=j zv0?%x4X>d4);}0nuS2IRZ(_QaG=tZ@9Q^hohRJy;!%cC{hH%9{SbAEV*)eKDaya~C z!45lW%>Rb{^7obV4wZda^5!d^8u){1G2fwc=T4-)&a8jnasa0eOjkV3tX=&I1n)E< z$zBRO-qdniWgmh^?01mWZ~;a%pHtGLN$(oQ0AJZV*m)`#`_PUm-c=x~qo!!uAqPvM z7T|E#N7xW$O_IKML&x5&7+Ep_{6p#&YQ!WbeIDx+W+9{_)K=fM44Wk$@A_iS*EGb znl69t0@Evv=po+!awRJ0NzJLhtpa(lM2YxmAPU`41?o@+>8e{nccPN@#A(zkO^}?d z)uGGPMKB@Tj!`+h7hmrTL2Z96nwBtynZGxKHEz%*cQ)?DR37&dq9x8O%esyuw&s|$ zGJ|#E|L(l<(l_>R_euy=P$sj}dYD-KU>J$%gkVK2xY=_H*7hUNeWrvs7}B2FK-j?u zvggDFur@vlcVxAJ&SgWgv0RZ_T&ae-jNcHdKEZGgi*ceCN)mmV%OO|nanJh|tj(hB z@Y3A@-nnbB#<81Wv4A`DU00xMcH2YOnRX~syv4k<3}LV3@bYwH9lCAiP4vET11Gm@ zk>AQz^z*f7=Jy&ans=0P1&6|5+HzB@FSR13p^DBT@um2u#2G^8u@I|YT6W~E25C3w zMV}*AKsEju%x^4&d4JEAmfiCLK0z6}Oh$%!irBMzPZhJPcZiVfeu9MZyyEpuH*vkD z46S-M_bK>AHhRLQXL@ewS$9YCM zJAHB))^6daQTyfT*qw2>uD1b?9=E0ex<=&o$CVrdw;??2Aw%~IsZq~z9s23H0l9eX zHx%^jX6)u_&>5Qm{iLQ*weTZQ?&^XqLmDXXWg|+`Y+%Q#;eM(PtX^HrKGlk5-|q9^ z`qhh2jqBD#d1?xdTyKQABTrE9T(YxzXCSO|Frw!2w_ty?1kF-@!kkHv#a(A#z{_t| zB-7FW47c6nR{Ip9>7Ov@alL~rQ3>#2AO-iwjetd)1Tma!hamNS=bIlgVbkzb>i$fD zJetDb(?7v*%z1>R`u_M;Fb9W1KSAhN4eI|l$!<Icj#At^&vImaL zfUbZqTn%Z;>P{R6{<(@&a!8%V6my(+1g=AWg+i*A(nX&UL1qZnZ%xjbdfw*NbWqKy= zKHCYz*o@}>JBuHLT5#v*R^Q4ppxeU&tSXYa@#Odxzq%O&nd&fy?;@- zx0aJ!z8xpEotUASyyN1q`LId4Fw#EMX7D;bPU=EBD0E$F>e zk93q=!<6ny_AyV7wvA+B+D&V+(m;&Vx~^bmPuSz;trr;U~7PegNB9yQU- zWB>AU;ji2qFd9x^Y;s4i&@&$H31%`o{u}^9PkZEhlY>=Pl!#c_0eC)m5fhi3X3lQW z#;11zSrB>(sfuz`t(nK&<_$ygoGCM;RpL?ex*~kNK8ROLKSM)c1+H8x0okgyRA|F( z=ujGgd=EZmzk2b0Cefh!}=W)U^wdq%QyO- znK|48F879cy}1&8H~(doy!B{uodsQ#j9;AyQVw7<9%RhwO^0Tn!` z775oYp5m%&N;LUBj|;~(tWyfd!}a#qndQk;)&=3_g=bJ^!U0QpGjj6zYDidk2g7)* z=|-n8j@GYjP`&p6S7opX+FaF$|McaMS-%%@BpTtZ^kLN6&j;VvC=fB9+xTMVMy56L zJZ4!-(l1pNFqk1nBrjFtul%!cQ_Keh)}IGykB4x2(u6$QEKV9W>(G|3=P`Be1Ms{3 z9QRwufQCadTD_13x6A^Nxa13ECk08A`yF;!WDQ8D?1U#e#4Y+A_IBV`P&RUuolGd0r zcypH(xgynpX_g5nVk|~CuM;Ng#(tsJy9Ce|4@GNVdvbYpJ>zDlO)qkOGxc%>%o^Sd ze7IO1ii^$3!VQbzs9!+&G7S|>-su7}7Np~tn=NTLszv(rvO(lc7|Q-Kr3ZI5pe(;N zS#NAazqOjvT@{PjsP$Xn@icq-UDkw(4;^)0*>!-&HT{L(J&RyMSDT(SHlPR7e?k9d zK3xCbRg_lP3@5#0uxsjV*mKnhUOq^{N6PZ_>fdUZ7t)5m&C=LmzB;DA9=ZF>C!6n5B~*T(|>|{Mg>EUYxB6_4m|v5KB%q^f@@DqNU2*X3^@fud~7ZE z-@8#5s0&8v9dY2WFAuM~zb+3RZ2^NRw|KdaKC*WexX;EP;goxsFqk|H>&Owz|K0~l;?F%+-We31sVPdDT98c|R#(AF8=!IjEpqO6R_z;bJV<9tw`K7wSiSp;6SmGDp}(oPS)?c$%y?5$NimqK;uad zM3F%5(b_uLVWdPPe^p|*qAni(txvlgjd(MdpYWwfVUkiGBjU3SS~LQ=tGQBSM5Kwi z@y&*o9OW^=YwW1?gKwO-Mb$7%rh=`g)F;Q?1@NA@5`=OL$oU!f!Q@#fH`m*eO!az= z6>VW4nehSMn!QIwiD~rRm2D8THUbR`OYn`b9EdFuq|Pt&N#VcKtbNZmoVHw*{7sn; z--pFPS>rbPhM5z-aBrLztwP-Tma}Ck#qi+5KNzkNp^p~+KtDb^`Y&35T*%WTI~Rvw z?vlG`v3-aU8mI#Aa$CskQ^BkAKH}Gx8bmH&0G&Qt(pBeUP~2Azgkr-$W+DpCl)uFP zh9a?gA`MlS=3@nqwd-?I0-?C2cz|O`qqjC8-PMMnW+@n6)QJlX{;(8J!+)m~=rOIs z?9+{lag$mZmORwOV}42KI=!6Zlq*0j7mLFbJ0p_zPnkSidL6wjH$axU89lUJjZ|&1 zr^k8qG;vIy7HjbCPfe*|$l;l|&dUNm+1s&h?T5KD>(5~R(^i}wtx6x>E`jo#a+G&h zMD17k5V6b-Gh!U^_nIHLx=VwSn%l5%y#cn|>H)qq!aVZZRhG5efS!7Il9PJF0D}^r zqfNXquG^`~sB96VU!Hwu4u1Lr>JqZh{luKi`;i5-N``(ubpekNOLVxh8oC{Ca<}Km z6H(_ZX3xA{yjdQCok=Qm&aJt4Z}Ki!RdhOkyJ}9$u4~Z%*I>BVr9%TY$jx?ywN9v3-!gP=zsmn!uQ=Fsyiv6tyif zMBDl1q{D-cZa($_YtrnfVVV_FzQ~m7?&Epp^+qJ|M*__2egN9P#+algby^ybg(Xo6 zG_Ru*FUAMJ&Zc?j)n`p&ehSj&qZ-r$8M?F4X#tvJ^I15JKOkqCbQvZUS&+>{ha#htJqXMVe+Fxf+_A1#Dc@M=q~gM6T6?W+m1FPZTg05 zA}ZLnyOXTJcpUJ6vN}Lg7-uwPW8FO9{XcLJiN-7_m|D-{q06XMtOug zKCc+TwgP*cHHqk1caHzqLooXB6>37xp;*fy=8@F^TAHeoTYeR2bL2bhJLL2~{+y?M zC$WBN5m)pw7up>AxE0Pu+-189V9^3m>T4#Culq8fH2N-N>za^Zjx2UZ^EiIJdpIaH z%zmgS#!f>K;y?Agv(3CXSYq%HINwBIo7NSE`MZpDU8zp*Ed9jH+Nq1(Mru?m(}d)0 zy25%D6k*8>Yid?*MC02dpm6q0RN0(@+@miMbri@NSxcnXI3%O59lw{A6 zz%Lst$l@=haEIpwWQK@Q2*~^NUOg9AH~nY=+2)Yya!R0V}gq` zZ*uo4DUf7 zdJZb8L%~Nzjx23GjQ5+Skf78uuzvgl`qlZ#?;b^Jc##G9L=!Tx@*mc54zuRN@9^s@ zODu1A#dv2`f~ezCoV&w}zBpD5o3x*T#wkD(8+nMI?3Gbk;adk zY%zZK4gAsl4C9{{!36&*l*_-3rw&aa-VuF_aEbu28=Xpm`080(!Pm^0W@EBV?KkSH zuR%eD=ggH8v)He7E>P}6L1s7}mxXR*mev}ORqyIi`eBmt0(rs};+XACl1lrM0%oalm$RGp|ucB8uFQ>Y7Yg{tmV+yR3L{P%bz zO!^8E*Z(x>21{cqIbuV7y^O(MdNFE!{ts&|n2?Y8({bCUUfk_|h6yRI!}CVVaV}Iq zqn0sn_WwffmF7hI$4_W`I*MucQlYjc9{%<4Q)d4^+{3%eWH37&Mm_&BI{Zm6Tj&~g zXz_CYPo+%pemUyk;)s0JrLfR(3N%Hwv#-AY#iNd07^kmA&iKAS)46=a!utT)Ug>jQ zJI9*pZ`g&0FRPKB5PixwtpplZsnJxYCT6qFRI1`BOTX(Y&>@AR=w(sMe7G+}Kc04i zwbE}uC03E_n5Rn~cz=WUjw`VDP$;|5RF~{gF9L007u=95M$XvkQ1+_^QH*&43u=Yv zybML!7%fc(<7`Pj9%Ix4&ZE20b9lG>23+D6vFpCd5h-3@TN`W-@_%?daI`5ENXW)G zdX@3+p9R~;YuV7kaBzw9h7${osjly4a6Gn)y{7*gy^dFN7$sYL;`#tD@jN;0eX3-0 z`&;ZdRmE@x<%ykN2`f1NEB5PJ(c;wa%~20^tYtVY8xxO40-PFr&i-x+b)?Yix_QzQ~RQ$tWaA|51l8o_pZ z6DF5uBt1Gd^wkXC6-Fgbg@ z;OfE-Ho8ol9OmT;f(xtI@t=>-U`U4?T&7MA$=Z^#C?hh@HyXYtOVBHEBPbeg0c%qA z=tjFZB`wn=T8en*g)&PrmNza*l*2!eU#LFT4ZI*uGRh3q{)@Ee$r``J3g zaqJc>I9mbde%R4sm&f>@wIyA9k3}UVUe-i}iO8~QNN{KYZfP0@{SqfKZZFx)Bn}RJ ztAdUPgOH^mK!;bDleMN7*~T^(*f`Uk?srOO>MRQJQ+_vUnC=Xl}K&waQ( znV-b(SOvoYCUi-UD)@KUz`8fh9OWh}cz^r=Te0FQn05Taj`dg2V40PkdhV3-s6+(O;$w8XlSXFkTg^@RquOVi3ZV-J<`%pgcgbX?)R_w z^#`=?eZ9^(kLTk!%x%StYuwp#!i;2|HKA*guA3+6W$&~)_I07|qobgMAI<_^k;Qiz;qb>3t2NlzCqTvVj$0r9oG+P5d z@BfL3hQj3C13onib+c!Z?~EXhjCwV?Vb3E8I~VA2A8YOuT$4Vv2V(hbUNat~*0 z^R=i$jtvnovoC$IxB@j+pXZ5exq_Q2J)lVbG@RNr4|l(v$OLSC4U>-cVfO(8+@1Q8 zJs0u~k9KZj9)Ehl>zzD-4uvX1d$kq$5_uo&HdNwBz6tsB;TPUhD#8C>_r?F*8~?u_ z%komdL`Z?0rB+1#jTZT|q6V$+nt;1c2g<(=@Ptlik-YrNkh7>B z^0eQ#&_ z8GE`&d;;&&F?;xL(RP(>RVrpRGe znYCVvPFjDNIqoqGcJVTlh~46+%t?o1YEf`=UKX=#c@#`v_XqPURmmOuQ@mMXyaX z9C45X^`>Gh|M7=Wjx!}ccJwnp{R)u%R>9MiGNsErdvNfC7`7$0;bbojA`&J8Qj=uJ zc&jlA_&sJ#(POlr?_BQlBSNNq3eLbgwW4h%=(hXzTGbFTnF z{yH|{S|Zr|?~Zk^3K9<~QqN~=!C{Xjy{p0TMZIUSQZ0jxC&E)`A|715e-baHK`qaxtggU?@ZU!Ps`+UPuZch#DirD~@CmNAMrH*97+8`8o z6oyl*s+j;|j)&~4gpCV@@M@GAy_6+E%@2>_O<@m6d?-bzzdU)f&I4b1HQ~WK*RbCE zC)Ve>;^AXPlmt~k`&tX?IH?Wajtsy7KXLk9!x|39iIdMe9f*qEJy^1EI+nK8gN9W( zUwg({F5{!>`gp4$$W{!%`Cpfz84pnjSLfWjt%Hp~&BI z>=L>NwF=7M#tbtL>22(qoes}h%n6$3F@^OpFr`tK{&0N_7Uzz_6RlJT9E}5JAHlx2 z)~u#F$21)fAsP1tso*tvGHT{T+V)ary{jyhTqy%C%mh4VDn!!ytSOn}0+nsz{QZKP z7(?k%sPERmFmg|c75WZ=9pbzN&m zE_R&d?q7EyV3`59Kl%!5wpx%@_8@yVM439cn3A*tb)1!$i?^CTLXC7SG`3YRFX|_; zZfB(M;BOOhJN5%?zVaGen!Dgz0OAB&HBd^>rtV9cAZUjpUHaLHI(&1Wy>2a#SQN)7 zJ#2FM>)XY|2%kfT)|L3Ac@TfUu)~Pzk2q;dJa3(=94Smwq4T8jVg5}q>LU<`Z{lP@ z#%7e!+^0;o<(@A|>NtmU%3`tqc_y!hFHVK}WvO*{BJQe-;dKQLVNYf~PMIo5E_#}g zqxBo`<~%KW`O0}r(T#%E@vTsLa2>AOp+|E{mgD*^1C%?~2v%ncaJ%PQsA_-2%q<#V z4Hg;D?qUaWOh*tGKIwpM?tBDKZr1CiK()@7;*t3OusHB3D>AA|mJFCu<$^*;^j0T+ z2C=TbkJ_-ry_9#d#uBvbu0gB%D!h2~FeW|C!W6F0|KGe5u=kw=J^LjG$9MOa?ciz! zzd?NxktIxf#@fp2$3>EFHFtYx(>$+;c`JtHz? zguaL8GtZ-(pbk}wgy}4RhOTA{@2;i67Rw;}t_w8t*MfZ9oF~lgimmv9d&W_bAiV z(92FgdX2B|Pz#>ixp!riIL>raMAcu4^yB7Q_S8}{$|OF(1$(+M>Wvr)yKhRzw%U-_ zGz981hj9Ft3b_-mhw2>fgj$bT-!J)4D_}?GNiSy$=10RZ^)Qs{Jk6MW3PaUT19t{GNVw!x878G1u? z4XliaMJu~Tw9UTDWw+L$m2o|;-Tso1yP@mQ1yUv>emeES-rwuIx-$tv{E`%DO!brwX{ z=>fwzWPBHAGr>C#!rZgb&^woNgV<`Z)8QR6_~AWU=Acf$uQ|tDcWVaYs}(p zm;mp00^E6e0Rxqk=~?zHUK+j2EBtj6^CuaQ(0kn&|9TLgZas-D{~3}Q7t-+PAu}TK zCkOUPeqigT@}XQO8QV?Gp ztB0ab$O6~RL_xP;>*0$}fl zJ~$cr6IR*2fr5RtoQrA({Qg)5>f3%{y-zPXbHCNCtJ|36MtXF9i!klb? z3z$VvEDLAi$&1hrMXK~;eiMZLF2w0KUc;XU`@wnSE;`81z>1(K z{8*9#JM1nYmK}gKpXW0+kqUGmrUM(t|gs* z?I7fxmnZIa@eqC8g8rN_2lu=xU?uVp#Q%ljInPE$&+Y^|%674buKvdl=xE^1<}tL| ztwOG=sS{jthwpVqn`9rCCdpEt@bCKw^vwE&JsMj{o4GQD(aMhTd zD?rDrOOQQmLli{s;sKrqDp;q&I&o>LOyA>TuCx2U8T`U#KI*LEvyYb7vz?1Yz>I$e zgrlWt`$=O`uW3cQcn2VA-dvp65s$Nc%*mDAHgr?HJ7e^>8_vBjAj>2*X#SC0re58V z%=fmZ0q^bL@$3KP-usw&0|DiH;_8nhQD@OJv-%zE#BdBbGK>|PqKRK;E76n# z_E39};CFv1a;x|UlO&}~m5L?l&RiC!a!f^L`DZ+kl7{#83xkEg81~M}$Jr*v#CJjh8q;|uFD$q- z<}eJD#7W@Wi%e)s0E}~U%m#@o_+xuCKXBwSE27m6zfP|}Np2QbCnbYt8!tk7 z`#8RTI?S9pYDa~`x7KlCapGe7m0jABW_Lv#~te zs2+^FD`g-{wjaV8TUfbqDf&cb7nsC2TK zU1>m{zgr9@&;O$Kg)bPk)*alqnfMeTcXsz`Lvl!Hob_Ax0WL0Ik0Jb^GH+s$c4k}-Mu zwhN}Wnvxld!ePz3eXu=(hfb?waQ*2BFw;2-!cX~pzolKw-?GEd{?&#qRMDrA_ore1 zB`&{x{VrrzTTxfP<@jDHfmM3815T?e6Vr{yc_!U5^iQ!pV_1IypSU$+T*xCBi+I4c zD&B^NgEFLeaX2&7`vm6y$-+SOTDZBjoA)mEI}|Edlg54ty6cT4>9dg{FPgafv6~Ns zu2G_U+P1>HA1zEmofY-iaT=;SX5p1xPQ*CJj_ex_fUleb;PZnZP@G{&BnyLCI^+PY zlfJRbu6Xgqcdoz%9ueH!=oxsgx&m{brQq^VJ$hU;56VuP(%BXQEL)*P3!~J?n*WqY zf2A@lnLCm8%D%$jIuH0OU4)sttg!Xm6ny?n1eB)UM(xa-Fd8aI+qzG(POKXF*;9kp z+N6lf4i?%v{4rYPD0l7*A<^__rfbdvea?sRDy|lpt~&gaSBT2X>}dLq32^hZ3awo` z9fv}%SaAwinHM2@sx7a*@gmZMt0 zYestUU1mak4ZHmJ0Ct$G5GnI~xZ|!ye|mAufUy<%VbIN-U15r%Ylgu<*c&uQPJw!o zJQ0xHz?vMlBERR4dOF!sk6jH+$lk@6p4NnqoFqVPmL2)AT$LnltA~SUCFsIy z_3&UCAC|}NfxIGq|2>m>%stUBlkw+@rH5QRvyJN@nZH z5hlwXKb#*y!?xFWy!QZ97DnLV$5UMOG`pE9+6tndAA|4pdib=Xj~BA>DQF+h#Io*4 zJnGfSmXaovIBY&59~Oa49#EkX^6s0*jUZY zwpTxa89UU;q=Ugs?7jVXNQjR+A6&zuhegpII;{R?L(iOlgtA3Du_z`P6T}ku`z1w)gt{)|Ju{;H+sp`5C=&d=5k@BE zLEn-P5abxua7`QfWNjP%?7qi#^Cj^PmxXALx29!NrKmw)D|;hTgAlRr&{^aSMJqpI zWZEU{KXrl0du&F}P7z^8KnZvmMkI&()RXVnlc_-~n0?k`_)UfLuzlAi*Cphsmq-)7 z(8__Q4?p5m%Ml2`Y_u&~ht2O7V0pe334XU99iPZxmi&9zS;)B-o!6FqXp*MIN@nz) zw+Sv(y~a#fqeKAH|nOsizMEraBe7^~Y6}`dYjX25i|Hm}seS^G^RxCd-4k0G@ z`G>X`Qt1L7&L7t!X@kd*HvVRpyokh-x0Sf5<2Tcr!uh3`+swpN2e#w98FQ72(~b&? z6Z#bx`MU} z&T|5^I^cL2&rz79`F)4)a{A=)zk{G~?jkI=nhwuwM92g#KkS_D3ny+K!<@-?Fj&M9 zH`ioC+=MD@XmW?fwYo&sLxXIdB})Hom=7jpU*PC=V;a7p8z#rLVO6^jopdMx-&uuX zL~bnS$?yhww~KJ9@FKX6j6$|TC#ttwQW0lGI{!~Ko=Zr>Ockzg`%j48UoJv5+biJu zsxEYI;BxC?Qqt+W)Mzfp5Z5D~de{leIo8IvSe>5U76Qg6hu}rDFvLY7y z@?rR+5HVNNq!XO&sp(ZQQa5*;O=7uv;CvBeD!VY6Z_4?H^j^cSBn7hM(QK5ss7b$^ zsYd@*oSXLTQAXvTCi&-e994rY>9hCiV3tQBX6=204~yEsb?a>AzV{(k#>Yn7UeOc4Fvq2 z?*=7V8N9Kd^Lf2*!IF$Fo~SX`%M&bQl^blTnriIX0}*|*wskFN-Mp4`hnR3c5H1Z2sd*|yZWLX`@|j zs0(_FVHdse>Xw`6Iz^xONIAlfBnkR7_aB4}bh0~^c`&+b`OFK0O!o2`OCt6>3I2W3 zg3T$X(K_7}eCmYh^`%F!C=l5+Nz{uFII|TK4pZ zVmy1uh}`7z^tx5NvQmzlHX7C?=ZqS`PqPe)eIj6lY7a|Cm3rV^&ti0>suS9NW22Cq!{+!DS14;U^Z?vFJ)if z^T4)_H<*5i^NP&0 z5~EY}v}mHwSGHGtBK_<50Y%Dv8O_LvDdbo~ZgSlNST?-s%P zrf?W;J^>nsGr(|g5{{XRlHB=KAaFJr$0bC`{lh`fy1R-uL)?ZJ9j8S~99W*}%@feC zAwm{}X*1d!pR!=yEA%)X#HFZ-|Dvd;!pIdxo0J@8i*@t|0l}Nme&}F3z@=qDQ?n32VL% zc%GCKPK7eh8XrK);&&*#u$%v-T916JZDq&)YSP!Eud!rbD`;j=NNyhmraKVt`edMx zq&{8u+=Cei(ngj0mZVH073*ZWu#fZCJy1J?{Jm#ch;<@S4Q&v-&4~)#k!H@!E#{?2 z+7qdh+H^t7M^GrYr+S0-G$pna6%DQF5^D$cmGuOY-Rg_I&=uOS&K1G^TY&;Do#!bkrhZDh4iK~kYTHs4( zA4q0YL(z~d9ZpyX4HZA}{MRxj)4m#HME9bx9QR(gG$60{bLZWV5UqZ)hxhG`44EP; z1W)pH$imo#%-P%NxPICfxShNUPIT13t3_|{!!$|Sbd+PC=N|{nT_VKjiz+R;d<)$! z-Q#ykvM6BJ4vm%aq_0z(#I1b7_^&sn58t1{ePd?udCLr3mz)SDC*ANvj0!2Q(jFdxI2ot8oBR(9_yYMkQb>+73t0XakBKE znGl4&{0TP0vFw(K+~&I(VH^!8n|k#y@58SYlyF7cMoSFQpS9#k>=DorytP`%EyeYr*k%!EnZ1pUw4G zfit}~QD|J4gvl$@*71o{f{`Ws%R~6S(wc0nzXnIjF2Hi%0`{A|1nj)y0vAaZ)`KV+ z=k6(`UKUL6l?5PCd52w56@(8$rD>S*1q@qZP3sCb;`*%!MmKDUa+(S4KEnNtx2TX? zv4NOcbHwF8%MK7-s!#T7odffgj-Zx(fe*hg@uXFvz|g~j1~9f%LU#xh9qg%!8wEpg zVZx|<2ZQ&kK)&-DbHgDW{$pQ*>3_vMhiO7|FKL6g#n+jqs_$Stc>q8Eb3!(%2NgR^ z7>^ek#PpgZgtl<5EdfyyAu@_R53R|SkNV8TNHy|pk_@bM`^hR~YY->SS&-3xir4wB zmi723#F*j&%q)>7{TtNSS@E%u+In;^Kk&xzJ3a()L<&bfccp2{Yv(yJdP z@|Q2RB*eQHGZw_dWVmO!Y;^TkiIB>&^ya^eBcRg2d zSDZb@URVX)OekdT$-Eb$<96FZpDEh*&#+;=^P#R^mLAYS&rz z(GQHzt7m`m(@|i;5gZm#p?-m!@5n=sC={tv&m-+Hmbw^%8=}y2jSt$tA7MTuX@JWe z4Knu2irh0ZV_vvhkXoHrSZz}2s&Yz&w(iJ*+^q^Qb^d!S*OQ{k)$;fbLs9-*G%n20 zp$fOVa12yQo_s#OTNMof)7_bl%qn(K+jhvDFF_1b)8YnVIvH`5S)9(hSu7(HiG;-RfiW1tVK zAIi|f8rEbv-xfL^3(?s-b?7m3F>;pk7@SmkkKY%%z|04ZC|CLl_Hli$a<`u#LbL)wA8JS&^lOU|L5t}o)vsIbn54k&V(v%D|2-PC0qH=inVKDN1IJZ;tMOJRl zFbafUVU%Jr@F!P)J>BsM^XDH0=HWi3lplq&c&9)%a|7-zmZs|2hoSY-L0lZM1haL7 z$N+b5*0QmuOQ&)BUD^$Z#W^85`G^ypw?LMv^QNJLvk?0|+yi{`tf0Yz>-R2{rF&g% zh|4!EV!ZJ#Ofl7kJ?6E@rY~j=*$C3m^`<20p*~eq)Fe}PSdd*a?;xHvpuu|r;cveb zeJpH7=jgn{-TnW7^tEH{r$`Kc9);=|0;JaCA~*y$fP}j^?w{}q4;&4GCyG%J|3Zay zTs5cl&qe6`kgu?M4v%|Ze24WKQ?MY=iTpao`DYC-Voz5to@5Aw+zRK*AG(7hvJX&e zbRRer=fltctjOEyc+6&Bq6_m0o2Cxqq(%o)BAdtcfU=O*i4dzndD4+(LZ@C&VK2oj zhoM8w4F7o@4z+N-)~iDFvh5x?9hQMv@m!sJ$bkfOrono#Wq4Gjn@P<$iAtIMIBlvK zJ^rr`)GW*>dyez%$*2(_p--5nwGW*Im$*hhaG`G`CMijLhxm zfz3L0q?pStd*34{RA%M1ZwS}Wx^(xz$bbiYMzj$k^%j= zrSB(p=ue=&McvHIcPH6-RfaU`Js)N+Z-vY%4H6pqm1iR3!yfzo0#1)Cw9Q-bThikN*q2$^|IS``@&%`r70={ z%yZ5o9UxcF#CE*}n*)Dg!=i6&^HY5?veJS*u%;0QreuO|`)}UjCOP`uwhQP!8*W=j zHB3wBVhca2QCSr&!ogK&Y50!Ot~H{)zAbQajUwD!Y(;kG-GS6LD>_G7o*ed^K+j0z zVC>g27?-R<4=(!VZx+imTg_tvQY$&8|1UI|H!|P-&qC(a3?@#q5WgygqWAcBur}gy z^3#rBcYz7ryZa4_CO=?&hKv!s_CQL%B>6s99WM1*lEn8zr5>p^%FHK> z7&8wFV+znQ&WU~8C4>)kI48jwK{AOi&0Y>EMVGe?_}efMog{V08&f4R>9;&6a4tw- z{GdHb3=eFpg$q_JqT8c+{M4)a%8?#FEfABZ1&ctr<^ZMm^80(BpOreabCDSb@@7k z&n{JB49D5ONDo3K`HLu;kybYM6b)V zo^>@FXDvhX_od>*X;{O>@hgH|0g7@PsI}XwxMP-&yc)WIxMaz`4-`R(_VEe>!c+l_EuS=G9`? zrfxQEi9A(E)uvq+7s5c%ESxA?iM|~3+~Oxo#Ma$lX89N5kWdxmtPDW$%jsMmQ5OFW z*rH*D4VgPAh>3p;>6xL;_^IIxlwM2%_2vQ2srwIeMd#xyCl_YfFJ&5e`xf|jisQn> zCh)9V3QN?wS;xE}Ry}qNoHS%HpooXR?xu0h5Ca-xs(|99s<6DM4CimU4f~cBfX>h} z_G9`M)ONW8Tg{D_nZLL*Hlq<@9399^j+>7yuECA+6Ny+`99mmR((!A8#Av}k{IkK7 zglw@VvwQ|o?BO@CRg%JOGZ{=Me9zp+P&mG21pQ9ilfmr2U~B#i8-}ysK$rmB9;t@j zj5Xazkxvir!NIUkAbR<}E8kKcb1Ooi zeYZFrl(!_Ze+rlt)44v2+%wz~lM8D!%|Py1K3ocNpxF_d${dSdL&m=vNcGZT&W$?I zS_fUi$X@{A?P0Ld=^fnu^^)`Qq_Prq-{88~2-Gfjgk#5~>5m_O;L4gj99gSL>;v0y zW1I&YId%c>@T$@8*-o_c{R|<_yFflNn628NNDoIo!la}+II5e8(_*FR!%Njztg{#U zJFcN|d^xmdN%6ItEr{|>Q(9P4gjVNwL)))r(7hdlNzWo7epe0Jy1&Q1-&G*7+K$yb zEJAB#TKQMJ^+>+A2^{QJKy6E5bgq1jZ@<++)Sjt4NeMpge{ROq*RO?a&tV>Z+|C+U zSWummT)uy*AM%zd^FM9+h|?27L1tnr-W~PFj6Ju|d3rIah~vNsP*zYP5fuk9ytW104z;qwFBY;l%QQjZtPS~H%yXTV znaBndgmJsrKB3b{0}S(2@qG6_6tUHT{gZz(YRY|VsHYm)a<&<={p}$8)NQ;l7Kl;eXEh zhag46n`P;l2@d4PrYk&o#T#I=-kNUt{G3(3C`*lWj<7A$3`oUVVbW0>oUUQASsp z8lTI@1vd^s<+@jBB)~aKq=T@s$c${Wwnh3u05f;LgyJWEp(Rggz%#iJ%HDD{xve6#d2Nj+H6tO96+_*Th!)Q;=E)tSm$WZ(?5C=t=~V!gYi?5aNO0`76mGJC<13}`VV9KOi0HGJ0_uV z1iltUusg!q*@y>q&^zY>MlE;^$5!Z)fX&emxwi$LddkpWf@b8y{C8NJ^91IKZ-9X> ziy-yain~ zx!%h|Wu|#S4XTzN<3CH2q&tSS=;s;ftbB47b8xy4Sr&5$`yH)_?p{Iq-|ZuC`kf(_ z*=t1dZ-!#A>SK6tMw;0jcL}4eTA}^45d65l88%Gfm}d_w;x)wzUwN*;e`Q7Z*nAfI zSJxIkZ`{fHefx|vQhgbJ&1}Bp69xL_`%Fxd`O4qZU`w}ce9TC{m<3MdA?$XIN9@!W z=XqNroWbf~4tCa@WtV(vfcAwx_{Tk{^xY0KYEmmnZ!f>V7T6xa+X8lEpkEUs<<#i% zm0Uhaq!%^0y2({Knyux$AqGMAv|ze1x$G^(e|@0{?{Kx+V2LpNZ-2{eCo%HgzZw>G zX_FQkj+vSF6167G#L}u}Fv{+R4>tDXRAMap*fcUWS(W^flZ~)=Z65w;H6kLbmZ1B~ z0%(twAXlC^QN_Wys#Ey=)|k(P!q$M0n*_ zfQlrx{O&DgYC|4VK5RpZe?Nm&0YZSbr+7xz{_wI)lwO=&32|ES@a$GPv&pClj8moY z#C{WcEkl*=%(5V-UU4k2sR%7Gv?obl|KS!bB_hEXkg09=*r|<8;B--nj4iaFhaXko zp_MH2Kelv|(NBCTr$VlPwusX2$BI3X{QD zUDCQO7;bu-(c4~Hr23Z}(Ue~f9}e$i=dT&XMc4E|=loHe%RSfI4mi;{y4vve*nYm) z#6g!9;X;;kHo%6RT6Fp=Eznl_ff<|^#l7nQyQS(L6KwVr)Q2KC_qHOpJ1zrj!<0zq zawR$`t`kn(4#Y#;e%IHYF|f`v5bHOLqMPgi@K1dQe>=}|e*MS%UH*y8WWjj&S^gg^ z-CKaZ$An19s|vR0c{sN2Ux99m?lWPF`M6S1gg$M543|rk$jn|xx>{C&@G4&5NpVZi z2r(m@|7*ty?_yB2z#F0uw4-LvMYJzhqa%WujEWRTl6-Xo_htcvcuQj8B};Y>RbrS% z1KiOy#l5HfLCiOq@jh@Ba?MN7H82-f1=M4)e=knbPQs=2*3hm!AJ#ib0xc3FH?kzj zIWBj+*QX86Wfzq_bZ~}<`NeqKW;)L>b|rI5K$(6vQzm-@^szZainx#5hJdt6OmOjm z?PpU^%RV37-);t3nLH4#t;aOUdni*RLHgB#@#pve_IPU0k(tk6v7Zp}+v`Y7Ka10# zj2x!qr3?hRM=)+S6JXh7Gx}VF^S@|uURTRt#%S|O=9NGLdz0J69bC|i&5)46|`@P4b<&TA<1iak)F5es=;_M%;s^1NuaCjtUXBx{FVLCF7&g z2=McL0D{$ORL0~c2B&A?^`VQbbLa^i`zlSH4F*~N&1PugyM~!CehycSXJBB80ny6v z#$&0k;dtE_eD@;>pFbD(&e5YirEA!lSOn?9H0INyG$?$jPQI>SVNZlA zKJ7CmN%@c9+fp%Z7hfY2b~P7XFk9fl3>#t*_Z4%#JYgWvmh{PVpxxgGXzM);do~us z{{3=f+KDkXD)AK<#%R%L*oegzJrHxx*M6D?~H>`7q;aE4*EL50kiD@#M;#aF6pMyT#|i>A#LdB+&&IB*+k_ zQ)|(#+niqEn4~iS`|#joOESke9y*k@$*!aKAgjcfpYQw)ubwfc_6PQI=V1yi?Bjgb zpKS2H*d5kzi7;$QP$o@6v8>2@gg;fIc$n*}2itsPwOi7l;c+|67CZyd4w7Vd%L^=j zZU`Dls)VP~iZxu1t@io^>Up#eT*_H!HP9t~+)l(xZhGv3({EAfmkBWleZ+rm%b<)S zFdoxcn45YJ(>brgUJpAQzPSPp=U-yPu1>?nzAO`da2tDnh77sQzQB%18!{yI!L`wA z9Qq${_g+(uCwwEw%}$TwtkZ*Nx#ueAewBbLYQ>nWk%j&we%q}5?KsWlPg zPa(I%Oq!@)t^w&w0kGY{nq6^y1deo9!lvn*)8p}3RFA&SldG}9uUnquvJzV$d@VwB zUcy~HFZ?lF4`&*du)Yhqzn_jH=Wm@3-tYD3e$ERqLrRFesgWXIf_GrEQy?=n4B#;w zMzhQ25 zUZ=-g)`fe123rh?z^E()>^TO~Tz`7~&n8gou;=;jWr@Ne4Ll(0Cy+KlN@1F z`diwGNL6szWfgf0$?1ieE|+o2;bIK-GshW0VVo=NH-B}HF}bxM38vVX(8&vgXycSk z%!x0}{KHbK&{cmY=zYHofyPbPwpRfIQcl4$`_u5U{}g*dq!>1@I|T-9;(MkV^HTAHj9yo6?4zU<=C9*#BQJnvb` zEd8?poxc_`-5)dI)An?jzoZy7#4C8O=6gWR<7Z5Mm@52yU<}p+Q(?ughcKRZ4?}8H z$ew>LXc4SL1r~jWp6OlqtJ{HIsye_bFR~#UjU}nbFKu{rT88eBiGg94VKB_;W0&a} z&=v0D^xF21oJ&3gXSn3xTJIF#HsHa=b_F`wp@i{j zRf`cww{au`!5>(1ESG(%V@0P4OeCQaoVQ@=74~k_Yp~p@N0-VEbG$5L-_XqK; zx?%-7c3RLQtzQ|pnq;JRpg*#X_xI#Fos0rswph7(@yV7My{7jwH_rBfv#Nj3*Ya$eyCCtI))w1pQ1 zO^_&IM%+R>;mB8e(sA=D4(X}T23rAGcVGeEEbKLV|G>2Jj>zPjNJygjD}rmXcse)SM8n>57w*uQjc&lrTw2Wp_0 z%RL|X6OU&&{)Zp0&PIQ@2JiEmxI2$A*&mt0_-%Z|c`$7-q3s#S35Ub6rLTB_h86q> zziTin=qWQe{SyY=T*vJ~{fZC!t>6{6S3UoK{pq^BpnG4Mq9NxGV9n`(V>tfF6yX=& z9%ir5CvbXoBnAoWh6DR|O{(N~>Ubx&_p}E$N@U>@MOp9- z6d*I@dl=OkO=@#`1lM_-h3*rxu$#KW*ee}!r@s%A7HZRi_j@tGXB`_+vJaKh?YQ32 z8!Y8NgA1!K!uWAhQsK&R#ZDq*fA)XqY#xYnCs*)p9vFu0#id{`(1b~{)=)m@3;)5K zolqAkUh0_Qh(5vT(9g|v_Lkp=g3UYeiKZ^+jF1G`eNA{SRDy{)z4mD)5r#~waZtEAW7USlh9cAF<8vyh8medH6p{__1ot8wymyO0mLVE&PSk}OM zIWHehJvF1;oQ)a#Z!7##Ed{HxAc%fC05QT|jCstQd!HS;nUc!*J%xwg2s<2&|mILsRS z3&&UccHw2Bi(Lqul?a=sf&#{QfXrp;DCgR%sy=B0bOfw235Al$4fI zNKw*|_TGE%y*KJPpRy&ZWbgeIQf4Il?%#jl)ys3AbFT5erco`&g?bwMz~e11;A^Ow z{v7y*t%92Rk2JnA-}wdoS3ijNi(1a_Z*_NJU-=`?8;iT5X5Du6tPv#6fFus?N1O@B*T2Gec(PN&mulR`c*{X*t9 z);ezij;{*G8K&#Hlwp89lhy1$-H?H`S9M|8XcY{S0a`Jc0~ZQE_8l>=!`+;E%v`q^ zYy9T-{eKsAPK_+;A9khks_`s_qEiuYPPq);yU)q#^V$T$eNk%i3F1T~h`vlkSpU;sysh4#O$OvwFSX zyFunl1QxB4gtECsAoaYPeWKAGrbNr&nN~7t?>+$iqZ&|Ps1MvH*Fbu@9qifrA9?s; zBivqIjas(dxZlYT+>N_oF_TqgFHHiyQwsEU$r{)n@`2MWtPQtUDWK}DY)s!}1kICD zxMO}CY-=(hBufk*X|=-g56y6=lZ|S#l%cEVCNbN+2j9QeCB8YOs3!J=y>+S>m9G`T z^V4fU+&zui$V(=77BO2&zt%C{up;o(Yyhsij`T9~Z)pAKfjxc|=v{x7h-X!SiJKGI z_3kF+b1y`DDL+8oPJ9OvSUj;22G9cKejdbg32RYlY#Ai%ki~6N)pW~b~3Rk$x z82zZ5?VmTKcUfJWD)=u0*^)?@9p6NTx}8A0=oJ;gRL;tQTaDu-O>WL~mU_8=csDVBrM4yoi z*!V>pzc{|3gPDsULB$@IofAN>Q$_5chEMG3(oFW+ImXoB#9OlUbt^;`eW8yu&6sy% z2T9{|hs$+LrY1~)E(_14X+uIdyLynSKa;^47kKBPR>G1OG^trUkUz z^gzeFc!pzKL=V1AVmf(eIa@Ajz{RE*xSf$e`1U5#(MB=2=F*K@3dQNc{aG}wA`zvf z6F8&p^+e=*8}2vB22;T;j7QZ8drIW+#FlVW`RxGC>Wwfp%Ni5iZ7{Rx4oB}}0i2z1 zqPtyM&}UErziVwpkKlOF*cXi{N^!8phz}+BHbN6#%xUlhIkDobp_iffRQmkUwn;7dWBtr$Y_v%nM1+3FdE^nvKeL_rr@7 zr${I(lst%CM0+o7?4C2l=)kNmcY1Y#SG z1v^qOTe$_zjpB*x!c^Gw%L6|Nm{6@MS5QBi4ASNan{lFxDxA3r7RN zQZ*AV2j_xKN;5pONM_@e7X0*_f!uE@M=P@)ejfrHR01WBeR0 zK+_{e$Ykbhd~=c;yjNHAj$6U62Tw_n;AwjF?;+Z1oX;>v z3Q2fNIyv~n8iJ2ULU-v6a#mH8x+yE+(9CY+n$f{HM&EH+m=5Cm4e)YzK9q}asP@m3 z^z;2Uq*gTx!&mOa7x%5HYx*l1IJ6cB^Mc=GRERqZOHkozE1z9k`}*bNcy%&*kA%ba{YP;ByM7`-%fX<#8Y~V@ zlKgeE8O<-8aB{Z6$hs?>?Wf|f?^zjuMLceq-ww6iWo$k%b>Od3V(X~N!t$;FIC!TK zoqlhD%(Kk<H;#$!?~5NDW-cWZPWS z(9B{ykXjTm0NA*kA4Te~k`!JMvLdG*FN!V0*p2>dcGF7s1r1&7|7QvQ3CnRt$`CE= z7Q*a}EEnP4GMWt6LH6k8$t& zmexv_>E%OI;8EN-ClOokC)2H~@3BSX4#8fRL6SlG`x-Yj;rP#`s8~;l_S669wN^)z z&pt_RX+?mL>MA0i6$h8qia{|?nBDQO8|u|2h~xcF#Lr+BBf-~SP|Lg zR7j)e#Sr0D?by@$o?e~N17&V)6fmd*qn*Ae;I|E*UiO9MOwP95LJ1UO3_);vIA&dr zCzsdqQB9kr7(84+MjnWQx?eCBm93;stw_2_8{E8_fr3}c7&g;9`c1NeM6D>skE}Ah z>!^pU-FG+}Iughf^ZmS8=0lCsgJ9uuAyTCV2DhGo_Ua zacs6LX(>uag)`YWBJTx;2Wv=PQ7SRH=RoK2nZxO1ek?ovh+ND6OtR;lCXuGcu%oUH zc^Ds(Kdumqo)@DS_cUE-Dnmui?I88HSm^%Q6D_w7&Ti?*T!0>>NWmaGT{9)`j^?rs#2Imr#7D3+%lMVfWi&oMI>8Kx6@C#B77}*?A;F zzYc3QJ*UIJ;-Iy1UteZQ72a9*&hyQvh{&#*Oe)4I=;@8Qv&uAl*9WkQ4s&Vvl<43k? z;~3{-c{t_ud_ymBR*}o|+BvJsx6(@eQ}kaYB|-BAaQ>BA&YBiSiu_^l>YNnVcS%9G zkWinOvpYU{eU`1bDisf`{=@oH!T4LNx6#x8HA6#y3$A3R!t76c5ZSAMk`0~M^k@-k z`Yr_?hV2uQ$9zL?F2XBAT!0(8X|Z`B9?bnpOWNL&htiA>63vU#%{Gt_u?(uu$YSV` zdYBpWBs^;vZ?5?l=TO>QT=c9MGIliJ)oXvquknSbIr|Mw(@F%^-)bfoW=t0jeW!og zqT#ek0(MNM&?w&p=yf_4?k!CPOSK8s69*9}e9#T5VMWNd`2p+gq8d281?hUeSQOY^ z4)4O6z&T+DM2N)0e{MfHfo37-C&BEbdrQLj@DSbfuN|~&mC4x6{mdR%AN$_kTAHOR zh<#~=OrNq1!62K4vPzh}tZuA2ssrn^6){k0WZlueUV4o=Z$b^9&<|mr$R8g`)cG4> zRIr);+-FG^lv_fI)N}Qs`V0Gy3mzH%uI}AlyF>Li|BX*r&>{KHdfq zqy2-d#Ro>Hy&J=$e;Ny0TauxO;dV;)UuTyrbi}Q3hfx1-2~O$@;NRg2aP2XN3JAuo zuQ}A>%tEMgw*fV&DDd*u$8i}^{PMvBIEJOfy`P2uWD>A-Up+@}qyVPugt19?6V00B zhX)JtaC@jI=Iq%*lnm7HVR{OLiEW{ohh>2NjG@!28dPmv4Hzn?WAmF3l+NMB9jytZ zKH)aqllF@I;L|{DYhJj0ryj5F-^~vEV*u9YL&$w+MSRS|hkeZ@#Q$CpW@aVhmZ_Y+ z6~_wEcSR-f0v{CO*$9iZt>N{uOc>kcfhnuw*c-X)Vd7smeGl9)YmX`8Pnxk~Xa3M! zQ6v`m7s<%1WNZ+bjh_zYqX2&a3a^fVn8*`Qc`^Wde=}MrcNUHgYeM!&ECa9F^~_B=`T*K7otj9gAKpBsMZxJ0fp{fa-^BEW2!8pn8`81_6VhKA@) z_-iUhgzFPvcr3B+nes>CVY&nzKc!*yBy*1IHRFxBHS}6#5wl&K3`wj!G&D|x(Dfxm z(!Zh_8J8|tBR7Bvj>Jb|+?%6da5MqZ_q9}Xi%{Z9p*@;aDvoP+f zGI`hUi~np@I5u7)%%;yC5?bCtZBO0cWbCfS59QK$!Bic19ZF!)XU5b29Zw{UlWA^N zB3K@aWM5u=w{LOFcJ^8;XY};BO=I<|m>wq|{fMnN*5p8@wG?~rs&WC(OBeJrlkGcm z{R+LY*bY0Eq`>E-P83oKhdD-FRItn((=N2(+fz+A`|K}Tnd1fj6{Zn>y=W9~&j)|Y zY)CyS3D!CtV5oK&f>O4S$LoUO_9jboEo4)NE`sW>PS6sQG%QpI05-2MexCn~2gmi=Po9=vTiMcHrlP*G|ZXhs%6+_*iY zJ4QokY%Am@l%dC3X{Zw)XBVxthrc_n5z825cE*@5{5l|t6Ymqyzdr&C$08U#^BXHb zJqy>lOcNe0!U;RS6Xz6{LZR_?IJcnyUfelASB^NLuyO$=87?90zEHIOX$w-a)#!9L z9EEMRlML++a&SPO-aVH~55()@a-R_@c$ZB^LMpL?KZFWD-%AJQ{AF<`#zOd+JgiWi zCbx8XA@=VesR+AC#m%PNEpbMWH8biX{9WcXuzh)b((hG6O z$Bfb_s;{(++&oeOSbhT3s#%cItxwMI*P(9C7%g_HfX%U=*geat=!&bq==9JEID5v8 z*)W{ptQNgZL@Z-KXzvk-xWb0ub;m&I>?}BV_!2Q))QM_qQ|Ttb`}Cb%CE+=FlNPAd z;I+U!W;e-*;_$c>~si{NZqK^kby0#A{hU<;F!t+tZB+MS0j$sQQHbAZP8 z+d|*fPPXG_12AR2s}~k$6JhgArZ+T7oi1d7)TzVl!1y?HH0?yKXAkM6srjfizaFy` zHo#AfcPv-SJydce38I6W@Tcm33C7xRHyR687>jzh&+-k`JmBwfkmq!ry2;N_AdC_85d zXs$U<4`kZoU|9*yx?Du~{aW$M^ac?25Pq*Y0D7NH)E}A}RB0|5S!NZXS!`@@0%wG|M1*7o3 zvmLm)JCQX{p0jWJtOLXI>NuCv#q6{P;Vew6HzH$=6NL|h(U+oz*)mP+tf;Z#rquDIcjhYmkw(KZ$Q zAeoPCAA)FPx*|HawNj(M_Avg{3c6Ge!kvO{$as92trKpDjapK4p2t4Iy_6e&Y+-u5 zEA%0Ia}=hmWVS0y_=#KXb28@2u){<%%a zVwnkdXLUe%;WPbOby3@2DT z7NeNF)52?2G)eX)9ex=Cwvjd9t+a?aPgt=0aSvNJDjH;-mtefZY8;)PL3tjwK-Bs) zxbp2hou%o7r%&l%gWM{@cfX4KVBY6kyw`}`VGZin8w=NY1dw;jW^75{LsC6+*rJCm z;M1vgaKA2sUzzUWN8zQQCtnDQ<(?7wXgm6v!~DI8G~2Z!27CBE(?Qt;FnO6m5x4WuI@2Vf1=xYtAwyK+Q zf}*kIjUdV18v`|#%`jlxfRCiP!NVpA_qB2{0*4vQ9BIb1`bd1TtrB%E^HHOg*>Gks z8vTw?&Y}e~^qF2Y^i9>^!Y>;!Q)pM8wMYt{-I)ZREo(8TaEvOuOCh(7O`i|XGHm`9 z0m65SklEcs`EE6go7I99NnOa4;B{d?OF$-DhnPPe5HJhqPj3HQs3Kf-?+Lqw{_syxYLsQw%P1>bt8zEvJ%xO8!Od zg_Ka>P9#QdXZ+6ZQD|=Jf#wFxy`n1?Qnmmt$o$5-m3)p)+(>}-=t^8uRf*$kR>BRw zM7GDxQCfXS5A55;;gvbVkiKULhhaNNT(+PS#f*0`TN>81*usP1E{r^0hiZ>k&~_}v zmv?lqe?tn$c}J5q=5`Pk+ep%xw}ts_2~gF&LxSffK-iEllg}xFT|efMn(i9#x@U{; z$B&}SDQ0&qWE;MJpF}PmZ-U9aail6M9JzcPaM2YrtVukIwpzxJ`Q3^XoZz9$Yp)RJ zT5He>_{ToVe*m&FjH%~+K~TS1NM^(~vi!QD;MG7YsI+H5LH{W#$?W_n@#H{jh$=3! zmLohtTd>!P(!Fc>u$IRaZh0-ln+&TgGC&6M<;yvmwSt6yPaIlpQ$&aNYhY-bI>>&n z!QZYSxY_*{u~Sb)o=gLD9<)G##fcbnPa8{Toq*ujTI_%A!*KYEk!(IswU`b2<4x`` zT9b*D(mGgX(uv>S+@>D$7l7#cCJ4J#3{H7xX!8w2{8nF1+LrhNT~Z0(&K`qBvcr9f z$vL=4VmH2SbpRciLR_2^jgH4Wup;Cf9k{v}&W;B&48SC?6kZHZ-K&XAT0iw3bwn}e zb+~C@1t;)yG3)K69xrZ?4)=+@^n`dJFRO5-AP>(&E*(_BnCQGgq=T5-tZGfi9J1funB zpw#e~)Lywp7P0)WE$t3XYtSNcze?~`b2Zx7F#BS$&&k1)y0p6~jIPvofeH;KBmCY8 zd&+C*4KBZ5 z9z(!r>j2FSUP|U2iox)I>hveWBk@}j0ovxG;2nF0%${IFT0shIoyAYe99!XAmjlW$ z9Pk6_IeI34V<3n&quWU>(CjEezmWShGJcG<*ngoHo@t?9P8R+bP>ScfvZ&?VEO@JP zj!GB%W7O#zbgOX==E|3F9-YnxjkE0-v&962SI0s^EaR)5+lfzHmXM$Y7VuNC2s`H` zK~d^$PQFbUHefGRwN_^@%oM}^zoK}0Oq^Ep{UY*8su1#%VYQC*as*_%NS;_GOzm3( zu{#6c^QF7wo8NpIEx8djJS5Ru={+e)%cE9xwIDXj4gHn{Fk7(uz>48kN=>AI*+wgT zNvxok)rDN<#`yD*JADwC593cWXt#0(MkiEY-tGG==e76QX)5jH%B3ON75kZ(Uy~vS zm-bWN1A5R>&S+YP*(f)wi)cN%K@SVhH^k3(Tyce5n)qt2RLg$Cx^$)WU}w=ETj&9Zq!!Bd!wyH&4Mn#h`SI(oF^;xgJg5 z`r!Lj84x1I>>UeFlk;n`7=D}wZ6!@iH;>uH-Im7Tz2#21yh0$cD;+Mrc0y0v0lLN@ z6^FDch+%a!=w(h(9UViq)6{bg&%a^}KHLs(Nin&0vjMDKOOUsElx9uz)0hpP$h*C# z=u)$7_(DG$?#QP?&#Z3P(6kHwEItH@t9&rn;6B@F?kn2r&rO>jePI8+-9qN_?8IsN z1OL}cP{O*8u2l5G(uIJr6UXtb)(O0MrJwWj4Wl9EH)H*DD9qgzi97+J@GWkTE%Ki= zZ4gd|ZFVV4_h&i7PGESBd(**?uK}Ns1h(f-UD~lV79C8ELD#<%wDYMyxP3Opo#{%b zqsR|0=IW8e(5=vNSQbj(ro$D6J!My1MMC{z!E(MTHm-9;-G!aNlb?rqK~Z>bNdbOc z(na)2+sNmarFg5|5$tkr5?!gpB!^_!dlOoRSNtdIXn=XiCnbMt?giv^M z36m2vASpr&dyBa@#Vxr*FEgF`isz2F{9YI6DIUbz0(iE?0nC0*$c+!2)JhHb zdvFI<6rKQSRx*>{C4}2gggLjmh~cg>NWRxdR_)M5rRhRi`K|*Wi^y?)@H6+4)m22F ztw;3J>OpzceY*N%9%opqruPW<9286E!VftT`0R!>vev1SYp#M2uqT}y?@5P)18eCk z^-;3^K@JXxxxw52a-cb109Ffm;ko0soL>TYD4Sc2?zb71pzUIKd?y~l(!Jo}lso;A zuL146%W%Xt3&n-pXwSVJDBE8Ts%x^a{YO2>Kli7>5>dppyaet?YoNU^Ukk>92j5_4qH#^E|WS(3;O-*uhREqggj4m8sB%vn%8 z>dLZFl>p1$b`%(!A(OwYh}Z>BJRA}YJ7NyPhs$rt;=~2$y}g-qyJ|pj-A=mioE$z$ zd%`-Lmyhr7od9b+A@Xu)A+Z5Bemen?Pt{KwlS=A3kR+l z=I*JmA6;x_lWaR?1Mjg96gic`8L!Rg5xSBjobF|}ea?gVU()fs&oAOJ9F8kJ3(@<0 zIh5_R0hc?Ss1#@i*54w~T&DoCwjLk~_W#k4&%DI$cqCO2cIwqC4n&=*RWO+wjHkIT zv#)7K;cQ}xR*Th1xgrk{H?GF~whQEC!aJh2TLWkK)k#{R0Uq)w#uRaX+Q9T#_no(9 z%c_QB%F|?6d2uf-KT(E8vnI$(lWhEVZ$Ah<&LHpOMv0AF8AccnkWJIGsl~%$dV0ro zqB~WI^Y=wy_e3?0N&lliKP$p^n+3o@5lL$jtPjqBb>CW0Q&JK6lA;mMMWXP;c1#}sK#qM$g7>`r zY`44L*p8jjRPuH!igk9uj}yXB^t%x2Nj*va8we8Fju5I{2}^TJ;Q{0Mr+m+)8UN0c z1p65h$(#+X9g}36rvvgAw?QoPt$%c=5(Zidpq^C?hV>nopi+PbmKLIiP&%+$^f7A8 zANWBSROWZWK*TDBjUGeNRdP`JVJ0p=+JK5(Hn4Qn6*^@9hV7`70mtSWLfQBW@?p#r zQ-*c`?_eYFFRDZ1ImI-G@!pqN_s}f4Vsg;85#OjDN7sP4@IoUKR2D42q6SkqE1rsz z<3ectG!AaBW6@(QX?R((3;%oYfoxh`P8Ws!?hB}z0}&3V$wTpAu*hxXtlayQieHF@ z=jNq-$F1_2jorgodB6$0>X*a8?@Q67HUef})Pkalg>dU}9{a{0O_2IfO*KgoE?>`~ za_cjR%IdXfenAAR25Ug`<^l*jc>vbF6rk&60ys_0H|W2QCBPFMhkFVXpgK?le)iNt z_lFhKP(%U0w6%fgXc#X48OgdPy9eKOM8F)W!{E?m$n+_%bEbNn`>dbUV?tOh4D?7s z>brHPzW&XJQJ%GM)N2tu6_Ehlc9!)O}Izomvlk&{Ri~bX&Nu2UEOTpgJ}^AEB&Fn?ux?l>{wtsyyk?o8N=C& zZLsmpEspzeJtqGw!wt)FKq+-U#$2q0LkE+HZbC9n`L@HjMIzku9H#5PN7ETDbL^O> zk5}dTNTNdx$Nc*dl%FW3vHO|6&(@RV^2b8Lyj`&KD8t?D86@2=4?zKcBq^6p#Q#ir z;PbRA{(4f6k4}Wc;glV)U-vNODsX|nnfz$3{D$o5X~%VX%ke^ZD%GvZC5pHhw0ykC z`n@`sK06gmE?I-Xh0S<##eVj}iaI#q{GDD}*o8+wM}xtX1iX6*Al79}O24?!65cAR z^jZyaH92(LSp)M=xpO`oyGOWFI#AUpi5;k{4v$i8(agyQOs`guvUMB5Kt~+hGtaWT z=Pd_Yb`Cb4iiC_*Tlz3#BWt~vG;Y)l!ui3wu(eMcxQqsz}gLlQa5jx^$J`BoAnl-4Vo6F^3dCOL@# zPS9^-iq^XrzO&i``sunPJdVx7B}H3_;wMHI3vq+gNEPVj{Gd$V;A2q*tqxv^@wYz3>?S?E7&9ewgI3sqh6 z!E=QiDd7pi<+n;;;kp2(PhCnUCf33owfepz*KTqC?vjDSBU2nlGcj!bbeD`xPtvv^ zYk(6KpxC<#g??Nltj1{aG<`KVycK89wY6rKDV!o}M;Og%p)CyWX=$scsH^Eie1giFE7nKUp zgtbO)>@4$}#Keiw28(luZ4`@`-K@iVVnL+4ofifNggGs1>u?cQDKwhqA*t92F7yCf zly#0?EepptjDHjtmkI}^e6jJt7HVJMi+3{!@=r_AXTz1?-;~B#(9r?v8eLeuA)Bfl z-a=MbY2ed~tJ#lTMEV*OL&5CwI+`cuPK};g!k@>pP~_`=fZ=@jP-TOJ;UH~lKS>`Z zG5s3ub+jorfiB}a0Ma*{$<60JAezL_c+90JDd9tSgz`YMAs#+1lSe)Kau^?X#-;-e zAS>F;Szya>ni5M1>tzOrW9%4E+8cC>qGaI_~+d!n@6=(FAJ6Y#;oM`CB;tOcTM?B78#>FtB93(g! z_mwa^r{3&OOrCSquPGAmUjs2C-N3UZhxt9!lBi1^aG58WnsXa+X0-*PZ=NLh_|1oq z8c|riBNCQm6vI7%CUjeIotAc|V_JFwT=rhWdGNIyS&cGaFy@HO<#+pxWsJae&n6iC z=L|s!BYp8Q^Qrb=4CmI~PK;^d!joqcpv_1R(o1Tvh{@TQY^{Zc$uwLkCqkcQ8{>bc zYWvFnQ^4+^GD%32iN2OKeGXeF~Jl#ZR+6+}O) z5-&OiP%kDAxa7%vy!0Z3^>eQl?q2VWE?ZTI*)?6fwyl)ysar@KHpU`EWmC4__i!v6~B2Z?1wFK~u&D(M5yZ zb?jZvr#a$$)|kvyLfQm3!N%8r*xC$}H=p6G>1@@<9m?~W+>aq^V$&beF)axCN{rXl z-cA-i2tQrmpMm!+-QdkeKJxJ>!$G?YZn2qmA)oNxV`XPsBB_FPpt3ti6l_X{>Wk!8vlH)J{bhw;!#6uQh8(obmn)Fx@%vkRw$F#coGk0 z)2TGrOA9{fG3>rok<{R$8eBXc2JVU#oSmKOL`LQ+RbI9j`!kt+2Q?Xtz4C`Dm_`GC zW;X0Q{gZAtn-*AW5@}mgQb2a*yR<);~x3WZN}LXR2YaX+E5$ZK$xU5%sFP>1V%PxLi0Le5zvL zlFI@3!F=mO6ykx~BM0{H-^zA8Qw)jRpULP#J(#;#2b1I#;ORsjzWF-Bs><7h$sK%z zQzJ&CGiB-Y+-|5jUx!7%I`EngWjC-7;mwV{DAi+!ruQm=E3#PO2RDUVVt~?0$gvNRgY74=7jR;UabPRjklj&1kXL!XMO@9vM zL#V)FSo1^*^ZukT{^NXRf9(JqVY-CP+nhmCDh2-Te8={DT7cg}Q{em(K{C5c2yJDZ z=m9SqJiBoboY!o{i?=mE>`4>Om(s@8CC7k$!jUc%JPtoErh(Dlm-Ot48rDv~1dMig zOg|qLgta$EXm)EB+|f6NU7`CRvMB``qt38(6kIs0sZ^>ETMr|z1EFhvB1&8wA{7rW z(JFQUY(2FNhkqm^S4=rkdvFY62TMTBtsbmD9LII3{`79hI!bx#@riH(YDU$=6Wth) zD`(EXfw}PMZ3*=~K{1%quurONG?q$_HNXNe)-Eh>44PI;gpr}y+(hC_cq4XFUJ7>eGXk^Xy!q0Z6@aW8=*kN$4g16(#w$ko9)n6s~u z{5X;ZUkox~FgOp5zs_bDX6+nAV@a?6<$^V z*tTNpUTwI?tlZeZSSNt0{?M)zJ&)Qk?x*V3%^JUn1jjY`8SFfv&NojBcaZ&L|t z-~=DebGpE~-=qQtm+K)`dKRhpng}m0e`4#zdx8>^dni*(h1HVNRN$^O-dgB~!+#~< zNL?v&r(rsSW%^)ewE~x#C1P{hJ=Ur8XSA~40#3DGA;u?XIJtb$$aY!=&bQ{{Be!1m z*$1cS{0miZEY2AC<&CiHLKOh~^4b;fm$%{K!7smZw0H+lT4? zMTSuOA{|6~qCxE<^Hy3cj~m}Lfyn!CxP5yAJ`ya&{HwK!I2-YtZ%{Va@p`8Ap=fITSMWz zR64m}E~aFErYWD?!IxbK*9@{@x~PQFZ~l<}I5%p*{e;AroFINomZ`EjmC+?P;T9fV zdemtLx-a}e_h%Idg57(MNDD#ia0?Dy&&L!wW@U4k8I|#fpuzKFF=f#<<_?kt z;Xb2Orl}q(x$014)DKIHwty#-p*wX$5^@;XFyU<;UZmb+?-xUO^l>Z3t&pJB>>Bd& zKqquu$sp?I+SwgU-d;OZnNF@;2saj#frQ%}oC-~YJ+Vh|iw75|F~5nIv%0__>jc>C zb%d_H52^TZULcJJX^_zzn0(~}UvA|w8O&4cKp{2ApLD|$MxyXZg1gT>QJ9l)JqDLP zWcEE4HzPlnA>NT`fIT6#@T$QC1hk!S&PX$q$PLrJTM-nNc*D{NQL5iE!|@m?BN*ER zdq_02af0!k)k(Ivn;A9Ds3x&PMntS+E;KONNIOq|Dqzeexkn>VyS|FN;Hton2R&%l ziaL7LUZ2S>+hc?pfP6h{6blDl0}1%G zXfv|fw3zoqG+p&%6C|}wZ~~PysHRLWdEdeZ&Nc~DioK252dRYFJ56wSrVL(6Fl?N$ zBxcw7E{*bZfc+n3L90`OM2_eXC)rPZCRR!K;{6jkN41*V1r3xBX$C&|WAsXG1D*SD zJznx!2jp5S7)f7Z=s`IN*o0N zv%ZY{f3(r>3_CvdF4>s=fSlvA0FFZf`4{$s*a{d^XT^Rh_)Z;U3Q|Fb-;cH2JsEdx zA0v6XshG6kJ$Y8t3_m~lqp|Tux+$jtL>RCB=yWq|=PbpJ-bFO({V^2wGR63YB66-Y z6S!|yFntAHJQPy^JxUSClhnYRAqM1JRXr>?mI#L8(Rh9;m+o3x&iIc2v9o@#6ib!y zoW46`z03!}%U;00TpFzFRIuSy8!l6*0)L$bB3ZtcH1_d9K|DX>wMC&;<$91TUqKu6 zTJWr7JRK?8MZ=dcdD?Ahc-p&;u654E4_|db=1wbK+pdZ?PDkPOlUz8E(}i6RhG?LY zFRstn14o!#GuPb)<}7Fgxq^P8ym}T@3EPPQ@09T1SAR@W+f9;Mnm~We7rMh#5(Cbb zlahlCIL+ib2tQK3S-6xk7@U=R%#kH5@LE@AG%t*$Z43OTl?`vF_Mz==U3?w70&X*R7N^xojE|B+r$eGi^?Xrs?RE_O z&NYFAqf+F4U^w*tZbWD6Kw>to#F;qmf!DRe=$j)?XbyiXc09Sye&yK>D>@h!`D%6g zds_`{TVsx2n0w*YZ7q0>>B3jn5aNuzgC3lpdd)*98UHaI{(CN(=2H$%>&?X!p3w(vZ%;xpzV_g4>^n@fE@f zu2y_i9*awIGI2n?7^Qf}*f%oGK=Vj3O0+oB@;#devMZYhluYXlI$VMc3a-cuEdH|@ zw_TH=Te)NUd)V0n;&wm#SO0VwINUzFKRrfopzWB3!G{AW{VuOE2Ht3d57^re>+`KE z=wEy}z#t-PK!4`rmjPx?Q9r_Li$S_jy1tq3BTRYDY@sZeG4Pjo*x#c$JP=UUtG`1w zh~`;*9q1_v*Wb|J-0$EqV$ebJ1{^=d48-{?AE>v{H0;PoHxPl80khIXgXtk<0|BYQ zev|ie`+Yj<2VQ&H8kia@>N`G4HTbUEW-#oe4^IwWflV9a3>_Vs?f(YS$q z<7|D`;*^2$k2eNhs`!I?X@tJnrz8E}4CW3jsW&uuR(W9Hy=Aw)XlkATUuUJh@$0KF z*frc=)*3u;%=c%%^u(co-A3mJF8bOGBwc#n?<=#y;K%w?20JAz3^H{N8Qk@l(bwX- ztItmBG}s=dV6fuItTVw}bN<($*?84_$8o&en{JETm9hz`rN@;Xv{mZ&{d`SvI=M3~ z#0f*YQA-+)B#SH3gV?$*Q@6WrWwc!^sdRte&v&x6HMuCakxj}|vsKt^N?U)!`*qIi zoY#57=WFaye(W)mntVv7L$|OXu7;VtS4SLqubRDYdBK=frD2Y@M#W~Y2Ga60Q81OO zgtKij5-vOu_EqPT{=_=LWAus;7`2eR3^$8UCRDQWuzE63ZpFCFkF^(TSg_oVq`qCm zf}$#$M=NVFWu+TGoV$$fiJnE<6SiaN`ctIi={eFcF^B0F+wdG41q#=%;r&7xDR`*E z>WT+&Y?39r)H8|CT0Vq|c_+YA(#F=eI@8RM5->c=;!(BhpyZ{L)~3Wmntl$+tv8D~ z->7MBpd5cHn9rx5QNg(D=gAah7FQ)PVP?iI)LRA9O(s{KvfG{yHg#iF*il;4co*Dz z-MBi}1kPzIL0X~4>kZDh%pA=B$<{*HZYx&4tqaH;0LzmAmBfc74re&k@gf~DBMx5!J3nwq_g>$dZ;hTz0Al7WBDqFh^NeueDofY&~*@}^rXjR3J5)&K-}t<;)cpW z?6ug1nyb(7Nsxk%cj-a3MGV^NILut~4o&X71MW5s^uwfs5WIRLY&E-6v#k!E$F1N8 zA7xUDEnB$d4+3vB)<^zx)gbMyKGW5G%e^??H>1Ke}Gpj5A*Tg}V#O;CI6vRC|pG zH8DZLsy;nA=wvMjIk9ZCBv16cuyl;mz3dI`Rx%tm)-}_+$c-vDoSl4zT}%`Oj}Ha~ z$^G*}V~8AmSJrs}QR7^C zwMNBCg&`#DD z9>8||*2enty#>#l8^S)n%}hPeXmp%m4Z*YiFz)ywfjlnMh*fq+$c?G_B-(zTIBfX} zE3Ez`(l_$9m=UZsb_K}E@J<5>Z0ID%W3pIc!k^;h*sE-RYLn>cm2GtDbY|DG>`)s} z&Z9@{g^QlG@Ej%R;M9T>2lG(Eg80^0FPPlF794xq(c<)>hWLTKR9)>NrmzthyR@(` zXE;-ZbPA2p9$@o!F2(_|lyrB5;0D!r(z{F#UT^u)OxxwuI?RN5N)L22hw;HfcbPmU KzS(C^GX4*0F|iT= literal 0 HcmV?d00001 diff --git a/tests/data/layer_norm.data b/tests/data/layer_norm.data new file mode 100644 index 0000000000000000000000000000000000000000..b72b7d5a7a319d5b8949039c16cecd43949d4c93 GIT binary patch literal 2048 zcmWO6i9eNT8vtMxWl8qPS~}I2H9`yLxld%7ipW^U8NQm|M6~!4HMUT;6B#EeTNIf> zB6WoKxnCq}mPsmGw3oK2WUp_oKjJ#6k%q~~h1j?KIOVSzrYg2x5Vytz_IDJ}df!Ft z#(72LYgGaMSa*ZeHs+GBgX3IbU>>A?PK5}INZPQn4DFO3qqw{R@K@b zXO#49c*&iV=780cQF1$>hshb*2Ch5LP*p__u5rHvxOph?MB;Urr`C=ab{gPturGX7 zNr0=G7DRq1kV>9up?nn?P~Fu=&xAgwoTCJels7<`tvB!STqCZW%*BRxuVHqbERq@p zgt$Q1WSk1;)Qh0gj6s1Thu2{BneMz-Lh4hxQLJt;_LYuP>%JefJ3SFSZl!|3VtG1j z>jXZA$izQ2hQ4ffT-6$|**D$;?0Y(>s_aR40{N7M7x?4#;J`cnggWnnb03G}u z^qx-4j>BQ~PMX2aar0F(z7?+c!E}Xf98gCA0rnPGGGfUB%Du8 z@z;PHd%J+g)g2W^;Y%O?|uy@|!pIAfSvO-x(!sfJQAN}dfRLQy3qh9)pBe~9Ro z(Pr!v-lUm+$zXhOA*l*zg?1SS2vbPLQz;wqqjeofE>9tPm3pu%Eee&n=kSU0c^tEm z!<;`QA=tH>z@j#io^MQ!ZuJA5xpb=D6bpQxbIj~L0bUoikRekwXir`X+WuLnAENqI z$$T42KwHnZ%&pdPYOd1?6&70wlh8ocSt`?an^%IJL>l$lbQSFb6=*XfiTbk=C=mF7 zg!~@-blV*i(uHs}a1_f{Gx*^42u{vsbIatCQI}s0=Fv|XtuiH;vNWW}bmEv5V<#YU zl|Q6}exb5&e#P(&%H&?!7Z@LHChE2;7zaZ~oE6PcU$xtGrd0(NP70B9DpKWr_H@{? zoGFN|gy2p!c)4MM+6ULtfx#>I?)xKHWg^bYNfT3%L$UPAlck_N zc?*8bD+O1rU@&+LXy$0m`@8cJS|kCleWn`^2t4tFtp%=;K7yAmlCUbVm9AenNo`cj z;r{G?5~-d}EH`Rl{WyU)e$)K65Z*7;gG`r^% z(De_qNy7N=#OIO+ICNLy=*(-b_k#Hl_0v3d>9-LS<3A(qch|y}<=cp=VKr&XxkY?o zf`llaCBeH7v;EV%NX7hNs4v`2)wix=ljX(P+{LZTlZxeVX*il#{_IK!_ZwZ3=gqwD zOhWk$IXw5`WN`7njPh?*^ZYXIqtOx>p7X^xqB(Ge^aRB~TGwf4{7{ayJSU!>b~!EX zoyU6q_7GDHGQcIaf_eqy(Ia=lQBTei%Le74y(=8A2 z*vg2FOX|X|+G#3h_>?}~A|Q6I->7$&6vXt3LGOCMU7g znvo1AT|hH4RiQ0@kUSYxL*ZR7Zq}eIDnDo>QB~f&#G@RxN$VBJ(0C}V&f02%&7#L})$=8-`S+jxwr;%q+=z92??g|ktiq`qDOjylv9668la(9G6quydKi2R-MRh*48^ zt*{awax%cac@+4M&Vugb70g=OMV!`NA#eWu|Ao_Q9EqJ~g3MC54gMuWP#FNR@eASH zj1=@l+<-f$l-aEZ5>RDYK+ny%k+&iayXT$~>f0_QX?p{JwmO1uV=yM@){}z)+Kg*1 cYt_C&jCP7DSd1}c!(2{5<(-{mMw*5H1NDIN3IG5A literal 0 HcmV?d00001 diff --git a/tests/data/linear.data b/tests/data/linear.data new file mode 100644 index 0000000000000000000000000000000000000000..febc77d0969162ee41aa6335b362076e63451f95 GIT binary patch literal 2496 zcmWO7i$7QQAHeZKD1CF!tumJ`Lqwr)KJO#BM3HVTW3;v3NQGbKPNE`W<`yEC$O?&+ z$oaevCG?eB`mPeCS+#Uqg%a8Rhu8D*I;N41$#Iz|a*Ctg%ZI75tp_BmG=>9RCA7sy zieEpsoP4Y=!PzyJ$@LqBBrI^8$qYCHY46h@+AM~)>Qtkh(gT!`lLv1dCrH_<2Hy$V z@%F8Y)Y<4Y1T17h|4(HcOEacrBbJ00)xsQs5DM7;!`d%y*rB)-_x?PWS;bD%Am3Qh zt2shM=FgZ<3v|Ki@d#;5?BfcC9ASq`7FAJjV~ifogI#WlY=cBIo>3KIk&6Ki2m8QB zIF=%5g=)j zM@R^O4Mu5@t6mP>>o`o=s>`-oy{9gBDoIOfFV1ORh@z@-x=}Psdoz;I?P?krER>_e zw$9+S8oA?7jX-pIH!kn=x9~ac2KIejR7G$M?)x;5s^?MkpqCfJP(pJ9qsvcdF+{Vi zlW@}O47cxxH%y1DDlXVig#Re_aoe1ZqKQH=T~iZ9xVJ}Xo#%Tdq&I?;%4dT$pGkDz zwUNtzZl(1J|8N66n}}kSEgmHw(4}`CDcG_OHFQcyzP=W*ac(1za>5|5AqJ)P=aZ?9 zLGEGxVMuYCrF!=_!lc78_a=Ubq_dc6C?=!ysZf&HP=`sOiJWU> z1GOG$$L`E4G{-j?j7lX*y?-YNWgH+(J{eD@uE)02iwzQQHjaL$4YrP zW+RIQzez*zj$Q&%w@F5^5eeVq3z{<-RJ|=8yuEU{={qTSsiA`mnW%y=`CC}#myh~K zG$34IA(Cg=u&An?X($SX{hDsj8di*sCDCxeW`?#{lwrV$5U|@)22Xy`!~7~ac(gT> z%wCq}CdLxc*8L@SwX=qrYIZ`c*(SmzwvsiLO7xY54%p30ryd(FpnZTmZRey>e|jFK zqEWl`(rQKcY`L07XeMx4 zV@DxpxgVs4exQQCzsD$ZC2}YI1B{Qf6LniH&SCXdoNkz*KB|rMOQ$kOe#%7Btw5Fb z+0$Xm8m{DU9Rzo)!ZY(pY9D-^4n8`MufE=g<;D_henueeRWJqVSS1+!{sgIL6X&z8 zNJE+Id|aXCjc#Flv;F66X!@p$C`UWd;qgq2*Dj^L4ME`7lYz|wS*W?>2;~|K4c=vm z_YBH8o4+FQd0QB>(S3^6Zrg^2W3Eu4qK(I!%3+4N4&1;pRFaB;5W692m!gRQf!?%o z?G%0g%VtP)f5aHBIDwLXz2Vdh;_*ATJQR+9B#ZVGqNjHawr8|~oP;!IvRgzmhvMm% zof4Ek@S1hq6os=l7cM?2b;wf%kodo zT?e%V1!(h17mSWYv3mVqIVtl~gdMvS% z{Gh1c7-M^;z|t`lZIc#2&h$FdsTbr{=qw4o=?m;n?c~u~qTIC*|j8Z+JqxWY}VKr3dwz?59_;Lfgm^+L~~ToVB$Y-CM`JXEyu3jTnO9K zKG8Uf7esfu3aa#nN%UTE?2jQ-UMH2rls{pDB(D*%aTc4N?;}xm@$@Nk8!9X!seg1H zb_Mh|Thu>;hSEfQ(eA;1mnDzRhd$6xnvY@4HdiV?x0L>rR}N{$RbY~5N6HL3VQ~Bg zRn4da_vjiT?t2HzJQkzuWi{wn`GiO(w{f$o=gCJ2mM*$*6-#PWpb)ZfVWa}{vxzrZ ze{>(3o^OXD(Pn;MnJhkcc|>CiE>QP(ZD^<&+#J?h0AABVcs|gJ{gZy&E@LhDU6f36 z3XJe2rr^$55uA_NgVHfd;1S(KN)FdELk<;~Ij{~LSFNJDb>Ud|J^@wgmqYp*8yXa9 z4(j4_c#*j}|KlZJViWQJO;1;VOZfmE6w09I4lO(wu8oBW5itH_H_00OXtloMA-D=R zV3MRA#+Il;lb{&ylx9G@EdxToVo1vUlgb}=g?N8g^oY7faw@wq&1o60qN>Cgo7Ygo z12<^!XFtw$(hd514WXU!V2|eC!;*F}IJ-ZCzDTJg3&e6j;%+&_%kBm5J5%(mS`L~$ zHA2^e%6v>N&y>7zrrAA*IEmId>_lJ{{p;B|GAlM8_^fH*V|LQeu&HLrwz>bE<%;EL z4<`r;?Ii{s*XY5k6;LY(Lq2d1EFSLTlD4fzt&RQku0aoOk`x1nN+s69F@QLql0vE9 zd~na%6w5%d671?pMw=Tfth!Q8mUf?{j>{?_{GK~0C;#OKkSDMbivtJ8&>2@P9 z3OUcJyQuN4MIT@+K+^iP!XJ>GH_Y#7@v)xR+(K`5YxCu|SU%h`o_AHh!EP&F+9Ej7 djmLkw4H_0BtbM{dD0W`T8w5BID-Rdze*pn`zn=gA literal 0 HcmV?d00001 diff --git a/tests/data/rms_norm.data b/tests/data/rms_norm.data new file mode 100644 index 0000000000000000000000000000000000000000..7822cdeb3b5119aaddddc2d63f65b88668c02dd8 GIT binary patch literal 1792 zcmV~$2{;w#8USF^SSNDqT}!goSh7S7o&Wn9sXVe(Bh{SJG>^)XJ0g-~3AvU=jxEci z#b|`2MalpFhaN?iv`My`68g`zPFNIQ}PT~r#k=i6dhn;X6wHir1b3>b-Cfy-P2 zu+pmqDw}+)-dR4yoI2-OM`kkjxp@jL6a}n*y$6ac zIZYb0!p?$i+@#sx(KpElH$G}*T1tu71rOfRBEX)tx+q#W0ftw;Vv$-h|1A20 z2JHQtG$}9U-#MJ+?=>D#uT2TuR<)Q}Ux@-bk-(~Vejs5Tk#Jrm8-J9D!T4PxTpMy{ z*H65sPaisiTKRgO03W$$hYS?D*>N4CbC7Rt0P8#&p*JT2&(uV7xuATQZt9`k`7xO4 zGtbProba>D68!q07(R#YgjIc0)bi9Vux&18I%+9gJ~tf1HPt*h{3P9cw1)jSaG$q} zZ?K#_DQIRQ<2Y9*dHnzCXZZL!WnOq% z9z(Wv0Lndp%wj7z;=Bj-XET|0V?6E><PmP7E|OEwnWjN=7qRJu!h~O zVEb(W-1BawEUO6hU-j|A_d~UI<;cdLS1~s&Lt))BBiy;;cWA4*!Q}sRCs)tK3;q6- ze$gs~ik(k$)w@B| z-a(eu2ASdlp)-h*frluqt_vEt$qxMNK^3Gk;dr%#68mbBx!e_PcRRvgdA>OBs>Io$ zhpfAxhQ_5!$TwplF#42%i69@dv(E5`SL|s|BPCCB5}82RfHw(*cZ{WeVS=$Y)W*XY zhLinSt@b1#Ij&^$=o9G9j)twO!%)`WMter2nd6Ix1oIX6_@IhJ^-IGVXY1vRoX`T3 z3lT(|FbZq7{R|qvmNS7UfY|&u?arOaL^u?t!Qbz-!POILsIKV^JLQsy`P6m#DK=av zSF(qy{dS23nMcEph-zN?nAWW?+s(Y)+WGT|AyzL6MW6i=V%hctpD*=xK*jp0WplMp)s zn0sC+KYw@vMi&2sTXP974ZACR^!PZD_6@GPTT;(jzRb~HzYN+DFbhkRzax+0_6fZN z0(#cc2KASPGT#I-e^OOV?Xq>)xtBMYRHi(+q^`z>1`ok}R2tUa?%-ODQ_Onc825L{ zCL6C6^M)-ssHu`e56>y1^k!2!n`cjthc5(=ou+WqPl4^45%TYgK9B{EN5OME7ybq2 CNm-2m literal 0 HcmV?d00001 diff --git a/tests/test_convert.py b/tests/test_convert.py index 39abb68..e8be3ad 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -5,6 +5,8 @@ import torch.nn.functional as F from convert import quantize_q4_0, quantize_q4_1, quantize_q5_0, quantize_q5_1, quantize_q8_0 +HERE = Path(__file__).resolve().parent + # generated by: # torch.manual_seed(0) # weight = torch.randn(2, 128) @@ -167,11 +169,11 @@ def test_quantize_q5_1(): CHATGLM_MODEL_PATH = Path( - "~/.cache/huggingface/hub/models--THUDM--chatglm-6b/snapshots/a70fe6b0a3cf1675b3aec07e3b7bb7a8ce73c6ae" + "~/.cache/huggingface/hub/models--THUDM--chatglm-6b/snapshots/294cb13118a1e08ad8449ca542624a5c6aecc401" ).expanduser() CHATGLM2_MODEL_PATH = Path( - "~/.cache/huggingface/hub/models--THUDM--chatglm2-6b/snapshots/fc442f7e7cf3ac073433cef0f301b4744c25edb6" + "~/.cache/huggingface/hub/models--THUDM--chatglm2-6b/snapshots/0ecfe0b857efd00836a4851b3dd2ed04bd4b197f" ).expanduser() @@ -185,47 +187,58 @@ def make_data_embedding(): def make_data_linear(): - w = torch.randn(8, 3) - b = torch.randn(8) - x = torch.randn(2, 3) + w = torch.randn(16, 32) + b = torch.randn(16) + x = torch.randn(2, 32) y = F.linear(x, w, b) - print("w", w.flatten()) - print("b", b.flatten()) - print("x", x.flatten()) - print("y", y.flatten()) + with open(HERE / "data/linear.data", "wb") as f: + w.numpy().tofile(f) + b.numpy().tofile(f) + x.numpy().tofile(f) + y.numpy().tofile(f) def make_data_layernorm(): - w = torch.randn(9) - b = torch.randn(9) - x = torch.randn(2, 9) - y = F.layer_norm(x, [9], w, b) - print("w", w.flatten()) - print("b", b.flatten()) - print("x", x.flatten()) - print("y", y.flatten()) + w = torch.randn(64) + b = torch.randn(64) + x = torch.randn(3, 64) + y = F.layer_norm(x, [64], w, b) + + with open(HERE / "data/layer_norm.data", "wb") as f: + w.numpy().tofile(f) + b.numpy().tofile(f) + x.numpy().tofile(f) + y.numpy().tofile(f) def make_data_rms_norm(): from modeling_chatglm import RMSNorm - m = RMSNorm(7, eps=1e-6).eval() + m = RMSNorm(64, eps=1e-6).eval() m.weight.data.uniform_() - x = torch.randn(2, 7) + x = torch.randn(3, 64) with torch.no_grad(): y = m(x) - print("weight", m.weight.data.flatten()) - print("x", x.flatten()) - print("y", y.flatten()) + + with open(HERE / "data/rms_norm.data", "wb") as f: + m.weight.data.numpy().tofile(f) + x.numpy().tofile(f) + y.numpy().tofile(f) -def make_data_glm_self_attention(): - from modeling_chatglm import SelfAttention +def make_data_glm_block(): + from modeling_chatglm import GLMBlock - m = SelfAttention(16, 2, layer_id=3, empty_init=False).float().eval() - x = torch.randn(4, 1, 16) # [seqlen, bs, hidden] + m = ( + GLMBlock( + hidden_size=32, num_attention_heads=8, layernorm_epsilon=1e-5, layer_id=3, num_layers=28, empty_init=False + ) + .float() + .eval() + ) + x1 = torch.randn(4, 1, 32) # [seqlen, bs, hidden] position_ids = torch.tensor([[[0, 1, 2, 2], [0, 0, 0, 1]]]) attention_mask = torch.tensor( [ @@ -237,23 +250,15 @@ def make_data_glm_self_attention(): [0, 0, 0, 0], ] ] - ] - ).bool() - y, layer_past = m( - x, - position_ids=position_ids, - attention_mask=attention_mask, - layer_id=m.layer_id, - use_cache=True, + ], + dtype=torch.bool, + ) + y1, layer_past = m( + x1, position_ids=position_ids, attention_mask=attention_mask, layer_id=m.layer_id, use_cache=True ) - print("x", x.flatten()) - print("query_key_value.weight", m.query_key_value.weight.flatten()) - print("query_key_value.bias", m.query_key_value.bias.flatten()) - print("dense.weight", m.dense.weight.flatten()) - print("dense.bias", m.dense.bias.flatten()) - print("y", y.flatten()) - x2 = torch.randn(1, 1, 16) + # cross attention + x2 = torch.randn(1, 1, 32) position_ids = torch.tensor([[[2], [2]]]) attention_mask = torch.zeros(1, 1, dtype=torch.bool) y2, layer_past = m( @@ -264,10 +269,8 @@ def make_data_glm_self_attention(): layer_past=layer_past, use_cache=True, ) - print("x2", x2.flatten()) - print("y2", y2.flatten()) - x3 = torch.randn(1, 1, 16) + x3 = torch.randn(1, 1, 32) position_ids = torch.tensor([[[2], [3]]]) attention_mask = torch.zeros(1, 1, dtype=torch.bool) y3, layer_past = m( @@ -278,47 +281,29 @@ def make_data_glm_self_attention(): layer_past=layer_past, use_cache=True, ) - print("x3", x3.flatten()) - print("y3", y3.flatten()) + print(m) -def make_data_glm_block(): - from modeling_chatglm import GLMBlock - - m = ( - GLMBlock(hidden_size=8, num_attention_heads=2, layernorm_epsilon=1e-5, layer_id=3, empty_init=False) - .float() - .eval() - ) - x = torch.randn(4, 1, 8) # [seqlen, bs, hidden] - position_ids = torch.tensor([[[0, 1, 2, 2], [0, 0, 0, 1]]]) - attention_mask = torch.tensor( - [ - [ - [ - [0, 0, 0, 1], - [0, 0, 0, 1], - [0, 0, 0, 1], - [0, 0, 0, 0], - ] - ] - ] - ).bool() - (y,) = m(x, position_ids=position_ids, attention_mask=attention_mask, layer_id=m.layer_id) - print("x", x.flatten()) - print("input_layernorm.weight", m.input_layernorm.weight.data.flatten()) - print("input_layernorm.bias", m.input_layernorm.bias.data.flatten()) - print("query_key_value.weight", m.attention.query_key_value.weight.data.flatten()) - print("query_key_value.bias", m.attention.query_key_value.bias.data.flatten()) - print("dense.weight", m.attention.dense.weight.data.flatten()) - print("dense.bias", m.attention.dense.bias.data.flatten()) - print("post_attention_layernorm.weight", m.post_attention_layernorm.weight.data.flatten()) - print("post_attention_layernorm.bias", m.post_attention_layernorm.bias.data.flatten()) - print("dense_h_to_4h.weight", m.mlp.dense_h_to_4h.weight.data.flatten()) - print("dense_h_to_4h.bias", m.mlp.dense_h_to_4h.bias.data.flatten()) - print("dense_4h_to_h.weight", m.mlp.dense_4h_to_h.weight.data.flatten()) - print("dense_4h_to_h.bias", m.mlp.dense_4h_to_h.bias.data.flatten()) - print("y", y.flatten()) + with open(HERE / "data/glm_block.data", "wb") as f: + m.input_layernorm.weight.data.numpy().tofile(f) + m.input_layernorm.bias.data.numpy().tofile(f) + m.attention.query_key_value.weight.data.numpy().tofile(f) + m.attention.query_key_value.bias.data.numpy().tofile(f) + m.attention.dense.weight.data.numpy().tofile(f) + m.attention.dense.bias.data.numpy().tofile(f) + m.post_attention_layernorm.weight.data.numpy().tofile(f) + m.post_attention_layernorm.bias.data.numpy().tofile(f) + m.mlp.dense_h_to_4h.weight.data.numpy().tofile(f) + m.mlp.dense_h_to_4h.bias.data.numpy().tofile(f) + m.mlp.dense_4h_to_h.weight.data.numpy().tofile(f) + m.mlp.dense_4h_to_h.bias.data.numpy().tofile(f) + + x1.numpy().tofile(f) + y1.data.numpy().tofile(f) + x2.numpy().tofile(f) + y2.data.numpy().tofile(f) + x3.numpy().tofile(f) + y3.data.numpy().tofile(f) def make_data_glm2_block(): @@ -327,8 +312,8 @@ def make_data_glm2_block(): config = AutoConfig.from_pretrained(CHATGLM2_MODEL_PATH, trust_remote_code=True) config.layernorm_epsilon = 1e-6 - config.hidden_size = 16 - config.num_attention_heads = 4 + config.hidden_size = 32 + config.num_attention_heads = 8 config.multi_query_group_num = 2 config.ffn_hidden_size = 6 config.kv_channels = config.hidden_size // config.num_attention_heads @@ -343,45 +328,52 @@ def make_data_glm2_block(): rotary_pos_emb = rotary_pos_emb_module(8)[None, :seq_length].transpose(0, 1).contiguous() # self attention - x = torch.randn(seq_length, 1, config.hidden_size) + x1 = torch.randn(seq_length, 1, config.hidden_size) with torch.no_grad(): - y, kv_cache = m(x, attention_mask=None, rotary_pos_emb=rotary_pos_emb) - - print(m) - - print("input_layernorm.weight", m.input_layernorm.weight.data.flatten()) - print("attn.qkv.weight", m.self_attention.query_key_value.weight.data.flatten()) - print("attn.qkv.bias", m.self_attention.query_key_value.bias.data.flatten()) - print("attn.dense.weight", m.self_attention.dense.weight.data.flatten()) - print("post_attention_layernorm.weight", m.post_attention_layernorm.weight.data.flatten()) - print("mlp.dense_h_to_4h.weight", m.mlp.dense_h_to_4h.weight.data.flatten()) - print("mlp.dense_4h_to_h.weight", m.mlp.dense_4h_to_h.weight.data.flatten()) - - print("x", x.flatten()) - print("y", y.flatten()) + y1, kv_cache = m(x1, attention_mask=None, rotary_pos_emb=rotary_pos_emb) # cross attention position_ids = torch.tensor([[seq_length]]) rotary_pos_emb = rotary_pos_emb_module(8)[position_ids].transpose(0, 1).contiguous() - x = torch.randn(1, 1, config.hidden_size) + x2 = torch.randn(1, 1, config.hidden_size) with torch.no_grad(): - y, kv_cache = m(x, attention_mask=None, rotary_pos_emb=rotary_pos_emb, kv_cache=kv_cache) - print("x2", x.flatten()) - print("y2", y.flatten()) + y2, kv_cache = m(x2, attention_mask=None, rotary_pos_emb=rotary_pos_emb, kv_cache=kv_cache) + # cross attention position_ids = torch.tensor([[seq_length + 1]]) rotary_pos_emb = rotary_pos_emb_module(8)[position_ids].transpose(0, 1).contiguous() - x = torch.randn(1, 1, config.hidden_size) + x3 = torch.randn(1, 1, config.hidden_size) with torch.no_grad(): - y, kv_cache = m(x, attention_mask=None, rotary_pos_emb=rotary_pos_emb, kv_cache=kv_cache) - print("x3", x.flatten()) - print("y3", y.flatten()) + y3, kv_cache = m(x3, attention_mask=None, rotary_pos_emb=rotary_pos_emb, kv_cache=kv_cache) + + print(m) + + with open(HERE / "data/glm2_block.data", "wb") as f: + m.input_layernorm.weight.data.numpy().tofile(f) + m.self_attention.query_key_value.weight.data.numpy().tofile(f) + m.self_attention.query_key_value.bias.data.numpy().tofile(f) + m.self_attention.dense.weight.data.numpy().tofile(f) + m.post_attention_layernorm.weight.data.numpy().tofile(f) + m.mlp.dense_h_to_4h.weight.data.numpy().tofile(f) + m.mlp.dense_4h_to_h.weight.data.numpy().tofile(f) + + x1.numpy().tofile(f) + y1.numpy().tofile(f) + x2.numpy().tofile(f) + y2.numpy().tofile(f) + x3.numpy().tofile(f) + y3.numpy().tofile(f) def main(): - sys.path.append(str(CHATGLM_MODEL_PATH)) + sys.path.append(str(CHATGLM2_MODEL_PATH)) torch.manual_seed(0) - make_data_glm2_block() + (HERE / "data").mkdir(parents=True, exist_ok=True) + # make_data_linear() + make_data_layernorm() + # make_data_rms_norm() + # make_data_glm_block() + # make_data_glm2_block() if __name__ == "__main__":