Repository navigation
Add Kimi Linear to unified delta net #19668
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 10 commits
fa44de7
403e78e
2371dfb
cff8f60
b0594c9
6c765eb
a93bcc4
7b26805
4a6393e
c07977a
117763e
6dad437
df269dc
1cea24d
a6fa6d5
6432f95
de6a842
23cccea
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -27,6 +27,7 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
|
|
||
| const int64_t S_v = v->ne[0]; | ||
| const int64_t H_v = v->ne[1]; | ||
| const bool kda = (g->ne[0] == S_k && g->ne[1] == H_k); | ||
|
|
||
| GGML_ASSERT(S_k == S_v); | ||
| GGML_ASSERT(H_v % H_k == 0); | ||
|
|
@@ -35,9 +36,10 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs); | ||
| GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs); | ||
|
|
||
| GGML_ASSERT(g->ne[0] == H_v && g->ne[1] == n_tokens && g->ne[2] == n_seqs); | ||
| GGML_ASSERT(b->ne[0] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); | ||
| GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); | ||
| GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v); | ||
| GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs); | ||
| GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); | ||
| GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); | ||
|
|
||
| const float scale = 1.0f / sqrtf(S_k); | ||
|
|
||
|
|
@@ -52,8 +54,8 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] | ||
| k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs] | ||
| v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs] | ||
| g = ggml_permute(ctx0, g, 2, 1, 3, 0); // [ 1, n_tokens, H_v, n_seqs] | ||
| b = ggml_permute(ctx0, b, 2, 0, 1, 3); // [ 1, n_tokens, H_v, n_seqs] | ||
| g = ggml_permute(ctx0, g, 0, 2, 1, 3); // [g_0, n_tokens, H_v, n_seqs] | ||
| b = ggml_permute(ctx0, b, 0, 2, 1, 3); // [ 1, n_tokens, H_v, n_seqs] | ||
|
|
||
| const int CS = CHUNK_SIZE; | ||
|
|
||
|
|
@@ -78,33 +80,77 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| v = ggml_reshape_4d(ctx0, v, S_v, CS, n_chunks, H_v * n_seqs); | ||
| v_b = ggml_reshape_4d(ctx0, v_b, S_v, CS, n_chunks, H_v * n_seqs); | ||
|
|
||
| g = ggml_reshape_4d(ctx0, g, CS, 1, n_chunks, H_v * n_seqs); | ||
| b = ggml_reshape_4d(ctx0, b, 1, CS, n_chunks, H_v * n_seqs); | ||
| g = ggml_reshape_4d(ctx0, g, g->ne[0], CS, n_chunks, H_v * n_seqs); | ||
| b = ggml_reshape_4d(ctx0, b, 1, CS, n_chunks, H_v * n_seqs); | ||
|
|
||
| // [CS, 1, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * g_cs = ggml_cumsum(ctx0, g); | ||
| // [CS, g_0, n_chunks, H_v * n_seqs] | ||
| // TODO: extend ggml_cumsum with axis parameter to avoid transpose | ||
| ggml_tensor * g_cs = ggml_cumsum(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, g))); | ||
| cb(g_cs, "g_cs", il); | ||
|
|
||
| ggml_tensor * g_cs_i = g_cs; | ||
| ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, n_chunks, H_v * n_seqs); | ||
|
|
||
| g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, n_chunks, H_v * n_seqs); | ||
|
|
||
| // [CS, CS, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * decay_mask; | ||
| decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i); | ||
| decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG); | ||
| decay_mask = ggml_exp(ctx0, decay_mask); | ||
| cb(decay_mask, "decay_mask", il); | ||
|
|
||
| // [CS, CS, n_chunks, H_k * n_seqs] | ||
| ggml_tensor * kb; | ||
| kb = ggml_mul_mat(ctx0, k, k_b); | ||
| kb = ggml_mul (ctx0, kb, decay_mask); | ||
| ggml_tensor * kb = nullptr; | ||
| ggml_tensor * kq = nullptr; | ||
| if (kda) { | ||
| const int64_t CHB = n_chunks * H_k * n_seqs; | ||
| ggml_tensor * g_cs_i = ggml_reshape_4d(ctx0, g_cs, CS, 1, S_k, CHB); // [chunk_size, 1, S_k, CHB] | ||
| ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] | ||
|
|
||
| g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] -> [chunk_size, chunk_size, S_k, CHB] | ||
| // decay_mask [chunk_size,chunk_size,S_k,CHB] | ||
| ggml_tensor * decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i); | ||
| cb(decay_mask, "decay_mask", il); | ||
|
|
||
| decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG); | ||
| decay_mask = ggml_exp(ctx0, decay_mask); | ||
| cb(decay_mask, "decay_masked", il); | ||
|
|
||
| // decay_mask [S_k,BT_j,BT_i,CHB] *Note* second and third chunk_sizes are switched | ||
| decay_mask = ggml_cont_4d(ctx0, ggml_permute(ctx0, decay_mask, 2, 1, 0, 3), S_k, CS, CS, CHB); | ||
|
|
||
| ggml_tensor * k_b_i = ggml_reshape_4d(ctx0, k_b, S_k, CS, 1, CHB); | ||
| ggml_tensor * k_j = ggml_reshape_4d(ctx0, k, S_k, 1, CS, CHB); | ||
| ggml_tensor * q_i = ggml_reshape_4d(ctx0, q, S_k, CS, 1, CHB); | ||
|
|
||
| ggml_tensor * decay_k_b_i = ggml_mul(ctx0, decay_mask, k_b_i); | ||
| ggml_tensor * decay_q_i = ggml_mul(ctx0, decay_mask, q_i); | ||
|
|
||
| // decay_k_i [S.BT,BT,CHB] @ k_j [S,1,BT,CHB] = Akk [BT,1,BT,CHB] | ||
| kb = ggml_mul_mat(ctx0, decay_k_b_i, k_j); | ||
| kq = ggml_mul_mat(ctx0, decay_q_i, k_j); | ||
|
|
||
| kb = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kb, CS, CS, n_chunks, H_v * n_seqs))); | ||
| kq = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kq, CS, CS, n_chunks, H_v * n_seqs))); | ||
|
|
||
| kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG); | ||
| cb(kq, "kq", il); | ||
| } else { | ||
| ggml_tensor * g_cs_i = g_cs; | ||
| ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, n_chunks, H_v * n_seqs); | ||
|
|
||
| g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, n_chunks, H_v * n_seqs); | ||
|
|
||
| // [CS, CS, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * decay_mask; | ||
| decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i); | ||
| decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG); | ||
| decay_mask = ggml_exp(ctx0, decay_mask); | ||
| cb(decay_mask, "decay_mask", il); | ||
|
|
||
| // [CS, CS, n_chunks, H_k * n_seqs] | ||
| kb = ggml_mul_mat(ctx0, k, k_b); | ||
| kb = ggml_mul (ctx0, kb, decay_mask); | ||
|
|
||
| // [CS, CS, n_chunks, H_k * n_seqs] | ||
| kq = ggml_mul_mat(ctx0, k, q); | ||
| kq = ggml_mul(ctx0, kq, decay_mask); | ||
| kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG); | ||
| cb(kq, "kq", il); | ||
| } | ||
|
|
||
| // [CS, CS, n_chunks, H_k * n_seqs] | ||
| ggml_tensor * attn; | ||
| attn = ggml_tri(ctx0, kb, GGML_TRI_TYPE_LOWER); | ||
| cb(attn, "attn", il); | ||
|
|
||
| ggml_tensor * identity; | ||
| identity = ggml_view_1d(ctx0, attn, CS, 0); | ||
|
|
@@ -115,6 +161,7 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| cb(lhs, "dnet_add_ch_lhs", il); | ||
|
|
||
| attn = ggml_neg(ctx0, attn); | ||
| cb(attn, "attn_pre_solve", il); | ||
|
|
||
| ggml_tensor * lin_solve = ggml_solve_tri(ctx0, lhs, attn, true, true, false); | ||
| attn = ggml_add(ctx0, lin_solve, identity); | ||
|
|
@@ -123,7 +170,7 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| // [S_v, CS, n_chunks, H_v * n_seqs] | ||
| v = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v_b)), attn); | ||
|
|
||
| // [CS, 1, n_chunks, H_v * n_seqs] | ||
| // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * g_exp = ggml_exp(ctx0, g_cs); | ||
|
|
||
| k_b = ggml_cont(ctx0, ggml_transpose(ctx0, k_b)); | ||
|
|
@@ -136,16 +183,10 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| ggml_tensor * k_cd = ggml_mul_mat(ctx0, kbg, attn); | ||
| cb(k_cd, "k_cumdecay", il); | ||
|
|
||
| // [S_k, CS, n_chunks, H_k * n_seqs] | ||
| ggml_tensor * g_exp_t = ggml_transpose(ctx0, g_exp); | ||
| // [1, CS, n_chunks, H_k * n_seqs] KDA: [S_k, CS, n_chunks, H_k * n_seqs] | ||
| ggml_tensor * g_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, g_exp)); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is the CONT needed here for KDA? If so, maybe add it conditionally.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ggml_cont not needed here. So it is removed now.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Most We can make the |
||
| ggml_tensor * q_g_exp = ggml_mul(ctx0, q, g_exp_t); | ||
|
|
||
| // [CS, CS, n_chunks, H_k * n_seqs] | ||
| ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); | ||
| kq = ggml_mul(ctx0, kq, decay_mask); | ||
| kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG); | ||
| cb(kq, "kq", il); | ||
|
|
||
| // vectorized calculation of key_gdiff | ||
| // improved from the chunked version: | ||
| // g_last = torch.clamp(g_cum[:, :, -1], max=50.0).exp().unsqueeze(-1).unsqueeze(-1) | ||
|
|
@@ -156,8 +197,8 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
|
|
||
| // get last element in g_cumsum along CS dimension (ne0) | ||
| // example: [[x, y, z, ..., last], ...] -> [[last], ...] | ||
| // [1, 1, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * g_last = ggml_view_4d(ctx0, g_cs, 1, 1, g_cs->ne[2], g_cs->ne[3], | ||
| // [1, 1, n_chunks, H_v * n_seqs] KDA: [1, S_k, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * g_last = ggml_view_4d(ctx0, g_cs, 1, g_cs->ne[1], g_cs->ne[2], g_cs->ne[3], | ||
| g_cs->nb[1], | ||
| g_cs->nb[2], | ||
| g_cs->nb[3], | ||
|
|
@@ -167,16 +208,15 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| // TODO: remove this cont when CUDA supports non-cont unary ops | ||
| g_last = ggml_cont(ctx0, g_last); | ||
|
|
||
| // [1, 1, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * g_last_exp = ggml_exp(ctx0, g_last); | ||
| cb(g_last_exp, "g_last_exp", il); | ||
| // [1, 1, n_chunks, H_v * n_seqs] KDA: [S_k, 1, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * g_last_exp_t = ggml_transpose(ctx0, ggml_exp(ctx0, g_last)); | ||
| cb(g_last_exp_t, "g_last_exp_t", il); | ||
|
|
||
| // [CS, 1, n_chunks, H_v * n_seqs] | ||
| // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * g_diff = ggml_neg(ctx0, ggml_sub(ctx0, g_cs, g_last)); | ||
| cb(g_diff, "g_diff", il); | ||
|
|
||
| ggml_tensor * g_diff_exp = ggml_exp(ctx0, g_diff); | ||
| ggml_tensor * g_diff_exp_t = ggml_transpose(ctx0, g_diff_exp); | ||
| ggml_tensor * g_diff_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_exp(ctx0, g_diff))); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ditto.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ggml_cont not needed here. So it is removed now. |
||
|
|
||
| // [S_k, CS, n_chunks, H_v * n_seqs] | ||
| ggml_tensor * kg = ggml_mul(ctx0, k, g_diff_exp_t); | ||
|
|
@@ -227,8 +267,9 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| ggml_tensor * kgv = ggml_mul_mat(ctx0, ch_kg_t, v_t_new); // [S_k, S_v, 1, H_k * n_seqs] | ||
|
|
||
| // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew | ||
| ggml_tensor * ch_g_last_exp = get_slice_2d(ctx0, g_last_exp, chunk); | ||
| s_t = ggml_mul(ctx0, s_t, ch_g_last_exp); | ||
| ggml_tensor * ch_g_last_exp_t = get_slice_2d(ctx0, g_last_exp_t, chunk); | ||
|
|
||
| s_t = ggml_mul(ctx0, s_t, ch_g_last_exp_t); | ||
| s_t = ggml_add(ctx0, s_t, kgv); | ||
| cb(s_t, "dnet_add_ch_state", il); | ||
| } | ||
|
|
@@ -241,9 +282,9 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| ggml_row_size(v->type, S_v), | ||
| ggml_row_size(v->type, S_v * CS * n_chunks), | ||
| ggml_row_size(v->type, S_v * CS * n_chunks * H_v), 0); | ||
|
|
||
| o = ggml_permute (ctx0, o, 0, 2, 1, 3); // [S_v, H_v, n_tokens, n_seqs] | ||
| s = ggml_transpose(ctx0, s_t); // [S_v, S_v, H_v, n_seqs] | ||
| s = ggml_transpose(ctx0, s_t); | ||
| cb(s, "output_state", il); | ||
|
|
||
| return {o, s}; | ||
| } | ||
|
|
@@ -273,9 +314,10 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs); | ||
| GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs); | ||
|
|
||
| GGML_ASSERT(g->ne[0] == H_v && g->ne[1] == n_tokens && g->ne[2] == n_seqs); | ||
| GGML_ASSERT(b->ne[0] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); | ||
| GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); | ||
| GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v); | ||
| GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs); | ||
| GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs); | ||
| GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs); | ||
|
|
||
| const float scale = 1.0f / sqrtf(S_k); | ||
|
|
||
|
|
@@ -291,8 +333,10 @@ std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_ne | |
| cb(b, "b_in", il); | ||
| cb(g, "g_in", il); | ||
|
|
||
| g = ggml_reshape_4d(ctx0, g, 1, 1, H_v, n_seqs); | ||
| b = ggml_reshape_4d(ctx0, b, 1, 1, H_v, n_seqs); | ||
| // GDA: [1, 1, H_v, n_seqs] | ||
| // KDA: [1, S_k, H_v, n_seqs] | ||
| g = ggml_reshape_4d(ctx0, g, 1, g->ne[0], H_v, n_seqs); | ||
| b = ggml_reshape_4d(ctx0, b, 1, 1, H_v, n_seqs); | ||
|
|
||
| // [S_v, S_v, H_v, n_seqs] | ||
| g = ggml_exp(ctx0, g); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Shouldn't this be:
For the current models, it does not make a difference because
S_k == S_v, but in general, I think the gate has dimensions ofV?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Based on my understanding of the fla code, gate is passed to the gk field of chunk_kda_fwd_intra, so its dimension should be S_k, H_k. This function also support gv that is S_v, H_v but I don't know whether any model is using gv.
https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/kda/chunk_fwd.py
Also, state has a dimension of [S_v, S_k, H_v, n_seqs]. That's why when I ggml_mul with it, I need to make sure S_k matches at the 2nd dim.