Skip to content

Dsv4 metal #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tarruda wants to merge 9 commits into from
Closed

Dsv4 metal #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
699fa27
merged ggml-org/llama.cpp#24231 New GGML_OP_LIGHTNING_INDEXER that im…
sszymczy Jun 29, 2026
41303e3
ggml : GGML_OP_LIGHTNING_INDEXER CUDA support
sszymczy Jun 29, 2026
b03cb7a
merged ggml-org/llama.cpp#24776 ggml : process data in smaller chunks…
sszymczy Jun 29, 2026
f9fb34a
model : use lightning indexer OP in DeepSeek V4 implementation
sszymczy Jun 29, 2026
6ad37f4
dsv4 hc-ops
am17an Jul 1, 2026
5a09621
Merge remote-tracking branch 'am17an/hc_ops' into dsv4
sszymczy Jul 1, 2026
51641a7
metal: add GGML_OP_LIGHTNING_INDEXER Metal kernel
tarruda Jul 3, 2026
8b24833
metal: add DeepSeek V4 hierarchical connection Metal kernels
tarruda Jul 3, 2026
32c7633
metal: fix DeepSeek V4 lightning indexer q loads
tarruda Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,10 @@ extern "C" {
GGML_OP_RWKV_WKV7,
GGML_OP_SOLVE_TRI,
GGML_OP_GATED_DELTA_NET,
GGML_OP_LIGHTNING_INDEXER,
GGML_OP_DSV4_HC_COMB,
GGML_OP_DSV4_HC_PRE,
GGML_OP_DSV4_HC_POST,

GGML_OP_UNARY,

Expand Down Expand Up @@ -2573,6 +2577,40 @@ extern "C" {
struct ggml_tensor * state,
int64_t K);

GGML_API struct ggml_tensor * ggml_lightning_indexer(
struct ggml_context * ctx,
struct ggml_tensor * q,
struct ggml_tensor * k,
struct ggml_tensor * weights,
float scale_embd,
float scale_heads);

// DeepSeek V4 hyper-connection helpers.
// hc_comb: mixes [(2 + hc)*hc, n_tokens], scale [3], base [(2 + hc)*hc]
// -> [dst_hc, src_hc, n_tokens]
// hc_pre : x [n_embd, hc, n_tokens], weights [hc, n_tokens] -> [n_embd, n_tokens]
// hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens],
// post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens] -> [n_embd, hc, n_tokens]
GGML_API struct ggml_tensor * ggml_dsv4_hc_comb(
struct ggml_context * ctx,
struct ggml_tensor * mixes,
struct ggml_tensor * scale,
struct ggml_tensor * base,
float eps,
int32_t n_iter);

GGML_API struct ggml_tensor * ggml_dsv4_hc_pre(
struct ggml_context * ctx,
struct ggml_tensor * x,
struct ggml_tensor * weights);

GGML_API struct ggml_tensor * ggml_dsv4_hc_post(
struct ggml_context * ctx,
struct ggml_tensor * x,
struct ggml_tensor * residual,
struct ggml_tensor * post,
struct ggml_tensor * comb);

// custom operators

typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
Expand Down
5 changes: 5 additions & 0 deletions ggml/src/ggml-backend-meta.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -984,6 +984,11 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
case GGML_OP_GATED_DELTA_NET: {
split_state = handle_gated_delta_net(src_ss);
} break;
case GGML_OP_DSV4_HC_COMB:
case GGML_OP_DSV4_HC_PRE:
case GGML_OP_DSV4_HC_POST: {
split_state = handle_generic(src_ss, /*scalar_only =*/ true);
} break;
case GGML_OP_UNARY: {
split_state = handle_generic(src_ss, /*scalar_only =*/ false);
} break;
Expand Down
26 changes: 26 additions & 0 deletions ggml/src/ggml-cpu/ggml-cpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2051,6 +2051,22 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
{
ggml_compute_forward_gated_delta_net(params, tensor);
} break;
case GGML_OP_LIGHTNING_INDEXER:
{
ggml_compute_forward_lightning_indexer(params, tensor);
} break;
case GGML_OP_DSV4_HC_COMB:
{
ggml_compute_forward_dsv4_hc_comb(params, tensor);
} break;
case GGML_OP_DSV4_HC_PRE:
{
ggml_compute_forward_dsv4_hc_pre(params, tensor);
} break;
case GGML_OP_DSV4_HC_POST:
{
ggml_compute_forward_dsv4_hc_post(params, tensor);
} break;
case GGML_OP_MAP_CUSTOM1:
{
ggml_compute_forward_map_custom1(params, tensor);
Expand Down Expand Up @@ -2231,6 +2247,9 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
case GGML_OP_COUNT_EQUAL:
case GGML_OP_SOLVE_TRI:
case GGML_OP_GATED_DELTA_NET:
case GGML_OP_DSV4_HC_COMB:
case GGML_OP_DSV4_HC_PRE:
case GGML_OP_DSV4_HC_POST:
{
n_tasks = n_threads;
} break;
Expand Down Expand Up @@ -2371,6 +2390,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
case GGML_OP_FLASH_ATTN_BACK:
case GGML_OP_SSM_CONV:
case GGML_OP_SSM_SCAN:
case GGML_OP_LIGHTNING_INDEXER:
{
n_tasks = n_threads;
} break;
Expand Down Expand Up @@ -2956,6 +2976,12 @@ struct ggml_cplan ggml_graph_plan(
{
GGML_ABORT("fatal error");
}
case GGML_OP_LIGHTNING_INDEXER:
{
// temp buffer for dequantizing lightning indexer keys
const int64_t ne10 = node->src[1]->ne[0];
cur += sizeof(float)*ne10*n_tasks;
} break;
default:
break;
}
Expand Down
Loading