model : support LongCat-Flash-Lite (ngram embeddings)

conversion/__init__.py

@@ -56,6 +56,8 @@
     "Qwen3DSparkModel": "qwen",
     "DeepseekV4ForCausalLM": "deepseek",
     "DeepseekV4DSparkModel": "deepseek",
+    "LongcatFlashForCausalLM": "deepseek",
+    "LongcatFlashNgramForCausalLM": "deepseek",
     "DistilBertForMaskedLM": "bert",
     "DistilBertForSequenceClassification": "bert",
     "DistilBertModel": "bert",

conversion/base.py

@@ -1345,15 +1345,15 @@
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
         vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))  # ty: ignore[unresolved-attribute]
         assert max(tokenizer.vocab.values()) < vocab_size  # ty: ignore[unresolved-attribute]
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
         reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}  # ty: ignore[unresolved-attribute]
         added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
 
         added_tokens_decoder = tokenizer.added_tokens_decoder  # ty: ignore[unresolved-attribute]
 
         for i in range(vocab_size):
             if i not in reverse_vocab:
@@ -1366,7 +1366,7 @@
                     # To avoid unexpected issues - we make sure to normalize non-normalized tokens
                     if not added_tokens_decoder[i].normalized:
                         previous_token = token
                         token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))  # ty: ignore[unresolved-attribute, invalid-assignment]
                         if previous_token != token:
                             logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
 
@@ -1685,6 +1685,9 @@
         if chkhsh == "972da7b59cec44d1f0a490a86c96df53859e486e481563e5dddac155013d87ac":
             # ref: https://huggingface.co/poolside/Laguna-XS.2
             res = "laguna"
+        if chkhsh == "27d87c17bcffe5262a1e80b2ceb9a5e002c4f8a17d796fd5afac9180dd8bd96e":
+            # ref: https://huggingface.co/meituan-longcat/LongCat-Flash-Chat
+            res = "longcat-flash"
 
         if res is None:
             logger.warning("\n")
@@ -1733,14 +1736,14 @@
     def _set_vocab_hybriddna(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
         vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))  # ty: ignore[unresolved-attribute]
         assert max(tokenizer.vocab.values()) < vocab_size  # ty: ignore[unresolved-attribute]
 
         reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}  # ty: ignore[unresolved-attribute]
         # k-mers can share text with a base-vocab BPE token (e.g. CCCCCC) and get
         # dropped by get_vocab(); a reserved marker suffix (U+E000) keeps each
         # k-mer's own id (llama.cpp strips it on detokenization)
         for kmer in tokenizer.kmers:  # ty: ignore[unresolved-attribute]
             reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000"  # ty: ignore[unresolved-attribute]
         added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
         added_tokens_decoder = tokenizer.added_tokens_decoder  # ty: ignore[unresolved-attribute]

conversion/deepseek.py

@@ -456,6 +456,162 @@ def prepare_tensors(self):
                 raise ValueError(f"Unprocessed experts: {experts}")
 
 
+@ModelBase.register("LongcatFlashForCausalLM")
+class LongcatFlashModel(DeepseekV2Model):
+    model_arch = gguf.MODEL_ARCH.LONGCAT_FLASH
+    merge_expert = True
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 2 attention+dense-ffn sub-blocks per HF layer -> 2 llama.cpp blocks per HF layer
+        self.block_count = self.hparams["num_layers"] * 2
+        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+        # compat with the DeepseekV2Model hparam names this class reads from
+        self.hparams["num_hidden_layers"]     = self.block_count
+        self.hparams["intermediate_size"]     = self.hparams["ffn_hidden_size"]
+        self.hparams["moe_intermediate_size"] = self.hparams["expert_ffn_hidden_size"]
+        self.hparams["num_experts_per_tok"]   = self.hparams["moe_topk"]
+        # modify_tensors() needs this to split kv_b_proj; set_gguf_parameters() later
+        # overwrites it to 1 for the GGUF MLA metadata.
+        self.hparams["num_key_value_heads"]   = self.hparams["num_attention_heads"]
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, _ = item
+        # MTP speculative decoding head, not supported yet
+        if name.startswith("model.mtp."):
+            return None
+        return super().filter_tensors(item)
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+
+        zero_expert_num  = self.hparams["zero_expert_num"]
+        zero_expert_type = self.hparams["zero_expert_type"]
+        assert zero_expert_type == "identity", "cpp implementation only supports the 'identity' zero expert type"
+        self.gguf_writer.add_n_zero_experts(zero_expert_num)
+
+        # fixed MLA lora-rank scale factors, applied at inference time.
+        # Not folded into the weights: the multiplier is large enough to hurt quantization.
+        if self.hparams.get("mla_scale_q_lora"):
+            self.gguf_writer.add_q_lora_scale((self.hparams["hidden_size"] / self.hparams["q_lora_rank"]) ** 0.5)
+        if self.hparams.get("mla_scale_kv_lora"):
+            self.gguf_writer.add_kv_lora_scale((self.hparams["hidden_size"] / self.hparams["kv_lora_rank"]) ** 0.5)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if bid is not None:
+            # doubled sub-components, e.g. model.layers.{L}.self_attn.{0,1}.q_a_proj.weight
+            match = re.match(r"model\.layers\.\d+\.(mlps|self_attn|input_layernorm|post_attention_layernorm)\.(\d+)\.(.*)", name)
+            if match:
+                sub_idx = int(match.group(2))
+                new_bid = bid * 2 + sub_idx
+                mid = "mlp" if match.group(1) == "mlps" else match.group(1)
+                new_name = f"model.layers.{new_bid}.{mid}.{match.group(3)}"
+                yield from super().modify_tensors(data_torch, new_name, new_bid)
+                return
+
+            # shared MoE (mlp.router.*, mlp.experts.*), no sub-index, attaches to the even block
+            new_bid = bid * 2
+            new_name = name.replace(f"model.layers.{bid}.", f"model.layers.{new_bid}.", 1)
+            for out_name, out_tensor in super().modify_tensors(data_torch, new_name, new_bid):
+                if out_name.endswith("_exps.weight"):
+                    # append a dummy all-zero expert, zero-computation experts route to it
+                    out_tensor = torch.cat([out_tensor, torch.zeros_like(out_tensor[:1])], dim=0)
+                yield out_name, out_tensor
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("LongcatFlashNgramForCausalLM")
+class LongcatNgramModel(LongcatFlashModel):
+    model_arch = gguf.MODEL_ARCH.LONGCAT_NGRAM
+
+    # the reference builds k*(n-1) independent hash tables, each with its own vocab size and its
+    # own [emb_dim -> hidden_size] projection, then sums all the projected lookups into the token
+    # embedding. Instead of emitting 2*k*(n-1) tiny tensors we merge them into two:
+    #
+    #   ngram_embd: all tables padded to the largest vocab size and stacked into one lookup table,
+    #               so the whole n-gram lookup is a single get_rows() over a flat row index
+    #   ngram_proj: sum_i P_i @ e_i == [P_0 | ... | P_{m-1}] @ concat(e_0, ..., e_{m-1}), so the
+    #               per-table projections concatenate into one [emb_dim*m, hidden_size] matmul
+    #
+    # see build_inp_ngram_embd() on the C++ side
+    _ngram_embedders: dict[int, Tensor]
+    _ngram_projs: dict[int, Tensor]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._ngram_embedders = {}
+        self._ngram_projs = {}
+
+        n = self.hparams["emb_neighbor_num"]
+        k = self.hparams["emb_split_num"]
+        self.ngram_n = n
+        self.ngram_k = k
+        self.ngram_count = k * (n - 1)
+
+        # int(ngram_vocab_size_ratio * vocab_size + 2*i + 1), matching NgramEmbedding
+        m = self.hparams["ngram_vocab_size_ratio"] * self.hparams["vocab_size"]
+        self.ngram_vocab_sizes = [int(m + i * 2 + 1) for i in range(self.ngram_count)]
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+
+        self.gguf_writer.add_ngram_neighbor_count(self.ngram_n)
+        self.gguf_writer.add_ngram_split_count(self.ngram_k)
+        self.gguf_writer.add_ngram_vocab_sizes(self.ngram_vocab_sizes)
+
+        # the n-gram context is reset at every EOS, see _shift_right_ignore_eos
+        eos_token_id = self.hparams["eos_token_id"]
+        assert isinstance(eos_token_id, int), "n-gram segmentation needs a single EOS token id"
+        self.gguf_writer.add_ngram_eos_token_id(eos_token_id)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # note: bid is bogus for these, it picks up the embedder index
+        if (match := re.match(r"model\.ngram_embeddings\.(embedders|post_projs)\.(\d+)\.weight", name)):
+            idx = int(match.group(2))
+            assert idx < self.ngram_count, f"unexpected n-gram index {idx} in {name}"
+
+            if match.group(1) == "embedders":
+                assert data_torch.shape[0] == self.ngram_vocab_sizes[idx], \
+                    f"{name}: expected {self.ngram_vocab_sizes[idx]} rows, got {data_torch.shape[0]}"
+                self._ngram_embedders[idx] = data_torch
+            else:
+                self._ngram_projs[idx] = data_torch
+
+            if len(self._ngram_embedders) == self.ngram_count and len(self._ngram_projs) == self.ngram_count:
+                yield from self._merge_ngram_tensors()
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+    def _merge_ngram_tensors(self) -> Iterable[tuple[str, Tensor]]:
+        embedders = [self._ngram_embedders.pop(i) for i in range(self.ngram_count)]
+        projs = [self._ngram_projs.pop(i) for i in range(self.ngram_count)]
+
+        emb_dim = embedders[0].shape[1]
+        assert all(e.shape[1] == emb_dim for e in embedders)
+        assert all(p.shape == (self.hparams["hidden_size"], emb_dim) for p in projs)
+
+        # pad every table to the largest one so the C++ side can index with a constant stride
+        stride = max(self.ngram_vocab_sizes)
+        padded = [
+            torch.cat([e, e.new_zeros(stride - e.shape[0], emb_dim)]) if e.shape[0] < stride else e
+            for e in embedders
+        ]
+
+        yield self.format_tensor_name(gguf.MODEL_TENSOR.NGRAM_EMBD), torch.cat(padded, dim=0)
+        yield self.format_tensor_name(gguf.MODEL_TENSOR.NGRAM_PROJ), torch.cat(projs, dim=1)
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+
+        if self._ngram_embedders or self._ngram_projs:
+            raise ValueError(f"Unprocessed n-gram tensors: {sorted(self._ngram_embedders)} {sorted(self._ngram_projs)}")
+
+
 @ModelBase.register("DeepseekV32ForCausalLM")
 class DeepseekV32Model(DeepseekV2Model):
     model_arch = gguf.MODEL_ARCH.DEEPSEEK32

convert_hf_to_gguf_update.py

@@ -163,6 +163,7 @@ class TOKENIZER_TYPE(IntEnum):
     {"name": "granite-embed-multi-311m", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ibm-granite/granite-embedding-311m-multilingual-r2", },
     {"name": "mellum2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Base"},
     {"name": "laguna",           "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/poolside/Laguna-XS.2", },
+    {"name": "longcat-flash",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meituan-longcat/LongCat-Flash-Chat", },
 ]
 
 # some models are known to be broken upstream, so we will skip them as exceptions

gguf-py/gguf/constants.py

@@ -158,6 +158,7 @@ class LLM:
         HIDDEN_ACT                        = "{arch}.hidden_activation"
         DENSE_FEAT_IN_SIZE                = "{arch}.{dense}_feat_in"
         DENSE_FEAT_OUT_SIZE               = "{arch}.{dense}_feat_out"
+        N_ZERO_EXPERTS                    = "{arch}.n_zero_experts" # longcat-flash
         TARGET_LAYERS                     = "{arch}.target_layers"
         TARGET_HIDDEN_SIZE                = "{arch}.target_hidden_size"
         BLOCK_SIZE                        = "{arch}.block_size"
@@ -178,6 +179,8 @@ class Attention:
         CAUSAL                       = "{arch}.attention.causal"
         Q_LORA_RANK                  = "{arch}.attention.q_lora_rank"
         KV_LORA_RANK                 = "{arch}.attention.kv_lora_rank"
+        Q_LORA_SCALE                 = "{arch}.attention.q_lora_scale"
+        KV_LORA_SCALE                = "{arch}.attention.kv_lora_scale"
         DECAY_LORA_RANK              = "{arch}.attention.decay_lora_rank"
         ICLR_LORA_RANK               = "{arch}.attention.iclr_lora_rank"
         VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"
@@ -307,6 +310,13 @@ class Tokenizer:
         SUFFIX_ID            = "tokenizer.ggml.suffix_token_id"
         MIDDLE_ID            = "tokenizer.ggml.middle_token_id"
 
+    class NGram:
+        # hash-based n-gram input embeddings, see longcat-flash-ngram
+        NEIGHBOR_COUNT = "{arch}.ngram.neighbor_count"
+        SPLIT_COUNT    = "{arch}.ngram.split_count"
+        VOCAB_SIZES    = "{arch}.ngram.vocab_sizes"
+        EOS_TOKEN_ID   = "{arch}.ngram.eos_token_id"
+
     class Adapter:
         TYPE                    = "adapter.type"
         LORA_ALPHA              = "adapter.lora.alpha"
@@ -567,6 +577,8 @@ class MODEL_ARCH(IntEnum):
     STEP35           = auto()
     LLAMA_EMBED      = auto()
     MAINCODER        = auto()
+    LONGCAT_FLASH    = auto()
+    LONGCAT_NGRAM = auto()
     KIMI_LINEAR      = auto()
     TALKIE           = auto()
     MELLUM           = auto()
@@ -792,6 +804,8 @@ class MODEL_TENSOR(IntEnum):
     POSNET_ATTN_K        = auto()
     POSNET_ATTN_V        = auto()
     POSNET_ATTN_OUT      = auto()
+    NGRAM_EMBD           = auto()
+    NGRAM_PROJ           = auto()
     SHORTCONV_CONV       = auto()
     SHORTCONV_INPROJ     = auto()
     SHORTCONV_OUTPROJ    = auto()
@@ -1239,6 +1253,8 @@ class MODEL_TENSOR(IntEnum):
     MODEL_ARCH.STEP35:           "step35",
     MODEL_ARCH.LLAMA_EMBED:      "llama-embed",
     MODEL_ARCH.MAINCODER:        "maincoder",
+    MODEL_ARCH.LONGCAT_FLASH:    "longcat-flash",
+    MODEL_ARCH.LONGCAT_NGRAM:    "longcat-ngram",
     MODEL_ARCH.KIMI_LINEAR:      "kimi-linear",
     MODEL_ARCH.TALKIE:           "talkie",
     MODEL_ARCH.MELLUM:           "mellum",
@@ -1462,6 +1478,8 @@ class MODEL_TENSOR(IntEnum):
     MODEL_TENSOR.POSNET_ATTN_K:             "posnet.{bid}.attn_k",
     MODEL_TENSOR.POSNET_ATTN_V:             "posnet.{bid}.attn_v",
     MODEL_TENSOR.POSNET_ATTN_OUT:           "posnet.{bid}.attn_output",
+    MODEL_TENSOR.NGRAM_EMBD:                "ngram_embd",
+    MODEL_TENSOR.NGRAM_PROJ:                "ngram_proj",
     MODEL_TENSOR.SHORTCONV_CONV:            "blk.{bid}.shortconv.conv",
     MODEL_TENSOR.SHORTCONV_INPROJ:          "blk.{bid}.shortconv.in_proj",
     MODEL_TENSOR.SHORTCONV_OUTPROJ:         "blk.{bid}.shortconv.out_proj",
@@ -3418,6 +3436,54 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
         MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
     ],
+    MODEL_ARCH.LONGCAT_FLASH: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q_A,
+        MODEL_TENSOR.ATTN_Q_B,
+        MODEL_TENSOR.ATTN_KV_A_MQA,
+        MODEL_TENSOR.ATTN_K_B,
+        MODEL_TENSOR.ATTN_V_B,
+        MODEL_TENSOR.ATTN_Q_A_NORM,
+        MODEL_TENSOR.ATTN_KV_A_NORM,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+    ],
+    MODEL_ARCH.LONGCAT_NGRAM: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.NGRAM_EMBD,
+        MODEL_TENSOR.NGRAM_PROJ,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q_A,
+        MODEL_TENSOR.ATTN_Q_B,
+        MODEL_TENSOR.ATTN_KV_A_MQA,
+        MODEL_TENSOR.ATTN_K_B,
+        MODEL_TENSOR.ATTN_V_B,
+        MODEL_TENSOR.ATTN_Q_A_NORM,
+        MODEL_TENSOR.ATTN_KV_A_NORM,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+    ],
     MODEL_ARCH.DEEPSEEK2OCR: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,

gguf-py/gguf/gguf_writer.py

@@ -942,6 +942,12 @@ def add_q_lora_rank(self, length: int) -> None:
     def add_kv_lora_rank(self, length: int) -> None:
         self.add_uint32(Keys.Attention.KV_LORA_RANK.format(arch=self.arch), length)
 
+    def add_q_lora_scale(self, value: float) -> None:
+        self.add_float32(Keys.Attention.Q_LORA_SCALE.format(arch=self.arch), value)
+
+    def add_kv_lora_scale(self, value: float) -> None:
+        self.add_float32(Keys.Attention.KV_LORA_SCALE.format(arch=self.arch), value)
+
     def add_decay_lora_rank(self, length: int) -> None:
         self.add_uint32(Keys.Attention.DECAY_LORA_RANK.format(arch=self.arch), length)
 
@@ -1199,6 +1205,23 @@ def add_eom_token_id(self, id: int) -> None:
     def add_classifier_output_labels(self, labels: Sequence[str]) -> None:
         self.add_array(Keys.Classifier.OUTPUT_LABELS.format(arch=self.arch), labels)
 
+    def add_n_zero_experts(self, n: int) -> None:
+        self.add_uint32(Keys.LLM.N_ZERO_EXPERTS.format(arch=self.arch), n)
+
+    # for n-gram input embeddings
+
+    def add_ngram_neighbor_count(self, n: int) -> None:
+        self.add_uint32(Keys.NGram.NEIGHBOR_COUNT.format(arch=self.arch), n)
+
+    def add_ngram_split_count(self, n: int) -> None:
+        self.add_uint32(Keys.NGram.SPLIT_COUNT.format(arch=self.arch), n)
+
+    def add_ngram_vocab_sizes(self, sizes: Sequence[int]) -> None:
+        self.add_array(Keys.NGram.VOCAB_SIZES.format(arch=self.arch), sizes)
+
+    def add_ngram_eos_token_id(self, id: int) -> None:
+        self.add_uint32(Keys.NGram.EOS_TOKEN_ID.format(arch=self.arch), id)
+
     # for vision models
 
     def add_clip_has_vision_encoder(self, value: bool) -> None:

gguf-py/gguf/tensor_mapping.py

@@ -462,6 +462,7 @@ class TensorNameMap:
             "backbone.layers.{bid}.mixer.gate",                 # nemotron-h-moe
             "model.layers.{bid}.moe.gate",                      # step3.5
             "model.layers.{bid}.router.proj",                   # gemma4
+            "model.layers.{bid}.mlp.router.classifier",         # longcat-flash
         ),
 
         MODEL_TENSOR.FFN_GATE_INP_SHEXP: (
@@ -480,6 +481,7 @@ class TensorNameMap:
             "model.layers.{bid}.block_sparse_moe.gate.e_score_correction",  # kimi
             "model.layers.{bid}.moe.router_bias",                           # step3.5 expert selection bias
             "model.layers.{bid}.mlp.experts.e_score_correction",       # laguna
+            "model.layers.{bid}.mlp.router.e_score_correction",        # longcat-flash
         ),
 
         # Feed-forward up