from transformers.models.longcat_flash import LongcatFlashConfig class LongcatFlashSparseConfig(LongcatFlashConfig): r""" Configuration class for LongCat-Flash-Lite-Sparse. Extends `LongcatFlashConfig` the same way Lite's `LongcatFlashNgramConfig` does, with three differences confirmed against the real checkpoint at meituan-longcat/LongCat-Flash-Lite-Sparse (config.json + model.safetensors.index.json), not assumed from Lite or the paper: 1. N-gram fields were renamed in the checkpoint: `oe_vocab_size_ratio`, `oe_neighbor_num`, `oe_split_num` (same concept as Lite's `ngram_vocab_size_ratio`, `emb_neighbor_num`, `emb_split_num`). Accepted here under their real config.json names, but stored under the ORIGINAL attribute names, so `NgramEmbedding` in modeling_longcat_ngram.py can be reused completely unmodified -- it reads `config.ngram_vocab_size_ratio`, `config.emb_split_num`, `config.emb_neighbor_num` and doesn't need to know the checkpoint key changed. 2. LSA (LongCat Sparse Attention) adds: `index_topk`, `index_n_heads`, `index_head_dim`, `index_k_norm_type` (this is RMSNorm -- weight only, no bias, per the checkpoint -- NOT DeepSeek-V3.2-style LayerNorm+bias, despite similar naming in some reference ports), `index_local_tokens` (SI sliding window, =1024), `index_init_tokens` (SI sink, =16), `cli_factor` (cross-layer indexer-reuse grouping, =2), `dsa_mtp_cli` (whether CLI also applies across the MTP layers), `indexer_rope_interleave`, `attention_method`. 3. MTP is 3 layers deep here, not 1: `mtp_num_layers` (=3), `mtp_replicate_modules`, `mtp_disable_over_tokenizer`. None of these fields exist in Lite's config.json or config class at all -- Lite has an implicit single MTP block. rope_theta / rope_scaling are passed straight through to LongcatFlashConfig as separate top-level fields, matching the *live* config.json on the Hub. Do not switch this to a unified `rope_parameters` dict -- that shape only appears if this config gets loaded and re-saved under a newer transformers install that normalizes RoPE config differently; the raw checkpoint (and this class) use the older shape. """ model_type = "longcat_flash_lite_sparse" keys_to_ignore_at_inference = ["past_key_values"] base_model_tp_plan = { "layers.*.self_attn.*.q_b_proj": "colwise", "layers.*.self_attn.*.kv_b_proj": "colwise", "layers.*.self_attn.*.o_proj": "rowwise", "layers.*.mlps.*.gate_proj": "colwise", "layers.*.mlps.*.up_proj": "colwise", "layers.*.mlps.*.down_proj": "rowwise", "layers.*.mlp.experts.*.gate_proj": "colwise", "layers.*.mlp.experts.*.up_proj": "colwise", "layers.*.mlp.experts.*.down_proj": "rowwise", } base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } def __init__( self, vocab_size=131072, hidden_size=3072, num_hidden_layers=28, num_layers=14, num_attention_heads=32, num_key_value_heads=None, hidden_act="silu", max_position_embeddings=983040, initializer_range=0.02, rms_norm_eps=1e-5, use_cache=True, pad_token_id=None, bos_token_id=1, eos_token_id=2, tie_word_embeddings=False, rope_theta=1000000.0, rope_scaling=None, attention_bias=False, attention_dropout=0.0, ffn_hidden_size=6144, q_lora_rank=1536, kv_lora_rank=512, qk_nope_head_dim=128, qk_rope_head_dim=64, head_dim=64, v_head_dim=128, qk_head_dim=None, moe_topk=12, n_routed_experts=256, zero_expert_num=128, expert_ffn_hidden_size=1024, routed_scaling_factor=6.0, # --- N-gram embedding params: real config.json keys are oe_*, stored # under the emb_*/ngram_* attribute names NgramEmbedding expects --- oe_neighbor_num=None, oe_split_num=None, oe_vocab_size_ratio=None, # --- LSA / indexer params (new vs. Lite) --- attention_method="LSA", index_n_heads=16, index_head_dim=128, index_topk=2048, index_k_norm_type="rms", index_local_tokens=1024, index_init_tokens=16, indexer_rope_interleave=True, cli_factor=2, dsa_mtp_cli=True, # --- multi-layer MTP (new vs. Lite: 3 layers, not 1) --- mtp_num_layers=3, mtp_replicate_modules=True, mtp_disable_over_tokenizer=False, **kwargs, ): # N-gram embedding parameters, stored under the SAME attribute names # Lite's NgramEmbedding reads, even though the checkpoint's keys are # renamed (oe_* -> emb_*/ngram_*). Do this before super().__init__() # so the attributes exist immediately, matching Lite's own pattern. self.emb_neighbor_num = oe_neighbor_num self.emb_split_num = oe_split_num self.ngram_vocab_size_ratio = oe_vocab_size_ratio # LSA / indexer parameters self.attention_method = attention_method self.index_n_heads = index_n_heads self.index_head_dim = index_head_dim self.index_topk = index_topk self.index_k_norm_type = index_k_norm_type self.index_local_tokens = index_local_tokens self.index_init_tokens = index_init_tokens self.indexer_rope_interleave = indexer_rope_interleave self.cli_factor = cli_factor self.dsa_mtp_cli = dsa_mtp_cli # Multi-layer MTP self.mtp_num_layers = mtp_num_layers self.mtp_replicate_modules = mtp_replicate_modules self.mtp_disable_over_tokenizer = mtp_disable_over_tokenizer # zero_expert_type, moe_switch_token_num, moe_impl, use_mla, etc. are # left to flow through **kwargs to LongcatFlashConfig, same as Lite's # own config class does for zero_expert_type -- no need to redeclare # fields the parent already handles. super().__init__( vocab_size=vocab_size, hidden_size=hidden_size, num_hidden_layers=num_hidden_layers, num_layers=num_layers, num_attention_heads=num_attention_heads, num_key_value_heads=num_key_value_heads, hidden_act=hidden_act, max_position_embeddings=max_position_embeddings, initializer_range=initializer_range, rms_norm_eps=rms_norm_eps, use_cache=use_cache, pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, rope_theta=rope_theta, rope_scaling=rope_scaling, attention_bias=attention_bias, attention_dropout=attention_dropout, ffn_hidden_size=ffn_hidden_size, q_lora_rank=q_lora_rank, kv_lora_rank=kv_lora_rank, qk_nope_head_dim=qk_nope_head_dim, qk_rope_head_dim=qk_rope_head_dim, head_dim=head_dim, v_head_dim=v_head_dim, qk_head_dim=qk_head_dim, moe_topk=moe_topk, n_routed_experts=n_routed_experts, zero_expert_num=zero_expert_num, expert_ffn_hidden_size=expert_ffn_hidden_size, routed_scaling_factor=routed_scaling_factor, **kwargs, ) __all__ = ["LongcatFlashSparseConfig"]