{ "architectures": [ "LongcatFlashSparseForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "attention_method": "LSA", "auto_map": { "AutoConfig": "configuration_longcat_flash_sparse.LongcatFlashSparseConfig", "AutoModel": "modeling_longcat_flash_sparse.LongcatFlashSparseModel", "AutoModelForCausalLM": "modeling_longcat_flash_sparse.LongcatFlashSparseForCausalLM" }, "bos_token_id": 1, "cli_factor": 2, "dsa_mtp_cli": true, "dtype": "bfloat16", "emb_neighbor_num": 4, "emb_split_num": 4, "eos_token_id": 2, "expert_ffn_hidden_size": 1024, "ffn_hidden_size": 6144, "head_dim": 64, "hidden_act": "silu", "hidden_size": 3072, "index_head_dim": 128, "index_init_tokens": 16, "index_k_norm_type": "rms", "index_local_tokens": 1024, "index_n_heads": 16, "index_topk": 2048, "indexer_rope_interleave": true, "initializer_range": 0.02, "kv_lora_rank": 512, "max_position_embeddings": 983040, "mla_scale_kv_lora": true, "mla_scale_q_lora": true, "model_type": "longcat_flash_lite_sparse", "moe_impl": "mix", "moe_switch_token_num": 1024, "moe_topk": 12, "mtp_disable_over_tokenizer": false, "mtp_num_layers": 3, "mtp_replicate_modules": true, "n_routed_experts": 256, "ngram_vocab_size_ratio": 78, "num_attention_heads": 32, "num_hidden_layers": 28, "num_key_value_heads": 32, "num_layers": 14, "pad_token_id": null, "q_lora_rank": 1536, "qk_head_dim": 192, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_parameters": { "beta_fast": 32, "beta_slow": 1, "factor": 120, "mscale": 1, "mscale_all_dim": 1, "original_max_position_embeddings": 8192, "rope_theta": 1000000.0, "rope_type": "yarn" }, "routed_scaling_factor": 6.0, "tie_word_embeddings": false, "transformers_version": "5.15.0", "use_cache": true, "use_mla": 1, "v_head_dim": 128, "vocab_size": 131072, "zero_expert_num": 128, "zero_expert_type": "identity" }