from transformers.configuration_utils import PretrainedConfig class KimiK31SuperExpertConfig(PretrainedConfig): model_type = "kimi_k31_superexpert" keys_to_ignore_at_loading = ["model.language_model.vision_tower"] def __init__( self, vocab_size=163840, hidden_size=7168, intermediate_size=18432, moe_intermediate_size=3072, num_hidden_layers=36, num_attention_heads=96, num_key_value_heads=96, hidden_act="silu", max_position_embeddings=1048576, initializer_range=0.02, rms_norm_eps=1e-5, use_cache=True, pad_token_id=0, bos_token_id=1, eos_token_id=2, tie_word_embeddings=False, compression_method="AW-SECD", compressed_footprint_gb=9.2, retained_intelligence_ratio=0.948, **kwargs ): self.vocab_size = vocab_size self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.moe_intermediate_size = moe_intermediate_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads self.num_key_value_heads = num_key_value_heads self.hidden_act = hidden_act self.max_position_embeddings = max_position_embeddings self.initializer_range = initializer_range self.rms_norm_eps = rms_norm_eps self.use_cache = use_cache self.compression_method = compression_method self.compressed_footprint_gb = compressed_footprint_gb self.retained_intelligence_ratio = retained_intelligence_ratio super().__init__( pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs ) def get_text_config(self, decoder=False): return self