import torch import torch.nn as nn import torch.nn.functional as F from typing import Optional, Tuple, Union, List from transformers.modeling_utils import PreTrainedModel from transformers.generation import GenerationMixin from transformers.modeling_outputs import CausalLMOutputWithPast try: from .configuration_kimi_k31_superexpert import KimiK31SuperExpertConfig except (ImportError, ValueError): from configuration_kimi_k31_superexpert import KimiK31SuperExpertConfig class DequantizedPackedLinear(nn.Module): def __init__(self, in_features: int, out_features: int, bias: bool = False): super().__init__() self.in_features = in_features self.out_features = out_features self.register_buffer("weight", torch.zeros((out_features, in_features // 2), dtype=torch.uint8)) self.register_buffer("scale", torch.tensor(1.0, dtype=torch.float16)) if bias: self.bias = nn.Parameter(torch.zeros(out_features)) else: self.register_parameter("bias", None) def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): scale_key = prefix + "weight.scale" if scale_key in state_dict: self.scale.copy_(state_dict[scale_key]) state_dict.pop(scale_key) elif prefix + "scale" in state_dict: self.scale.copy_(state_dict[prefix + "scale"]) state_dict.pop(prefix + "scale") super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) def forward(self, x: torch.Tensor) -> torch.Tensor: w = self.weight if w.numel() == self.out_features * self.in_features: weight_f = (w.float() - 8.0) * (self.scale.float() / 7.0) weight_f = weight_f.reshape(self.out_features, self.in_features) else: low = (w & 0x0F).to(torch.float32) high = (w >> 4).to(torch.float32) unpacked = torch.stack([low, high], dim=-1).reshape(self.out_features, self.in_features) weight_f = (unpacked - 8.0) * (self.scale.float() / 7.0) return F.linear(x, weight_f.to(x.dtype), self.bias) class DequantizedPackedEmbedding(nn.Module): def __init__(self, num_embeddings: int, embedding_dim: int): super().__init__() self.num_embeddings = num_embeddings self.embedding_dim = embedding_dim self.register_buffer("weight", torch.zeros((num_embeddings, embedding_dim // 2), dtype=torch.uint8)) self.register_buffer("scale", torch.tensor(1.0, dtype=torch.float16)) def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): scale_key = prefix + "weight.scale" if scale_key in state_dict: self.scale.copy_(state_dict[scale_key]) state_dict.pop(scale_key) elif prefix + "scale" in state_dict: self.scale.copy_(state_dict[prefix + "scale"]) state_dict.pop(prefix + "scale") super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) def forward(self, input_ids: torch.Tensor) -> torch.Tensor: packed = self.weight[input_ids] if packed.shape[-1] == self.embedding_dim: return ((packed.float() - 8.0) * (self.scale.float() / 7.0)).to(self.scale.dtype) low = (packed & 0x0F).to(torch.float32) high = (packed >> 4).to(torch.float32) unpacked = torch.stack([low, high], dim=-1).reshape(*input_ids.shape, self.embedding_dim) return ((unpacked - 8.0) * (self.scale.float() / 7.0)).to(self.scale.dtype) class KimiRMSNorm(nn.Module): def __init__(self, hidden_size: int, eps: float = 1e-5): super().__init__() self.weight = nn.Parameter(torch.ones(hidden_size)) self.variance_epsilon = eps def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: input_dtype = hidden_states.dtype hidden_states = hidden_states.to(torch.float32) variance = hidden_states.pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) return self.weight * hidden_states.to(input_dtype) class KimiSuperExpertMLP(nn.Module): def __init__(self, config: KimiK31SuperExpertConfig): super().__init__() self.gate_proj = DequantizedPackedLinear(config.hidden_size, config.intermediate_size, bias=False) self.up_proj = DequantizedPackedLinear(config.hidden_size, config.intermediate_size, bias=False) self.down_proj = DequantizedPackedLinear(config.intermediate_size, config.hidden_size, bias=False) self.act_fn = F.silu def forward(self, x: torch.Tensor) -> torch.Tensor: return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) class KimiSuperExpertAttention(nn.Module): def __init__(self, config: KimiK31SuperExpertConfig): super().__init__() self.hidden_size = config.hidden_size self.num_heads = config.num_attention_heads self.head_dim = self.hidden_size // self.num_heads self.q_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False) self.k_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False) self.v_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False) self.o_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: b_sz, seq_len, _ = hidden_states.shape q = self.q_proj(hidden_states).view(b_sz, seq_len, self.num_heads, self.head_dim).transpose(1, 2) k = self.k_proj(hidden_states).view(b_sz, seq_len, self.num_heads, self.head_dim).transpose(1, 2) v = self.v_proj(hidden_states).view(b_sz, seq_len, self.num_heads, self.head_dim).transpose(1, 2) attn_out = F.scaled_dot_product_attention(q, k, v, is_causal=True) attn_out = attn_out.transpose(1, 2).contiguous().view(b_sz, seq_len, self.hidden_size) return self.o_proj(attn_out) class KimiSuperExpertDecoderLayer(nn.Module): def __init__(self, config: KimiK31SuperExpertConfig): super().__init__() self.input_layernorm = KimiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.self_attn = KimiSuperExpertAttention(config) self.post_attention_layernorm = KimiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.mlp = KimiSuperExpertMLP(config) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: residual = hidden_states hidden_states = residual + self.self_attn(self.input_layernorm(hidden_states)) residual = hidden_states hidden_states = residual + self.mlp(self.post_attention_layernorm(hidden_states)) return hidden_states class KimiSuperExpertModel(nn.Module): def __init__(self, config: KimiK31SuperExpertConfig): super().__init__() self.config = config self.embed_tokens = DequantizedPackedEmbedding(config.vocab_size, config.hidden_size) self.layers = nn.ModuleList([KimiSuperExpertDecoderLayer(config) for _ in range(config.num_hidden_layers)]) self.norm = KimiRMSNorm(config.hidden_size, eps=config.rms_norm_eps) def forward(self, input_ids: torch.Tensor) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) for layer in self.layers: hidden_states = layer(hidden_states) return self.norm(hidden_states) class KimiK31SuperExpertForCausalLM(PreTrainedModel, GenerationMixin): config_class = KimiK31SuperExpertConfig base_model_prefix = "model" _supports_flash_attn_2 = True def __init__(self, config: KimiK31SuperExpertConfig): super().__init__(config) self.model = KimiSuperExpertModel(config) self.lm_head = DequantizedPackedLinear(config.hidden_size, config.vocab_size, bias=False) self.post_init() def get_input_embeddings(self): return self.model.embed_tokens def set_input_embeddings(self, value): self.model.embed_tokens = value def get_output_embeddings(self): return self.lm_head def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings def forward( self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, labels: Optional[torch.LongTensor] = None, return_dict: Optional[bool] = None, **kwargs ) -> Union[Tuple, CausalLMOutputWithPast]: return_dict = return_dict if return_dict is not None else self.config.use_return_dict hidden_states = self.model(input_ids) logits = self.lm_head(hidden_states) loss = None if labels is not None: shift_logits = logits[..., :-1, :].contiguous() shift_labels = labels[..., 1:].contiguous() loss_fct = nn.CrossEntropyLoss() loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1)) if not return_dict: output = (logits,) return ((loss,) + output) if loss is not None else output return CausalLMOutputWithPast(loss=loss, logits=logits) def prepare_inputs_for_generation(self, input_ids, **kwargs): return {"input_ids": input_ids}