Kimi-K3.1-Pro-SuperExpert-12GB / modeling_kimi_k31_superexpert.py
resuurcct1's picture shashanth-27xA9's picture
Duplicate from shashanth-27xA9/Kimi-K3.1-Pro-SuperExpert-12GB
776beef
Raw History Blame Contribute Delete
9.56 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
from typing import Optional, Tuple, Union, List
from transformers.modeling_utils import PreTrainedModel
from transformers.generation import GenerationMixin
from transformers.modeling_outputs import CausalLMOutputWithPast
try:
from .configuration_kimi_k31_superexpert import KimiK31SuperExpertConfig
except (ImportError, ValueError):
from configuration_kimi_k31_superexpert import KimiK31SuperExpertConfig
class DequantizedPackedLinear(nn.Module):
def __init__(self, in_features: int, out_features: int, bias: bool = False):
super().__init__()
self.in_features = in_features
self.out_features = out_features
self.register_buffer("weight", torch.zeros((out_features, in_features // 2), dtype=torch.uint8))
self.register_buffer("scale", torch.tensor(1.0, dtype=torch.float16))
if bias:
self.bias = nn.Parameter(torch.zeros(out_features))
else:
self.register_parameter("bias", None)
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
scale_key = prefix + "weight.scale"
if scale_key in state_dict:
self.scale.copy_(state_dict[scale_key])
state_dict.pop(scale_key)
elif prefix + "scale" in state_dict:
self.scale.copy_(state_dict[prefix + "scale"])
state_dict.pop(prefix + "scale")
super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)
def forward(self, x: torch.Tensor) -> torch.Tensor:
w = self.weight
if w.numel() == self.out_features * self.in_features:
weight_f = (w.float() - 8.0) * (self.scale.float() / 7.0)
weight_f = weight_f.reshape(self.out_features, self.in_features)
else:
low = (w & 0x0F).to(torch.float32)
high = (w >> 4).to(torch.float32)
unpacked = torch.stack([low, high], dim=-1).reshape(self.out_features, self.in_features)
weight_f = (unpacked - 8.0) * (self.scale.float() / 7.0)
return F.linear(x, weight_f.to(x.dtype), self.bias)
class DequantizedPackedEmbedding(nn.Module):
def __init__(self, num_embeddings: int, embedding_dim: int):
super().__init__()
self.num_embeddings = num_embeddings
self.embedding_dim = embedding_dim
self.register_buffer("weight", torch.zeros((num_embeddings, embedding_dim // 2), dtype=torch.uint8))
self.register_buffer("scale", torch.tensor(1.0, dtype=torch.float16))
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
scale_key = prefix + "weight.scale"
if scale_key in state_dict:
self.scale.copy_(state_dict[scale_key])
state_dict.pop(scale_key)
elif prefix + "scale" in state_dict:
self.scale.copy_(state_dict[prefix + "scale"])
state_dict.pop(prefix + "scale")
super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
packed = self.weight[input_ids]
if packed.shape[-1] == self.embedding_dim:
return ((packed.float() - 8.0) * (self.scale.float() / 7.0)).to(self.scale.dtype)
low = (packed & 0x0F).to(torch.float32)
high = (packed >> 4).to(torch.float32)
unpacked = torch.stack([low, high], dim=-1).reshape(*input_ids.shape, self.embedding_dim)
return ((unpacked - 8.0) * (self.scale.float() / 7.0)).to(self.scale.dtype)
class KimiRMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
class KimiSuperExpertMLP(nn.Module):
def __init__(self, config: KimiK31SuperExpertConfig):
super().__init__()
self.gate_proj = DequantizedPackedLinear(config.hidden_size, config.intermediate_size, bias=False)
self.up_proj = DequantizedPackedLinear(config.hidden_size, config.intermediate_size, bias=False)
self.down_proj = DequantizedPackedLinear(config.intermediate_size, config.hidden_size, bias=False)
self.act_fn = F.silu
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
class KimiSuperExpertAttention(nn.Module):
def __init__(self, config: KimiK31SuperExpertConfig):
super().__init__()
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.head_dim = self.hidden_size // self.num_heads
self.q_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False)
self.k_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False)
self.v_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False)
self.o_proj = DequantizedPackedLinear(config.hidden_size, config.hidden_size, bias=False)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
b_sz, seq_len, _ = hidden_states.shape
q = self.q_proj(hidden_states).view(b_sz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
k = self.k_proj(hidden_states).view(b_sz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
v = self.v_proj(hidden_states).view(b_sz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
attn_out = F.scaled_dot_product_attention(q, k, v, is_causal=True)
attn_out = attn_out.transpose(1, 2).contiguous().view(b_sz, seq_len, self.hidden_size)
return self.o_proj(attn_out)
class KimiSuperExpertDecoderLayer(nn.Module):
def __init__(self, config: KimiK31SuperExpertConfig):
super().__init__()
self.input_layernorm = KimiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.self_attn = KimiSuperExpertAttention(config)
self.post_attention_layernorm = KimiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.mlp = KimiSuperExpertMLP(config)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
residual = hidden_states
hidden_states = residual + self.self_attn(self.input_layernorm(hidden_states))
residual = hidden_states
hidden_states = residual + self.mlp(self.post_attention_layernorm(hidden_states))
return hidden_states
class KimiSuperExpertModel(nn.Module):
def __init__(self, config: KimiK31SuperExpertConfig):
super().__init__()
self.config = config
self.embed_tokens = DequantizedPackedEmbedding(config.vocab_size, config.hidden_size)
self.layers = nn.ModuleList([KimiSuperExpertDecoderLayer(config) for _ in range(config.num_hidden_layers)])
self.norm = KimiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
hidden_states = self.embed_tokens(input_ids)
for layer in self.layers:
hidden_states = layer(hidden_states)
return self.norm(hidden_states)
class KimiK31SuperExpertForCausalLM(PreTrainedModel, GenerationMixin):
config_class = KimiK31SuperExpertConfig
base_model_prefix = "model"
_supports_flash_attn_2 = True
def __init__(self, config: KimiK31SuperExpertConfig):
super().__init__(config)
self.model = KimiSuperExpertModel(config)
self.lm_head = DequantizedPackedLinear(config.hidden_size, config.vocab_size, bias=False)
self.post_init()
def get_input_embeddings(self):
return self.model.embed_tokens
def set_input_embeddings(self, value):
self.model.embed_tokens = value
def get_output_embeddings(self):
return self.lm_head
def set_output_embeddings(self, new_embeddings):
self.lm_head = new_embeddings
def forward(
self,
input_ids: torch.LongTensor = None,
attention_mask: Optional[torch.Tensor] = None,
labels: Optional[torch.LongTensor] = None,
return_dict: Optional[bool] = None,
**kwargs
) -> Union[Tuple, CausalLMOutputWithPast]:
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
hidden_states = self.model(input_ids)
logits = self.lm_head(hidden_states)
loss = None
if labels is not None:
shift_logits = logits[..., :-1, :].contiguous()
shift_labels = labels[..., 1:].contiguous()
loss_fct = nn.CrossEntropyLoss()
loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1))
if not return_dict:
output = (logits,)
return ((loss,) + output) if loss is not None else output
return CausalLMOutputWithPast(loss=loss, logits=logits)
def prepare_inputs_for_generation(self, input_ids, **kwargs):
return {"input_ids": input_ids}