{ "architectures": [ "Lfm3ForCausalLM" ], "auto_map": { "AutoConfig": "configuration_lfm3.Lfm3Config", "AutoModel": "modeling_lfm3.Lfm3Model", "AutoModelForCausalLM": "modeling_lfm3.Lfm3ForCausalLM" }, "bos_token_id": 124894, "conv_L_cache": 7, "conv_bias": false, "dtype": "bfloat16", "eos_token_id": [ 124900 ], "generation_config": { "_from_model_config": true, "bos_token_id": 124894, "eos_token_id": [ 124900 ], "output_attentions": false, "output_hidden_states": false, "pad_token_id": 124893, "transformers_version": "5.12.1", "use_cache": true }, "head_dim": 128, "hidden_size": 2048, "intermediate_size": 7680, "layer_types": [ "conv", "conv", "full_attention", "conv", "sliding_attention", "conv", "full_attention", "conv", "sliding_attention", "conv", "full_attention", "conv", "sliding_attention", "conv", "full_attention", "conv", "sliding_attention", "conv", "full_attention", "conv", "sliding_attention", "conv", "full_attention", "conv" ], "max_position_embeddings": 262144, "model_type": "lfm3", "norm_eps": 1e-05, "num_attention_heads": 16, "num_hidden_layers": 24, "num_key_value_heads": 4, "pad_token_id": 124893, "quantization": { "group_size": 64, "bits": 5, "mode": "affine", "language_model.model.embed_tokens": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.feed_forward.w2": { "bits": 5, "group_size": 64 } }, "quantization_config": { "group_size": 64, "bits": 5, "mode": "affine", "language_model.model.embed_tokens": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.0.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.1.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.2.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.3.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.4.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.5.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.6.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.7.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.8.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.9.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.10.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.11.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.12.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.13.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.14.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.15.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.16.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.17.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.18.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.19.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.20.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.21.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.q_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.k_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.v_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.o_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.self_attn.gate_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.22.feed_forward.w2": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.conv.in_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.conv.out_proj": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.feed_forward.w1": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.feed_forward.w3": { "bits": 5, "group_size": 64 }, "language_model.model.layers.23.feed_forward.w2": { "bits": 5, "group_size": 64 } }, "rope_parameters": { "full_attention": { "partial_rotary_factor": 0.5, "rope_theta": 1000000.0, "rope_type": "default" }, "sliding_attention": { "rope_theta": 10000, "rope_type": "default" } }, "sliding_window": 256, "tie_word_embeddings": true, "transformers_version": "5.12.1", "use_cache": true, "use_sandwich_norm": true, "vision_config": {}, "vocab_size": 128000 }