| |
| vocab_size = 248320 |
| d_model = 5120 |
| n_layers = 64 |
| n_heads = 24 |
| ffn_mult = 3.4000000953674316 |
| max_seq_len = 32768 |
| dropout = 0.0 |
| n_kv_heads = 4 |
| attn_pattern = "global" |
| rope_theta_global = 10000000.0 |
| rope_dim_global = 64 |
| compute_dtype = "f32" |
| layer_pattern = "mohawk_teacher_attention" |
|
|
| [blocks] |
| norm_eps = 0.0000009999999974752427 |
| attn_head_dim = 256 |
| attn_qk_norm = true |
| attn_output_gate = true |
| ffn_activation = "swiglu" |
|
|
| [embedding] |
| weight_tying = false |
|
|
| [mohawk] |
| attention_every = 4 |
| conv_kernel = 4 |
|
|
| [mohawk.gated_delta] |
| heads = 48 |
| head_dim = 128 |
| d_state = 128 |
|
|
| [mohawk.attention] |
| heads = 24 |
| head_dim = 256 |
| d_state = 256 |
|
|
