{ "architectures": [ "MMGPTStepRoboticsForCausalLM" ], "auto_map": { "AutoConfig": "configuration_step_robotics.StepRoboticsConfig" }, "model_type": "step3p5v", "im_end_token": "", "im_patch_token": "", "im_start_token": "", "image_token_len": 169, "patch_token_len": 81, "image_token_id": 128001, "understand_projector_stride": 2, "use_im_start_end": "true", "vision_select_layer": -1, "projector_bias": false, "vision_config": { "model_type": "perception_encoder", "image_size": 728, "patch_size": 14, "width": 1536, "layers": 47, "heads": 16, "pool_type": "none", "output_dim": null, "use_cls_token": false, "ls_init_value": 0.1, "use_ln_post": false, "hidden_act": "quick_gelu" }, "text_config": { "architectures": [ "Step4ForCausalLM" ], "model_type": "step4", "att_impl_type": "GQA", "torch_dtype": "bfloat16", "norm_dtype": "float32", "bos_token_id": 0, "eos_token_id": [ 1, 2 ], "hidden_size": 4096, "intermediate_size": 13824, "num_hidden_layers": 92, "vocab_size": 128896, "rms_norm_eps": 1e-05, "fp32_residual_connection": true, "num_attention_heads": 64, "num_attention_groups": 4, "num_sliding_attention_heads": 64, "head_dim": 192, "head_dim_full_attention": 192, "head_dim_sliding_attention": 192, "num_attention_groups_full_attention": 4, "num_attention_groups_sliding_attention": 4, "rope_theta": [ 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0, 10000000.0, 10000.0, 10000.0, 10000.0 ], "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention" ], "partial_rotary_factors": [ 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0, 0.3333333333333333, 1.0, 1.0, 1.0 ], "sliding_window": 512, "attention_other_setting": { "attention_type": "sliding_attention", "num_attention_heads": 64, "num_attention_groups": 4, "head_dim": 192, "true_head_dim": 192 }, "use_head_wise_attn_gate": true, "num_nextn_predict_layers": 3, "use_moe": true, "moe_intermediate_size": 1536, "share_expert_dim": 1536, "moe_num_experts": 352, "moe_top_k": 8, "num_experts_per_tok": 8, "moe_layer_list": [ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90 ], "moe_layers_enum": "3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90", "use_moe_router_bias": true, "need_fp32_gate": true, "swiglu_limits": [ 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0 ], "swiglu_limits_shared": [ 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0 ], "max_seq_len": 1048576, "max_position_embeddings": 1048576, "norm_expert_weight": true, "moe_router_scaling_factor": 3.0, "rope_scaling": { "rope_type": "llama3", "factor": 1.0, "original_max_position_embeddings": 1048576, "low_freq_factor": 1.0, "high_freq_factor": 32.0 }, "yarn_only_types": [ "full_attention" ], "sparse_config": { "enabled": true, "proxy_dim": 256, "sparse_indexer_rope_dim": 32, "sparse_indexer_use_rope": true, "sparse_indexer_num_heads": 16, "sparse_indexer_num_k_heads": 1, "sparse_indexer_q_norm_type": "rmsnorm", "sparse_indexer_k_norm_type": "layernorm", "sparse_indexer_csa_z_norm_type": "none", "sparse_indexer_softmax_variant": "ssmax", "sparse_indexer_ssmax_s_granularity": "q_head", "num_provider_groups": 4, "topk": 512, "region_block_size": 8, "compression_method": "csa_block_compress", "attention_impl": "sparse_gqa", "apply_to_layer_types": [ "full_attention" ] } } }