{ "architectures": [ "NemotronH_Omni_Reasoning_V3" ], "auto_map": { "AutoConfig": "configuration_nemotron_h_omni.NemotronH_Omni_Reasoning_V3_Config", "AutoModel": "modeling_nemotron_h_omni.NemotronH_Omni_Reasoning_V3", "AutoModelForCausalLM": "modeling_nemotron_h_omni.NemotronH_Omni_Reasoning_V3", "AutoModelForImageTextToText": "modeling_nemotron_h_omni.NemotronH_Omni_Reasoning_V3" }, "downsample_ratio": 0.5, "dtype": "bfloat16", "eos_token_id": 11, "force_image_size": 512, "image_tag_type": "internvl", "img_context_token": "", "img_context_token_id": 18, "llm_config": { "_megatron_bridge_hf_prefix": "model", "architectures": [ "NemotronHPuzzleForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_nemotron_h_puzzle.NemotronHPuzzleConfig", "AutoModelForCausalLM": "modeling_nemotron_h_puzzle.NemotronHPuzzleForCausalLM" }, "aux_loss_alpha": 0.0, "block_configs": [ { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1536, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 304, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 304, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 304, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 304, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 304, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 496, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 304, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 240, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2048, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 368, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2304, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 496, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 432, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 496, "num_experts_per_tok": 9 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2560, "n_routed_experts": 368, "num_experts_per_tok": 9 } ], "blockwise_members": [ "num_experts_per_tok", "n_routed_experts", "moe_intermediate_size", "block_type" ], "bos_token_id": 1, "chunk_size": 128, "conv_kernel": 4, "dtype": "bfloat16", "eos_token_id": 2, "expand": 2, "head_dim": 128, "hidden_act": "relu2", "hidden_dropout": 0.0, "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 2688, "layer_norm_epsilon": 1e-05, "layers_block_type": [ "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe" ], "mamba_head_dim": 64, "mamba_hidden_act": "silu", "mamba_num_heads": 128, "mamba_proj_bias": false, "mamba_ssm_cache_dtype": "float32", "max_position_embeddings": 262144, "mlp_bias": false, "mlp_hidden_act": "relu2", "model_type": "nemotron_h_puzzle", "moe_latent_size": 1024, "moe_shared_expert_intermediate_size": 5376, "moe_shared_expert_overlap": true, "mtp_block_configs": [], "mtp_layers_block_type": [], "mtp_num_hidden_layers": null, "n_group": 1, "n_groups": 8, "n_shared_experts": 1, "norm_eps": 1e-05, "norm_topk_prob": true, "num_attention_heads": 32, "num_experts": 512, "num_key_value_heads": 2, "num_local_experts": 512, "num_logits_to_keep": 1, "num_nextn_predict_layers": 0, "pad_token_id": 0, "rescale_prenorm_residual": true, "residual_in_fp32": false, "rms_norm_eps": 1e-05, "routed_scaling_factor": 5.0, "router_aux_loss_coef": 0.0, "scoring_func": "sigmoid", "sliding_window": null, "ssm_state_size": 128, "tie_word_embeddings": false, "time_step_floor": 0.0001, "time_step_limit": [ 0.0, { "__float__": "Infinity" } ], "time_step_max": 0.1, "time_step_min": 0.001, "topk_group": 1, "use_bias": false, "use_cache": true, "use_conv_bias": true, "use_mamba_kernels": true, "use_qk_norm": false, "vocab_size": 131072 }, "model_type": "nemotron_h_omni", "norm_mean": [ 0.48145466, 0.4578275, 0.40821073 ], "norm_std": [ 0.26862954, 0.26130258, 0.27577711 ], "patch_size": 16, "processor_class": "NemotronH_Omni_Reasoning_V3Processor", "projector_hidden_size": 20480, "ps_version": "v2", "sound_config": null, "sound_context_token": "