{ "source": { "repository": "ornith-ai/Ornith-1.5-35B-A3B", "revision": "10fbf86fed7ecee4a061f8b499a618f46001cac1", "architecture": "Qwen3_5MoeForConditionalGeneration", "gguf_architecture": "qwen35moe", "layers": 40, "parameter_count": 34660610688, "routed_experts_per_layer": 256, "active_routed_experts_per_token": 8, "embeddings_tied": false, "scope": "Text model; vision and MTP tensors excluded." }, "runtime": { "repository": "https://github.com/ggml-org/llama.cpp", "revision": "58367713a6935c0810103378144008df32e3d5db", "patches": [] }, "bitrate_convention": "Complete GGUF bytes * 8 / 34,660,610,688 stored text-model parameters; includes headers, alignment, quantization metadata and F32 tensors.", "calibration": { "train_tokens": 1000000, "validation_tokens": 32768, "rco_pool_tokens": 32768, "train_mixture_tokens": { "calibration_mixture": 450000, "fineweb": 150000, "open_thoughts": 400000 }, "train_sequences_captured": 1557, "validation_sequences_captured": 50, "train_rows_per_expert_max": 128, "validation_rows_per_expert_max": 64, "exclusion_policy": "13-word benchmark overlap; exact problem substring or 64-word passage overlap with native PPL text" }, "gsq": { "scope": "Routed expert gate, up and down matrices in all 40 layers.", "routed_projection_tensors": 120, "formats": [ "Q2_K", "Q3_K", "Q4_K" ], "steps_per_projection_and_format": 40, "sampled_rows_per_step": 32, "validation_interval_steps": 10, "optimizer": "Lion", "learning_rate_start": 0.0003, "learning_rate_end": 3e-05, "betas": [ 0.9, 0.95 ], "weight_decay": 1, "temperature_start_end": [ 2.0, 0.05 ], "sharpness_start_end": [ 100, 500 ], "seed": 42, "objective": "Independent projection-output MSE, normalized by teacher output energy. Down projections use intermediate activations from the original gate and up weights.", "initialization": "Native imatrix quantization weighted by teacher-input second moments.", "optimized_parameters": "Categorical quantization-code logits; native scales and offsets are fixed.", "minimum_train_rows_per_expert": 32, "minimum_validation_rows_per_expert": 8, "acceptance": "Select hard-code checkpoints by projection validation error, including the initializer. Retain the initializer wherever complete expert-output validation worsens; check all 27 gate/up/down format combinations.", "packing": "Assemble selected native packed tensors without requantization; verify readback." }, "fixed_precision": { "input_embeddings": "Q4_K", "output_projection": "Q4_K", "attention_and_shared_expert_matrices": "Q8_0", "routers_norms_ssm_convolution_and_state": "F32", "f32_tensor_count": 301, "q8_tensor_count": 310 }, "rco": { "allocation_unit": "One format per stacked routed projection tensor, shared across its 256 experts.", "completed_steps_per_budget": 50, "training_sequences": 4, "validation_sequences": 4, "positions_per_sequence": 32, "draws_per_training_sequence": 4, "total_draws_per_step": 16, "generated_tokens": 0, "loss": "Full-vocabulary teacher-to-student KL.", "optimizer": "Adam with manifold gradient projection, retraction and first-moment transport.", "learning_rate": 0.05, "temperature": 1.0, "seed": 42, "vocabulary_policy_selection": "Four RCO steps per feasible Q4_K or Q8_0 vocabulary policy; select lowest best calibration validation KL, breaking ties in favor of Q4_K.", "selected_vocabulary_policy": "Q4_K for both sizes.", "hard_budget": "Integer-byte dynamic programming under the complete-file size ceiling.", "final_selection": "Lowest calibration validation KL among step 0 through step 50.", "held_out_evaluation_used_for_selection": false }, "models": { "3bit": { "filename": "Ornith-1.5-35B-A3B-GSQ-RCO-3bit.gguf", "bytes": 12992461216, "sha256": "9cae3b59a34308664b53e5b80fd74914cd0c1cf504bcd05609498b0f9126a74f", "parameter_count": 34660610688, "complete_file_bits_per_parameter": 2.9987841432922417, "max_bytes": 12997729008, "unused_bytes": 5267792, "tensor_count": 733, "format_counts": { "F32": 301, "Q2_K": 112, "Q3_K": 7, "Q4_K": 3, "Q8_0": 310 }, "format_parameter_counts": { "F32": 22213248, "Q2_K": 30064771072, "Q3_K": 1879048192, "Q4_K": 1285554176, "Q8_0": 1409024000 }, "routed_projection_format_counts": { "Q2_K": 112, "Q3_K": 7, "Q4_K": 1 }, "rco_best_step": 3, "calibration_validation_kl": 0.18772967904806137 }, "3.5bit": { "filename": "Ornith-1.5-35B-A3B-GSQ-RCO-3.5bit.gguf", "bytes": 15163013536, "sha256": "69191d3d2c402f7c13950815bf20d826d7066c8d7f1bef4594cf70c13ea56ccc", "parameter_count": 34660610688, "complete_file_bits_per_parameter": 3.499768350301953, "max_bytes": 15164017176, "unused_bytes": 1003640, "tensor_count": 733, "format_counts": { "F32": 301, "Q2_K": 52, "Q3_K": 52, "Q4_K": 18, "Q8_0": 310 }, "format_parameter_counts": { "F32": 22213248, "Q2_K": 13958643712, "Q3_K": 13958643712, "Q4_K": 5312086016, "Q8_0": 1409024000 }, "routed_projection_format_counts": { "Q2_K": 52, "Q3_K": 52, "Q4_K": 16 }, "rco_best_step": 12, "calibration_validation_kl": 0.09444386046379805 } } }