{ "base_model": { "repo_id": "ornith-ai/Ornith-1.5-35B-A3B", "revision": "10fbf86fed7ecee4a061f8b499a618f46001cac1" }, "runtime": { "repo": "https://github.com/ggml-org/llama.cpp", "revision": "58367713a6935c0810103378144008df32e3d5db", "patched": false }, "protocol": { "ppl": { "context_tokens": 1024, "contexts": 8, "scored_next_tokens": 4088, "batch_tokens": 1024, "ubatch_tokens": 512, "flash_attention": false, "cache_type_k": "f32", "cache_type_v": "f32", "threads": 8 }, "mmlu_pro": { "dataset": "TIGER-Lab/MMLU-Pro", "split": "test", "n": 2000, "seed": 20260909, "stratified_categories": 14, "shots": 0, "chat_template": false, "reasoning": false, "generated_tokens": 0, "context_tokens": 2048, "answer_scoring": "Sum of native conditional log probabilities for each complete letter continuation, selected by argmax.", "flash_attention": false, "cache_type_k": "f32", "cache_type_v": "f32", "threads": 8 }, "chat": { "context_tokens": 8192, "temperature": 0, "seed": 20260908, "chat_template": "Pinned source template embedded in the GGUF", "source_template_sha256": "e84f32a23fdda27689f868aa4a1a5621f41133e51a48d7f3efcbea2839574259", "effective_server_template_sha256": "e84f32a23fdda27689f868aa4a1a5621f41133e51a48d7f3efcbea2839574259", "reasoning_format": "none", "flash_attention": true, "cache_type_k": "f32", "cache_type_v": "f32", "context_shift": false, "ifeval": { "n": 16, "max_tokens": 1024, "reasoning": false, "chat_template_kwargs": { "enable_thinking": false, "preserve_thinking": true }, "scoring": "Strict prompt-level instruction following on content after ; unclosed thinking blocks produce an empty answer." }, "gsm8k": { "n": 8, "row_indices": [ 17, 18, 19, 20, 21, 22, 23, 24 ], "max_tokens": 2048, "reasoning": true, "chat_template_kwargs": { "enable_thinking": true, "preserve_thinking": true }, "scoring": "Exact numeric answer after a final #### marker, after removing thinking." } } }, "input_identities": [ { "name": "heldout.txt", "bytes": 65450, "sha256": "6fb5d759981fe48325b0e331feb9c2b9febca598b9d591a47f11cea06e5a1f9b" }, { "name": "subset.json", "bytes": 16726, "sha256": "1eeeed9260b6b73923ded59a95fb3988650b7e008310479f2b0e31c1943571d1" }, { "name": "questions.json", "bytes": 1668898, "sha256": "125e3c9ede66f30c5d155621f89ce15fb9aa156d87071fd6c65b723faf654bb2" }, { "name": "task-index.json", "bytes": 185572, "sha256": "c7bb5ac1088db99757ec941a19102f6dbadaecf4a2ca7740fa9720554a70cb12" }, { "name": "tasks.bin", "bytes": 1708205, "sha256": "72a4a04d667f78f39a388bf56202efd3af1136affda4f6a01e4b8c518812b43c" }, { "name": "build_tasks.py", "bytes": 2046, "sha256": "ca359d580e07d6a0d80f72f08eccb8c46d6b8fd0ef4e0a096e365f295555b2bd" }, { "name": "mc-score.cpp", "bytes": 8930, "sha256": "48b746edd6adb572824513eb8afa58ae6d281858cdd10b807f054143a1f74a35" }, { "name": "analyze_mmlu.py", "bytes": 14919, "sha256": "961f8e4ac1b2b497fcb3b8e9e59d8ac3072d5151e7c594ff8d12e57725ab0caa" }, { "name": "quality.py", "bytes": 7073, "sha256": "7e6f2f8e31cd2f266141bba6630a076651f0b0a433ef10185847d26e0b0c2c0c" } ], "builds": { "reference": { "label": "BF16 reference", "artifact": { "filename": null, "bytes": 69376637344, "sha256": "a3b760ab39e3e668fd7282057066da55ee6c455fd2b3bb599bce6b62aa63fd96" }, "ppl": { "context_tokens": 1024, "contexts": 8, "ppl": 3.4112, "reported_standard_error": 0.11196, "scored_next_tokens": 4088, "native_final_estimate": "Final estimate: PPL = 3.4112 +/- 0.11196", "cumulative_chunk_ppl": [ 2.1898, 1.5741, 1.6884, 2.3801, 2.3849, 3.0242, 3.368, 3.4112 ], "original_log_sha256": "e458de61ea95e87f833f868da7e0b79c220aa3e24709baa86ebb766329cda8a9" }, "mmlu_pro": { "accuracy_pct": 47.599999999999994, "binomial_se_pp": 1.1167452708652945, "context_tokens": 2048, "correct": 952, "generated_tokens": 0, "gold_counts": { "0": 225, "1": 229, "2": 217, "3": 241, "4": 192, "5": 179, "6": 189, "7": 182, "8": 180, "9": 166 }, "n": 2000, "per_category": { "biology": { "accuracy_pct": 69.74789915966386, "binomial_se_pp": 4.210850808038787, "correct": 83, "n": 119 }, "business": { "accuracy_pct": 31.297709923664126, "binomial_se_pp": 4.051410409300031, "correct": 41, "n": 131 }, "chemistry": { "accuracy_pct": 36.702127659574465, "binomial_se_pp": 3.515290756427134, "correct": 69, "n": 188 }, "computer science": { "accuracy_pct": 50.0, "binomial_se_pp": 6.063390625908324, "correct": 34, "n": 68 }, "economics": { "accuracy_pct": 69.28571428571428, "binomial_se_pp": 3.8987738223426422, "correct": 97, "n": 140 }, "engineering": { "accuracy_pct": 46.58385093167702, "binomial_se_pp": 3.9313439656233915, "correct": 75, "n": 161 }, "health": { "accuracy_pct": 61.029411764705884, "binomial_se_pp": 4.181851612943798, "correct": 83, "n": 136 }, "history": { "accuracy_pct": 42.857142857142854, "binomial_se_pp": 6.234796863885496, "correct": 27, "n": 63 }, "law": { "accuracy_pct": 37.704918032786885, "binomial_se_pp": 3.582616621235298, "correct": 69, "n": 183 }, "math": { "accuracy_pct": 36.0, "binomial_se_pp": 3.2, "correct": 81, "n": 225 }, "other": { "accuracy_pct": 50.649350649350644, "binomial_se_pp": 4.028775025496727, "correct": 78, "n": 154 }, "philosophy": { "accuracy_pct": 55.42168674698795, "binomial_se_pp": 5.455852735374259, "correct": 46, "n": 83 }, "physics": { "accuracy_pct": 39.351851851851855, "binomial_se_pp": 3.3240265499390174, "correct": 85, "n": 216 }, "psychology": { "accuracy_pct": 63.1578947368421, "binomial_se_pp": 4.18273375915277, "correct": 84, "n": 133 } }, "prediction_counts": { "0": 238, "1": 189, "2": 267, "3": 161, "4": 290, "5": 279, "6": 143, "7": 73, "8": 160, "9": 200 }, "reasoning": false, "scores": "eval/mmlu-pro/reference.tsv", "scores_sha256": "2f5fdbbef022460477683c6a06c258bd0d17a8a9711e161152539c7c7dc0f697" }, "chat": { "gsm8k": { "completed_correct": 4, "errors": 0, "passed": 4, "requested": 8, "truncated": 0 }, "ifeval": { "completed_correct": 12, "errors": 0, "passed": 14, "requested": 16, "truncated": 2 } } }, "gsq-rco-3": { "label": "GSQ-RCO 3-bit", "artifact": { "filename": "Ornith-1.5-35B-A3B-GSQ-RCO-3bit.gguf", "bytes": 12992461216, "sha256": "9cae3b59a34308664b53e5b80fd74914cd0c1cf504bcd05609498b0f9126a74f" }, "ppl": { "context_tokens": 1024, "contexts": 8, "ppl": 3.5369, "reported_standard_error": 0.11776, "scored_next_tokens": 4088, "native_final_estimate": "Final estimate: PPL = 3.5369 +/- 0.11776", "cumulative_chunk_ppl": [ 2.1343, 1.5799, 1.7564, 2.4971, 2.4657, 3.1573, 3.4842, 3.5369 ], "original_log_sha256": "832358c4c04b0eddb00bce46a35bbe3ee59046a452ace926791de7a4fafc0605" }, "mmlu_pro": { "accuracy_pct": 43.7, "binomial_se_pp": 1.109123527836282, "context_tokens": 2048, "correct": 874, "generated_tokens": 0, "gold_counts": { "0": 225, "1": 229, "2": 217, "3": 241, "4": 192, "5": 179, "6": 189, "7": 182, "8": 180, "9": 166 }, "n": 2000, "per_category": { "biology": { "accuracy_pct": 71.42857142857143, "binomial_se_pp": 4.141221683231774, "correct": 85, "n": 119 }, "business": { "accuracy_pct": 32.06106870229007, "binomial_se_pp": 4.077675905843862, "correct": 42, "n": 131 }, "chemistry": { "accuracy_pct": 35.1063829787234, "binomial_se_pp": 3.481088858162498, "correct": 66, "n": 188 }, "computer science": { "accuracy_pct": 42.64705882352941, "binomial_se_pp": 5.997467916761112, "correct": 29, "n": 68 }, "economics": { "accuracy_pct": 55.00000000000001, "binomial_se_pp": 4.20458932936041, "correct": 77, "n": 140 }, "engineering": { "accuracy_pct": 39.75155279503105, "binomial_se_pp": 3.8568881108892743, "correct": 64, "n": 161 }, "health": { "accuracy_pct": 57.35294117647059, "binomial_se_pp": 4.240850233890538, "correct": 78, "n": 136 }, "history": { "accuracy_pct": 42.857142857142854, "binomial_se_pp": 6.234796863885496, "correct": 27, "n": 63 }, "law": { "accuracy_pct": 40.98360655737705, "binomial_se_pp": 3.635514447317571, "correct": 75, "n": 183 }, "math": { "accuracy_pct": 26.222222222222225, "binomial_se_pp": 2.932285634071231, "correct": 59, "n": 225 }, "other": { "accuracy_pct": 50.0, "binomial_se_pp": 4.029114820126901, "correct": 77, "n": 154 }, "philosophy": { "accuracy_pct": 48.19277108433735, "binomial_se_pp": 5.484626843295494, "correct": 40, "n": 83 }, "physics": { "accuracy_pct": 32.870370370370374, "binomial_se_pp": 3.196189708329014, "correct": 71, "n": 216 }, "psychology": { "accuracy_pct": 63.1578947368421, "binomial_se_pp": 4.18273375915277, "correct": 84, "n": 133 } }, "prediction_counts": { "0": 294, "1": 168, "2": 292, "3": 124, "4": 183, "5": 258, "6": 225, "7": 50, "8": 250, "9": 156 }, "reasoning": false, "scores": "eval/mmlu-pro/gsq-rco-3.tsv", "scores_sha256": "8ce2ca965b3434e362f70e772d15657ebb49c44c4b2770c9480ad98f6962f027" }, "chat": { "gsm8k": { "completed_correct": 8, "errors": 0, "passed": 8, "requested": 8, "truncated": 0 }, "ifeval": { "completed_correct": 10, "errors": 0, "passed": 11, "requested": 16, "truncated": 3 } } }, "gsq-rco-3.5": { "label": "GSQ-RCO 3.5-bit", "artifact": { "filename": "Ornith-1.5-35B-A3B-GSQ-RCO-3.5bit.gguf", "bytes": 15163013536, "sha256": "69191d3d2c402f7c13950815bf20d826d7066c8d7f1bef4594cf70c13ea56ccc" }, "ppl": { "context_tokens": 1024, "contexts": 8, "ppl": 3.4575, "reported_standard_error": 0.11435, "scored_next_tokens": 4088, "native_final_estimate": "Final estimate: PPL = 3.4575 +/- 0.11435", "cumulative_chunk_ppl": [ 2.1137, 1.5489, 1.7094, 2.4084, 2.3917, 3.0583, 3.406, 3.4575 ], "original_log_sha256": "f39005ad94e69fec26baa9c7df21c1e3a81dca0bb09dd90c9b7f154291b4f74b" }, "mmlu_pro": { "accuracy_pct": 43.2, "binomial_se_pp": 1.1076461528845754, "context_tokens": 2048, "correct": 864, "generated_tokens": 0, "gold_counts": { "0": 225, "1": 229, "2": 217, "3": 241, "4": 192, "5": 179, "6": 189, "7": 182, "8": 180, "9": 166 }, "n": 2000, "per_category": { "biology": { "accuracy_pct": 60.50420168067227, "binomial_se_pp": 4.481204207765529, "correct": 72, "n": 119 }, "business": { "accuracy_pct": 25.954198473282442, "binomial_se_pp": 3.8301729266939692, "correct": 34, "n": 131 }, "chemistry": { "accuracy_pct": 35.638297872340424, "binomial_se_pp": 3.4929575896917204, "correct": 67, "n": 188 }, "computer science": { "accuracy_pct": 33.82352941176471, "binomial_se_pp": 5.737290043970798, "correct": 23, "n": 68 }, "economics": { "accuracy_pct": 61.42857142857143, "binomial_se_pp": 4.113903043428328, "correct": 86, "n": 140 }, "engineering": { "accuracy_pct": 44.72049689440994, "binomial_se_pp": 3.918523336205298, "correct": 72, "n": 161 }, "health": { "accuracy_pct": 58.82352941176471, "binomial_se_pp": 4.220176788114348, "correct": 80, "n": 136 }, "history": { "accuracy_pct": 31.746031746031743, "binomial_se_pp": 5.864600072946583, "correct": 20, "n": 63 }, "law": { "accuracy_pct": 42.62295081967213, "binomial_se_pp": 3.655655955723626, "correct": 78, "n": 183 }, "math": { "accuracy_pct": 27.111111111111114, "binomial_se_pp": 2.963555496308145, "correct": 61, "n": 225 }, "other": { "accuracy_pct": 50.0, "binomial_se_pp": 4.029114820126901, "correct": 77, "n": 154 }, "philosophy": { "accuracy_pct": 40.963855421686745, "binomial_se_pp": 5.397844383629204, "correct": 34, "n": 83 }, "physics": { "accuracy_pct": 35.18518518518518, "binomial_se_pp": 3.2493028183855466, "correct": 76, "n": 216 }, "psychology": { "accuracy_pct": 63.1578947368421, "binomial_se_pp": 4.18273375915277, "correct": 84, "n": 133 } }, "prediction_counts": { "0": 258, "1": 216, "2": 345, "3": 112, "4": 191, "5": 245, "6": 104, "7": 52, "8": 300, "9": 177 }, "reasoning": false, "scores": "eval/mmlu-pro/gsq-rco-3.5.tsv", "scores_sha256": "d4e8b624b61d36cabecbec94d901a97a34c7eed218277daf85acaf6d2195dbfb" }, "chat": { "gsm8k": { "completed_correct": 7, "errors": 0, "passed": 7, "requested": 8, "truncated": 0 }, "ifeval": { "completed_correct": 14, "errors": 0, "passed": 14, "requested": 16, "truncated": 1 } } }, "imatrix-3": { "label": "Native imatrix 3-bit", "artifact": { "filename": null, "bytes": 12984072544, "sha256": "61c8c3533a651e483c02be98e7610a132d3543b6f542f340f5f1b593e5c40330" }, "ppl": { "context_tokens": 1024, "contexts": 8, "ppl": 3.532, "reported_standard_error": 0.11687, "scored_next_tokens": 4088, "native_final_estimate": "Final estimate: PPL = 3.5320 +/- 0.11687", "cumulative_chunk_ppl": [ 2.0926, 1.5717, 1.7462, 2.4739, 2.4745, 3.1719, 3.4854, 3.532 ], "original_log_sha256": "1b912ec0ed6f32caee92afdb078378123aaf4624de25860e675e237638a85bd6" }, "mmlu_pro": { "accuracy_pct": 41.9, "binomial_se_pp": 1.1032656071862297, "context_tokens": 2048, "correct": 838, "generated_tokens": 0, "gold_counts": { "0": 225, "1": 229, "2": 217, "3": 241, "4": 192, "5": 179, "6": 189, "7": 182, "8": 180, "9": 166 }, "n": 2000, "per_category": { "biology": { "accuracy_pct": 68.0672268907563, "binomial_se_pp": 4.273796761700158, "correct": 81, "n": 119 }, "business": { "accuracy_pct": 24.427480916030532, "binomial_se_pp": 3.753925444382835, "correct": 32, "n": 131 }, "chemistry": { "accuracy_pct": 34.57446808510639, "binomial_se_pp": 3.4687456844079723, "correct": 65, "n": 188 }, "computer science": { "accuracy_pct": 41.17647058823529, "binomial_se_pp": 5.9682312493634395, "correct": 28, "n": 68 }, "economics": { "accuracy_pct": 57.14285714285714, "binomial_se_pp": 4.182428884065143, "correct": 80, "n": 140 }, "engineering": { "accuracy_pct": 39.75155279503105, "binomial_se_pp": 3.8568881108892743, "correct": 64, "n": 161 }, "health": { "accuracy_pct": 52.20588235294118, "binomial_se_pp": 4.283290106919754, "correct": 71, "n": 136 }, "history": { "accuracy_pct": 47.61904761904761, "binomial_se_pp": 6.29226164422295, "correct": 30, "n": 63 }, "law": { "accuracy_pct": 40.43715846994536, "binomial_se_pp": 3.627876331532134, "correct": 74, "n": 183 }, "math": { "accuracy_pct": 27.111111111111114, "binomial_se_pp": 2.963555496308145, "correct": 61, "n": 225 }, "other": { "accuracy_pct": 46.75324675324675, "binomial_se_pp": 4.020611339244851, "correct": 72, "n": 154 }, "philosophy": { "accuracy_pct": 44.57831325301205, "binomial_se_pp": 5.455852735374259, "correct": 37, "n": 83 }, "physics": { "accuracy_pct": 34.72222222222222, "binomial_se_pp": 3.239362581741255, "correct": 75, "n": 216 }, "psychology": { "accuracy_pct": 51.127819548872175, "binomial_se_pp": 4.334446763411796, "correct": 68, "n": 133 } }, "prediction_counts": { "0": 242, "1": 132, "2": 193, "3": 299, "4": 302, "5": 389, "6": 138, "7": 42, "8": 153, "9": 110 }, "reasoning": false, "scores": "eval/mmlu-pro/imatrix-3.tsv", "scores_sha256": "8249688afb3aed37f5c2923d3f0322879e4ab0f988f9b3b0883b41effe3b69bb" } }, "imatrix-3.5": { "label": "Native imatrix 3.5-bit", "artifact": { "filename": null, "bytes": 15155673440, "sha256": "71c9c77288528c9af31701b79c91995597923cc6918c2f26471c98c46807509c" }, "ppl": { "context_tokens": 1024, "contexts": 8, "ppl": 3.4795, "reported_standard_error": 0.11424, "scored_next_tokens": 4088, "native_final_estimate": "Final estimate: PPL = 3.4795 +/- 0.11424", "cumulative_chunk_ppl": [ 2.1131, 1.5684, 1.7242, 2.4378, 2.4422, 3.1147, 3.433, 3.4795 ], "original_log_sha256": "4d4910cb94a538d1611e5be7c5a3940362110774b0853f03afd81fd14eb7f334" }, "mmlu_pro": { "accuracy_pct": 43.95, "binomial_se_pp": 1.1098192420389907, "context_tokens": 2048, "correct": 879, "generated_tokens": 0, "gold_counts": { "0": 225, "1": 229, "2": 217, "3": 241, "4": 192, "5": 179, "6": 189, "7": 182, "8": 180, "9": 166 }, "n": 2000, "per_category": { "biology": { "accuracy_pct": 63.86554621848739, "binomial_se_pp": 4.403728966143581, "correct": 76, "n": 119 }, "business": { "accuracy_pct": 26.717557251908396, "binomial_se_pp": 3.8660075266409604, "correct": 35, "n": 131 }, "chemistry": { "accuracy_pct": 30.851063829787233, "binomial_se_pp": 3.3685957705974148, "correct": 58, "n": 188 }, "computer science": { "accuracy_pct": 42.64705882352941, "binomial_se_pp": 5.997467916761112, "correct": 29, "n": 68 }, "economics": { "accuracy_pct": 63.57142857142857, "binomial_se_pp": 4.067129846649154, "correct": 89, "n": 140 }, "engineering": { "accuracy_pct": 42.857142857142854, "binomial_se_pp": 3.9001350445139145, "correct": 69, "n": 161 }, "health": { "accuracy_pct": 56.61764705882353, "binomial_se_pp": 4.249746313713087, "correct": 77, "n": 136 }, "history": { "accuracy_pct": 36.507936507936506, "binomial_se_pp": 6.065730203397998, "correct": 23, "n": 63 }, "law": { "accuracy_pct": 40.43715846994536, "binomial_se_pp": 3.627876331532134, "correct": 74, "n": 183 }, "math": { "accuracy_pct": 28.444444444444443, "binomial_se_pp": 3.007662824402849, "correct": 64, "n": 225 }, "other": { "accuracy_pct": 50.0, "binomial_se_pp": 4.029114820126901, "correct": 77, "n": 154 }, "philosophy": { "accuracy_pct": 59.036144578313255, "binomial_se_pp": 5.397844383629204, "correct": 49, "n": 83 }, "physics": { "accuracy_pct": 32.870370370370374, "binomial_se_pp": 3.196189708329014, "correct": 71, "n": 216 }, "psychology": { "accuracy_pct": 66.16541353383458, "binomial_se_pp": 4.102703496421568, "correct": 88, "n": 133 } }, "prediction_counts": { "0": 330, "1": 132, "2": 226, "3": 199, "4": 412, "5": 288, "6": 128, "7": 59, "8": 107, "9": 119 }, "reasoning": false, "scores": "eval/mmlu-pro/imatrix-3.5.tsv", "scores_sha256": "1bd986f71d6215f88983de1b9e20adf4bcd72b209b8362f04c186e6c867278cf" } }, "initializer-3": { "label": "Matched initializer 3-bit", "artifact": { "filename": null, "bytes": 12992461216, "sha256": "fce1feb1559cf728ea610b6a67fa155bbd060fa89e90b3b6cb4a050de22010d1" }, "ppl": { "context_tokens": 1024, "contexts": 8, "ppl": 3.5353, "reported_standard_error": 0.1169, "scored_next_tokens": 4088, "native_final_estimate": "Final estimate: PPL = 3.5353 +/- 0.11690", "cumulative_chunk_ppl": [ 2.084, 1.5676, 1.7552, 2.498, 2.4708, 3.1635, 3.4831, 3.5353 ], "original_log_sha256": "45ffd240b2d24eefdbb3acb079ff54b33f49fa719c17312977b1fd374e9d9149" }, "mmlu_pro": { "accuracy_pct": 45.35, "binomial_se_pp": 1.113188550965199, "context_tokens": 2048, "correct": 907, "generated_tokens": 0, "gold_counts": { "0": 225, "1": 229, "2": 217, "3": 241, "4": 192, "5": 179, "6": 189, "7": 182, "8": 180, "9": 166 }, "n": 2000, "per_category": { "biology": { "accuracy_pct": 71.42857142857143, "binomial_se_pp": 4.141221683231774, "correct": 85, "n": 119 }, "business": { "accuracy_pct": 25.190839694656486, "binomial_se_pp": 3.792827265206377, "correct": 33, "n": 131 }, "chemistry": { "accuracy_pct": 37.234042553191486, "binomial_se_pp": 3.5257640868465945, "correct": 70, "n": 188 }, "computer science": { "accuracy_pct": 47.05882352941176, "binomial_se_pp": 6.0528912403238415, "correct": 32, "n": 68 }, "economics": { "accuracy_pct": 61.42857142857143, "binomial_se_pp": 4.113903043428328, "correct": 86, "n": 140 }, "engineering": { "accuracy_pct": 44.72049689440994, "binomial_se_pp": 3.918523336205298, "correct": 72, "n": 161 }, "health": { "accuracy_pct": 58.82352941176471, "binomial_se_pp": 4.220176788114348, "correct": 80, "n": 136 }, "history": { "accuracy_pct": 49.2063492063492, "binomial_se_pp": 6.298614257293926, "correct": 31, "n": 63 }, "law": { "accuracy_pct": 39.34426229508197, "binomial_se_pp": 3.6111963520001775, "correct": 72, "n": 183 }, "math": { "accuracy_pct": 32.88888888888889, "binomial_se_pp": 3.132063484913002, "correct": 74, "n": 225 }, "other": { "accuracy_pct": 50.649350649350644, "binomial_se_pp": 4.028775025496727, "correct": 78, "n": 154 }, "philosophy": { "accuracy_pct": 43.373493975903614, "binomial_se_pp": 5.439801353912797, "correct": 36, "n": 83 }, "physics": { "accuracy_pct": 33.7962962962963, "binomial_se_pp": 3.21846523284188, "correct": 73, "n": 216 }, "psychology": { "accuracy_pct": 63.90977443609023, "binomial_se_pp": 4.164401747949396, "correct": 85, "n": 133 } }, "prediction_counts": { "0": 247, "1": 152, "2": 282, "3": 125, "4": 170, "5": 362, "6": 171, "7": 63, "8": 257, "9": 171 }, "reasoning": false, "scores": "eval/mmlu-pro/initializer-3.tsv", "scores_sha256": "7aafc12c6492073c494028e4e19725a345868ef17bd04f0498796296dc63062a" } }, "initializer-3.5": { "label": "Matched initializer 3.5-bit", "artifact": { "filename": null, "bytes": 15163013536, "sha256": "2cfb79ba2fd14c0207541bab9066449b34d52a4c3d473cc3f9cfff41e2a085cb" }, "ppl": { "context_tokens": 1024, "contexts": 8, "ppl": 3.4575, "reported_standard_error": 0.114, "scored_next_tokens": 4088, "native_final_estimate": "Final estimate: PPL = 3.4575 +/- 0.11400", "cumulative_chunk_ppl": [ 2.1299, 1.5629, 1.7115, 2.4138, 2.3974, 3.068, 3.405, 3.4575 ], "original_log_sha256": "2d1c32789a301a8e0d866335e1389d964989a9485c4668f292aa006613f3dae6" }, "mmlu_pro": { "accuracy_pct": 44.65, "binomial_se_pp": 1.1116153786269782, "context_tokens": 2048, "correct": 893, "generated_tokens": 0, "gold_counts": { "0": 225, "1": 229, "2": 217, "3": 241, "4": 192, "5": 179, "6": 189, "7": 182, "8": 180, "9": 166 }, "n": 2000, "per_category": { "biology": { "accuracy_pct": 63.02521008403361, "binomial_se_pp": 4.425236927870432, "correct": 75, "n": 119 }, "business": { "accuracy_pct": 25.954198473282442, "binomial_se_pp": 3.8301729266939692, "correct": 34, "n": 131 }, "chemistry": { "accuracy_pct": 36.17021276595745, "binomial_se_pp": 3.5043566995933286, "correct": 68, "n": 188 }, "computer science": { "accuracy_pct": 36.76470588235294, "binomial_se_pp": 5.847104599738657, "correct": 25, "n": 68 }, "economics": { "accuracy_pct": 62.857142857142854, "binomial_se_pp": 4.083672959438616, "correct": 88, "n": 140 }, "engineering": { "accuracy_pct": 47.20496894409938, "binomial_se_pp": 3.9343903387776322, "correct": 76, "n": 161 }, "health": { "accuracy_pct": 59.55882352941176, "binomial_se_pp": 4.208385268035399, "correct": 81, "n": 136 }, "history": { "accuracy_pct": 41.269841269841265, "binomial_se_pp": 6.202641946766029, "correct": 26, "n": 63 }, "law": { "accuracy_pct": 37.15846994535519, "binomial_se_pp": 3.5721257740537795, "correct": 68, "n": 183 }, "math": { "accuracy_pct": 30.22222222222222, "binomial_se_pp": 3.061473505917509, "correct": 68, "n": 225 }, "other": { "accuracy_pct": 51.298701298701296, "binomial_se_pp": 4.027755469609088, "correct": 79, "n": 154 }, "philosophy": { "accuracy_pct": 48.19277108433735, "binomial_se_pp": 5.484626843295494, "correct": 40, "n": 83 }, "physics": { "accuracy_pct": 36.11111111111111, "binomial_se_pp": 3.268181880331384, "correct": 78, "n": 216 }, "psychology": { "accuracy_pct": 65.41353383458647, "binomial_se_pp": 4.124402872916361, "correct": 87, "n": 133 } }, "prediction_counts": { "0": 253, "1": 240, "2": 295, "3": 101, "4": 177, "5": 247, "6": 104, "7": 63, "8": 315, "9": 205 }, "reasoning": false, "scores": "eval/mmlu-pro/initializer-3.5.tsv", "scores_sha256": "86a5086eb6ad28b49f9685905328b448454d3c05322c93e2404932eb4d501cb8" } } } }