{ "scope": "complete public roster", "weights": { "old_core": 0.3, "v3_core": 0.25, "v4": 0.15, "v5": 0.15, "transfer_v9_test": 0.15 }, "protocol_sha256": "ab97f652ab890eaefdd5373dd2deeb1e26588815f1f339eee4bcaa610846b0af", "statistics_sha256": "8406aea215dc1c4ee2645130c94472b336bb065ba46c6efac4f742589499463b", "source_per_model": { "Jev": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Jev", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "kev-9b": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "kev-9b", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "kev-4b": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "kev-4b", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Qwen3.5-9B": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Qwen3.5-9B", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Decider": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Decider", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Qwen3.5-4B": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Qwen3.5-4B", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Sol": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Sol", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "kev-0.8b": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "kev-0.8b", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Qwen3.5-2B": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Qwen3.5-2B", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Laya-base": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Laya-base", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Laya-multilingual": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Laya-multilingual", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Nox": { "statistics_sha256": "b319a993ac569ea1fa8da027ccb2195868557e4efe232a5d06091db2884086be", "source_model_key": "Nox-null-description-candidate", "manifest_sha256": "cf455486f8eba2b7158bbb4b5bcdcd8a4bd4daee4f05184b1df61b5f70ef2b04", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Lux": { "statistics_sha256": "71251a31ed6435d8c94df6d2a5241e9183613c19a87f8970145afec8ec948efc", "source_model_key": "Lux-reading-source-candidate", "manifest_sha256": "7107a2a8264a0416fa75fe40e57f3d06e4cb0395527eaedfa7b7335d8e37109a", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" } }, "Kai": { "statistics_sha256": "fc72b6c9f3d1594f5e4a6b8d33f9a438cbd7b762e6794e2d0ae6f47296b83d4d", "source_model_key": "Kai-fixed-noul-restore-v2", "manifest_sha256": "610a5a29f7619a5d4057c864c36cfa4958493a156fbe4dfdcb9185f89fb1e07d", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" }, "hub_revision": "db41e715facc312cfdf95f0f3ed763ef662db401", "parameters": 571909635 }, "Eos": { "statistics_sha256": "b25a7628e52247f69a165f8f1e71937bff814a12bf4b030817607320e9101645", "source_model_key": "Eos-v1-candidate", "manifest_sha256": "06d0e9f4980253b01ff43dbfd2dcbef53fe7c5c46cbc564dc3c6f7c1d73a7ef1", "qualified_runtime": { "python": "3.12.13", "numpy": "2.3.5" }, "hub_revision": "a121640a2332b14548453089cb77badcc9fb9a06", "parameters": 753446208 } }, "models": { "Jev": { "panels": { "old_core": { "point": 0.7909598214285715, "exact_fraction": "7087/8960", "ci95": [ 0.7616062127976191, 0.819271763392857 ] }, "v3_core": { "point": 0.66375, "exact_fraction": "531/800", "ci95": [ 0.6316666666666667, 0.6945833333333333 ] }, "v4": { "point": 0.9453125, "exact_fraction": "121/128", "ci95": [ 0.9197115384615384, 0.9685135514495712 ] }, "v5": { "point": 0.8979166666666667, "exact_fraction": "431/480", "ci95": [ 0.8713251366120218, 0.9229854264081012 ] }, "transfer_v9_test": { "point": 0.8718929254302104, "exact_fraction": "456/523", "ci95": [ 0.8485404372986687, 0.893925148300751 ] }, "original_four_panel_mean": { "point": 0.8244847470238095, "exact_fraction": "443243/537600", "ci95": [ 0.8104711862785249, 0.8378880015726611 ] }, "weighted_mean": { "point": 0.8104937602431029, "exact_fraction": "18990193/23430400", "ci95": [ 0.7970371095822852, 0.8234617528862396 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 109, "requested": 128 }, "boolean_constraints": { "correct": 64, "requested": 64 }, "dbpedia_14": { "correct": 108, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 29, "requested": 96 }, "ordinal_rubric": { "correct": 64, "requested": 64 }, "relational_composition": { "correct": 54, "requested": 96 }, "scoped_evidence": { "correct": 86, "requested": 96 }, "state_tracking": { "correct": 32, "requested": 96 }, "unknown_rejection": { "correct": 64, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 61, "requested": 80 }, "capacitated_assignment": { "correct": 55, "requested": 80 }, "constraint_assignment": { "correct": 44, "requested": 80 }, "massive_en": { "correct": 110, "requested": 120 }, "massive_zh": { "correct": 106, "requested": 120 }, "multiset_reconciliation": { "correct": 47, "requested": 80 }, "ordinal_service_loss": { "correct": 34, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 53, "requested": 80 }, "temporal_exclusion": { "correct": 33, "requested": 80 }, "transaction_recovery": { "correct": 60, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 148, "requested": 160 }, "belebele_en": { "correct": 155, "requested": 160 }, "belebele_zh": { "correct": 154, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 104, "requested": 120 }, "squad2_answerability": { "correct": 109, "requested": 120 }, "snli": { "correct": 99, "requested": 120 }, "qasc": { "correct": 119, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 912, "accuracy": 0.8718929254302104, "full_probability_metrics": { "n": 1046, "nll": 0.5742897471161112, "acc": 0.8718929254302104, "ece": 0.03352357224250148, "brier": 0.19121748011041134, "mean_conf": 0.8897186009231899, "confident_error_rate": 0.02581261950286807, "coverage_at_0_9": 0.7093690248565966, "accuracy_at_0_9": 0.9636118598382749, "coverage_at_5pct_error": 0.7695984703632888, "coverage_at_1pct_error": 0.31644359464627153, "aurc": 0.0345843507379907, "error_rate_at_0_9": 0.03638814016172507, "confidence_bias": 0.01782567549297953, "top_bins": { "0.9": { "n": 742, "errors": 27, "error_rate": 0.03638814016172507 }, "0.95": { "n": 649, "errors": 21, "error_rate": 0.032357473035439135 }, "0.99": { "n": 410, "errors": 6, "error_rate": 0.014634146341463415 } }, "selective": { "0.5": { "coverage": 0.5630975143403442, "accuracy": 0.9745331069609507, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8040152963671128, "accuracy": 0.9417360285374554, "confidence_cutoff": 0.79 } }, "score_mae": 0.11841919191919192, "ranked_probability_score": 0.024702299408223646 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.5742897471161112, "acc": 0.8718929254302104, "ece": 0.03352357224250148, "brier": 0.19121748011041134, "mean_conf": 0.8897186009231899, "confident_error_rate": 0.02581261950286807, "coverage_at_0_9": 0.7093690248565966, "accuracy_at_0_9": 0.9636118598382749, "coverage_at_5pct_error": 0.7695984703632888, "coverage_at_1pct_error": 0.31644359464627153, "aurc": 0.0345843507379907, "error_rate_at_0_9": 0.03638814016172507, "confidence_bias": 0.01782567549297953, "top_bins": { "0.9": { "n": 742, "errors": 27, "error_rate": 0.03638814016172507 }, "0.95": { "n": 649, "errors": 21, "error_rate": 0.032357473035439135 }, "0.99": { "n": 410, "errors": 6, "error_rate": 0.014634146341463415 } }, "selective": { "0.5": { "coverage": 0.5630975143403442, "accuracy": 0.9745331069609507, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8040152963671128, "accuracy": 0.9417360285374554, "confidence_cutoff": 0.79 } }, "score_mae": 0.11841919191919192, "ranked_probability_score": 0.024702299408223646 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 14, "accuracy": 0.7, "full_probability_metrics": { "n": 20, "nll": 0.8467821724294178, "acc": 0.7, "ece": 0.14900000000000002, "brier": 0.42706999999999995, "mean_conf": 0.7550000000000001, "confident_error_rate": 0.05, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.16393868558342242, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.05500000000000016, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.81 }, "0.8": { "coverage": 0.9, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.53 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.8467821724294178, "acc": 0.7, "ece": 0.14900000000000002, "brier": 0.42706999999999995, "mean_conf": 0.7550000000000001, "confident_error_rate": 0.05, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.16393868558342242, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.05500000000000016, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.81 }, "0.8": { "coverage": 0.9, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.53 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 19, "accuracy": 0.95, "full_probability_metrics": { "n": 20, "nll": 0.24546392070418083, "acc": 0.95, "ece": 0.1315, "brier": 0.13135000000000002, "mean_conf": 0.8185, "confident_error_rate": 0.0, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.75, "aurc": 0.013791709666322668, "error_rate_at_0_9": 0.0, "confidence_bias": -0.13149999999999995, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.85, "accuracy": 0.9411764705882353, "confidence_cutoff": 0.65 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.24546392070418083, "acc": 0.95, "ece": 0.1315, "brier": 0.13135000000000002, "mean_conf": 0.8185, "confident_error_rate": 0.0, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.75, "aurc": 0.013791709666322668, "error_rate_at_0_9": 0.0, "confidence_bias": -0.13149999999999995, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.85, "accuracy": 0.9411764705882353, "confidence_cutoff": 0.65 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 19, "accuracy": 0.95, "full_probability_metrics": { "n": 20, "nll": 0.2431438312479735, "acc": 0.95, "ece": 0.0515000000000001, "brier": 0.12456999999999999, "mean_conf": 0.9155, "confident_error_rate": 0.05, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.4, "aurc": 0.04093456680917982, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": -0.034499999999999975, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.9166666666666666, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.85, "accuracy": 0.9411764705882353, "confidence_cutoff": 0.9 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.2431438312479735, "acc": 0.95, "ece": 0.0515000000000001, "brier": 0.12456999999999999, "mean_conf": 0.9155, "confident_error_rate": 0.05, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.4, "aurc": 0.04093456680917982, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": -0.034499999999999975, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.9166666666666666, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.85, "accuracy": 0.9411764705882353, "confidence_cutoff": 0.9 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 11, "accuracy": 0.55, "full_probability_metrics": { "n": 20, "nll": 0.7003153021897492, "acc": 0.55, "ece": 0.221, "brier": 0.50352, "mean_conf": 0.6940000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3102403959989099, "error_rate_at_0_9": 0.0, "confidence_bias": 0.14400000000000002, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.71 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.56 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.7003153021897492, "acc": 0.55, "ece": 0.221, "brier": 0.50352, "mean_conf": 0.6940000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3102403959989099, "error_rate_at_0_9": 0.0, "confidence_bias": 0.14400000000000002, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.71 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.56 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 31, "accuracy": 0.96875, "full_probability_metrics": { "n": 32, "nll": 0.12990463418239556, "acc": 0.96875, "ece": 0.10812499999999997, "brier": 0.0653875, "mean_conf": 0.891875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "aurc": 0.0009765625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07687500000000003, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.76 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.12990463418239556, "acc": 0.96875, "ece": 0.10812499999999997, "brier": 0.0653875, "mean_conf": 0.891875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "aurc": 0.0009765625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07687500000000003, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.76 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 32, "accuracy": 1.0, "full_probability_metrics": { "n": 32, "nll": 0.036098701740559194, "acc": 1.0, "ece": 0.032500000000000064, "brier": 0.0121875, "mean_conf": 0.9675, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03249999999999997, "top_bins": { "0.9": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6875, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.036098701740559194, "acc": 1.0, "ece": 0.032500000000000064, "brier": 0.0121875, "mean_conf": 0.9675, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03249999999999997, "top_bins": { "0.9": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6875, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 29, "accuracy": 0.90625, "full_probability_metrics": { "n": 32, "nll": 0.191794305363818, "acc": 0.90625, "ece": 0.0840625, "brier": 0.14094374999999998, "mean_conf": 0.9846875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.90625, "aurc": 0.006961945564516129, "error_rate_at_0_9": 0.0, "confidence_bias": 0.07843750000000005, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.71875, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.99 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.191794305363818, "acc": 0.90625, "ece": 0.0840625, "brier": 0.14094374999999998, "mean_conf": 0.9846875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.90625, "aurc": 0.006961945564516129, "error_rate_at_0_9": 0.0, "confidence_bias": 0.07843750000000005, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.71875, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.99 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 40, "accuracy": 1.0, "full_probability_metrics": { "n": 40, "nll": 0.023792482376035658, "acc": 1.0, "ece": 0.023500000000000076, "brier": 0.0011500000000000013, "mean_conf": 0.9764999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.023500000000000076, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.65, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.023792482376035658, "acc": 1.0, "ece": 0.023500000000000076, "brier": 0.0011500000000000013, "mean_conf": 0.9764999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.023500000000000076, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.65, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 37, "accuracy": 0.925, "full_probability_metrics": { "n": 40, "nll": 0.21517298280454894, "acc": 0.925, "ece": 0.05920454545454551, "brier": 0.11905616671768186, "mean_conf": 0.8882045454545455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.7, "aurc": 0.013344366138263706, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03679545454545452, "top_bins": { "0.9": { "n": 27, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.81 } }, "score_mae": 0.1527979797979798, "ranked_probability_score": 0.03465699852055913 }, "covered_only_probability_metrics": { "n": 40, "nll": 0.21517298280454894, "acc": 0.925, "ece": 0.05920454545454551, "brier": 0.11905616671768186, "mean_conf": 0.8882045454545455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.7, "aurc": 0.013344366138263706, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03679545454545452, "top_bins": { "0.9": { "n": 27, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.81 } }, "score_mae": 0.1527979797979798, "ranked_probability_score": 0.03465699852055913 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 54, "accuracy": 0.675, "full_probability_metrics": { "n": 80, "nll": 3.465926158827294, "acc": 0.675, "ece": 0.21861742424242422, "brier": 0.5462414212835425, "mean_conf": 0.8728674242424243, "confident_error_rate": 0.125, "coverage_at_0_9": 0.6125, "accuracy_at_0_9": 0.7959183673469388, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.16764373290710635, "error_rate_at_0_9": 0.20408163265306123, "confidence_bias": 0.1978674242424242, "top_bins": { "0.9": { "n": 49, "errors": 10, "error_rate": 0.20408163265306123 }, "0.95": { "n": 47, "errors": 10, "error_rate": 0.2127659574468085 }, "0.99": { "n": 36, "errors": 5, "error_rate": 0.1388888888888889 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.8048780487804879, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.72 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 3.465926158827294, "acc": 0.675, "ece": 0.21861742424242422, "brier": 0.5462414212835425, "mean_conf": 0.8728674242424243, "confident_error_rate": 0.125, "coverage_at_0_9": 0.6125, "accuracy_at_0_9": 0.7959183673469388, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.16764373290710635, "error_rate_at_0_9": 0.20408163265306123, "confidence_bias": 0.1978674242424242, "top_bins": { "0.9": { "n": 49, "errors": 10, "error_rate": 0.20408163265306123 }, "0.95": { "n": 47, "errors": 10, "error_rate": 0.2127659574468085 }, "0.99": { "n": 36, "errors": 5, "error_rate": 0.1388888888888889 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.8048780487804879, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.72 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 71, "accuracy": 0.8875, "full_probability_metrics": { "n": 80, "nll": 0.2850843907580966, "acc": 0.8875, "ece": 0.039125000000000125, "brier": 0.1542, "mean_conf": 0.9158750000000001, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9841269841269841, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.75, "aurc": 0.01330352725886717, "error_rate_at_0_9": 0.015873015873015872, "confidence_bias": 0.02837500000000015, "top_bins": { "0.9": { "n": 63, "errors": 1, "error_rate": 0.015873015873015872 }, "0.95": { "n": 58, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 45, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.89 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.2850843907580966, "acc": 0.8875, "ece": 0.039125000000000125, "brier": 0.1542, "mean_conf": 0.9158750000000001, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9841269841269841, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.75, "aurc": 0.01330352725886717, "error_rate_at_0_9": 0.015873015873015872, "confidence_bias": 0.02837500000000015, "top_bins": { "0.9": { "n": 63, "errors": 1, "error_rate": 0.015873015873015872 }, "0.95": { "n": 58, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 45, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.89 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 168, "accuracy": 0.84, "full_probability_metrics": { "n": 200, "nll": 0.7427812144511485, "acc": 0.84, "ece": 0.10400959595959602, "brier": 0.2620076191204979, "mean_conf": 0.8334904040404041, "confident_error_rate": 0.05, "coverage_at_0_9": 0.64, "accuracy_at_0_9": 0.921875, "coverage_at_5pct_error": 0.515, "coverage_at_1pct_error": 0.325, "aurc": 0.052116153271370716, "error_rate_at_0_9": 0.078125, "confidence_bias": -0.006509595959595882, "top_bins": { "0.9": { "n": 128, "errors": 10, "error_rate": 0.078125 }, "0.95": { "n": 111, "errors": 7, "error_rate": 0.06306306306306306 }, "0.99": { "n": 85, "errors": 1, "error_rate": 0.011764705882352941 } }, "selective": { "0.5": { "coverage": 0.515, "accuracy": 0.9514563106796117, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.9125, "confidence_cutoff": 0.6 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 0.7427812144511485, "acc": 0.84, "ece": 0.10400959595959602, "brier": 0.2620076191204979, "mean_conf": 0.8334904040404041, "confident_error_rate": 0.05, "coverage_at_0_9": 0.64, "accuracy_at_0_9": 0.921875, "coverage_at_5pct_error": 0.515, "coverage_at_1pct_error": 0.325, "aurc": 0.052116153271370716, "error_rate_at_0_9": 0.078125, "confidence_bias": -0.006509595959595882, "top_bins": { "0.9": { "n": 128, "errors": 10, "error_rate": 0.078125 }, "0.95": { "n": 111, "errors": 7, "error_rate": 0.06306306306306306 }, "0.99": { "n": 85, "errors": 1, "error_rate": 0.011764705882352941 } }, "selective": { "0.5": { "coverage": 0.515, "accuracy": 0.9514563106796117, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.9125, "confidence_cutoff": 0.6 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 70, "accuracy": 0.875, "full_probability_metrics": { "n": 80, "nll": 0.257969262004998, "acc": 0.875, "ece": 0.06662499999999996, "brier": 0.1524375, "mean_conf": 0.8696249999999999, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9807692307692307, "coverage_at_5pct_error": 0.85, "coverage_at_1pct_error": 0.15, "aurc": 0.03460551485249203, "error_rate_at_0_9": 0.019230769230769232, "confidence_bias": -0.00537500000000013, "top_bins": { "0.9": { "n": 52, "errors": 1, "error_rate": 0.019230769230769232 }, "0.95": { "n": 38, "errors": 1, "error_rate": 0.02631578947368421 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.975609756097561, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.75 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.257969262004998, "acc": 0.875, "ece": 0.06662499999999996, "brier": 0.1524375, "mean_conf": 0.8696249999999999, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9807692307692307, "coverage_at_5pct_error": 0.85, "coverage_at_1pct_error": 0.15, "aurc": 0.03460551485249203, "error_rate_at_0_9": 0.019230769230769232, "confidence_bias": -0.00537500000000013, "top_bins": { "0.9": { "n": 52, "errors": 1, "error_rate": 0.019230769230769232 }, "0.95": { "n": 38, "errors": 1, "error_rate": 0.02631578947368421 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.975609756097561, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.75 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 73, "accuracy": 0.9125, "full_probability_metrics": { "n": 80, "nll": 0.1977134395476539, "acc": 0.9125, "ece": 0.0483750000000001, "brier": 0.11659250000000002, "mean_conf": 0.914875, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9841269841269841, "coverage_at_5pct_error": 0.9125, "coverage_at_1pct_error": 0.5, "aurc": 0.015283246404822943, "error_rate_at_0_9": 0.015873015873015872, "confidence_bias": 0.002375000000000016, "top_bins": { "0.9": { "n": 63, "errors": 1, "error_rate": 0.015873015873015872 }, "0.95": { "n": 51, "errors": 1, "error_rate": 0.0196078431372549 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.89 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.1977134395476539, "acc": 0.9125, "ece": 0.0483750000000001, "brier": 0.11659250000000002, "mean_conf": 0.914875, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9841269841269841, "coverage_at_5pct_error": 0.9125, "coverage_at_1pct_error": 0.5, "aurc": 0.015283246404822943, "error_rate_at_0_9": 0.015873015873015872, "confidence_bias": 0.002375000000000016, "top_bins": { "0.9": { "n": 63, "errors": 1, "error_rate": 0.015873015873015872 }, "0.95": { "n": 51, "errors": 1, "error_rate": 0.0196078431372549 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.89 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 80, "accuracy": 1.0, "full_probability_metrics": { "n": 80, "nll": 0.0002512583963375363, "acc": 1.0, "ece": 0.00024999999999997247, "brier": 5.000000000000005e-06, "mean_conf": 0.99975, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00024999999999997247, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 80, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.975, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.975, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.0002512583963375363, "acc": 1.0, "ece": 0.00024999999999997247, "brier": 5.000000000000005e-06, "mean_conf": 0.99975, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00024999999999997247, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 80, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.975, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.975, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 61, "accuracy": 0.7625, "full_probability_metrics": { "n": 80, "nll": 0.4679571929121237, "acc": 0.7625, "ece": 0.07849999999999999, "brier": 0.30820499999999995, "mean_conf": 0.841, "confident_error_rate": 0.025, "coverage_at_0_9": 0.4875, "accuracy_at_0_9": 0.9487179487179487, "coverage_at_5pct_error": 0.4625, "coverage_at_1pct_error": 0.1625, "aurc": 0.09418592802405981, "error_rate_at_0_9": 0.05128205128205128, "confidence_bias": 0.07850000000000001, "top_bins": { "0.9": { "n": 39, "errors": 2, "error_rate": 0.05128205128205128 }, "0.95": { "n": 27, "errors": 1, "error_rate": 0.037037037037037035 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.9024390243902439, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.67 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4679571929121237, "acc": 0.7625, "ece": 0.07849999999999999, "brier": 0.30820499999999995, "mean_conf": 0.841, "confident_error_rate": 0.025, "coverage_at_0_9": 0.4875, "accuracy_at_0_9": 0.9487179487179487, "coverage_at_5pct_error": 0.4625, "coverage_at_1pct_error": 0.1625, "aurc": 0.09418592802405981, "error_rate_at_0_9": 0.05128205128205128, "confidence_bias": 0.07850000000000001, "top_bins": { "0.9": { "n": 39, "errors": 2, "error_rate": 0.05128205128205128 }, "0.95": { "n": 27, "errors": 1, "error_rate": 0.037037037037037035 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.9024390243902439, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.67 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.025330957401114018, "acc": 1.0, "ece": 0.025000000000000022, "brier": 0.0013000000000000021, "mean_conf": 0.975, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.025000000000000022, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.025330957401114018, "acc": 1.0, "ece": 0.025000000000000022, "brier": 0.0013000000000000021, "mean_conf": 0.975, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.025000000000000022, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.02225400735095729, "acc": 1.0, "ece": 0.02199999999999991, "brier": 0.0010000000000000013, "mean_conf": 0.9780000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02199999999999991, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.02225400735095729, "acc": 1.0, "ece": 0.02199999999999991, "brier": 0.0010000000000000013, "mean_conf": 0.9780000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02199999999999991, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.14371680526465735, "acc": 1.0, "ece": 0.12199999999999996, "brier": 0.06044000000000001, "mean_conf": 0.8779999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.12200000000000011, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.81 } }, "score_mae": 0.13499999999999998, "ranked_probability_score": 0.017395 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.14371680526465735, "acc": 1.0, "ece": 0.12199999999999996, "brier": 0.06044000000000001, "mean_conf": 0.8779999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.12200000000000011, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.81 } }, "score_mae": 0.13499999999999998, "ranked_probability_score": 0.017395 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.001005033585350145, "acc": 1.0, "ece": 0.0010000000000000009, "brier": 2.000000000000002e-05, "mean_conf": 0.999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0010000000000000009, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0010000000000000009, "ranked_probability_score": 5.000000000000009e-06 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.001005033585350145, "acc": 1.0, "ece": 0.0010000000000000009, "brier": 2.000000000000002e-05, "mean_conf": 0.999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0010000000000000009, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0010000000000000009, "ranked_probability_score": 5.000000000000009e-06 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.025012694561760796, "acc": 1.0, "ece": 0.02400000000000002, "brier": 0.003459999999999999, "mean_conf": 0.976, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02400000000000002, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.025012694561760796, "acc": 1.0, "ece": 0.02400000000000002, "brier": 0.003459999999999999, "mean_conf": 0.976, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02400000000000002, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.7682522471645241, "acc": 0.5, "ece": 0.378, "brier": 0.5646399999999999, "mean_conf": 0.836, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.20277777777777778, "error_rate_at_0_9": 0.0, "confidence_bias": 0.33599999999999997, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.82 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7682522471645241, "acc": 0.5, "ece": 0.378, "brier": 0.5646399999999999, "mean_conf": 0.836, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.20277777777777778, "error_rate_at_0_9": 0.0, "confidence_bias": 0.33599999999999997, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.82 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.14478249721081218, "acc": 0.9, "ece": 0.11100000000000004, "brier": 0.07976000000000001, "mean_conf": 0.901, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0010000000000000009, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.84 } }, "score_mae": 0.12800000000000003, "ranked_probability_score": 0.021660000000000002 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.14478249721081218, "acc": 0.9, "ece": 0.11100000000000004, "brier": 0.07976000000000001, "mean_conf": 0.901, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0010000000000000009, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.84 } }, "score_mae": 0.12800000000000003, "ranked_probability_score": 0.021660000000000002 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.00201006717070029, "acc": 1.0, "ece": 0.0020000000000000018, "brier": 4.000000000000004e-05, "mean_conf": 0.998, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0020000000000000018, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.002000000000000001, "ranked_probability_score": 1.0000000000000018e-05 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.00201006717070029, "acc": 1.0, "ece": 0.0020000000000000018, "brier": 4.000000000000004e-05, "mean_conf": 0.998, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0020000000000000018, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.002000000000000001, "ranked_probability_score": 1.0000000000000018e-05 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0, "ranked_probability_score": 0.0 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0, "ranked_probability_score": 0.0 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.511280564057713, "acc": 0.9, "ece": 0.284, "brier": 0.27598, "mean_conf": 0.6160000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": null, "confidence_bias": -0.2839999999999999, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.65 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.56 } }, "score_mae": 0.30700000000000005, "ranked_probability_score": 0.069325 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.511280564057713, "acc": 0.9, "ece": 0.284, "brier": 0.27598, "mean_conf": 0.6160000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": null, "confidence_bias": -0.2839999999999999, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.65 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.56 } }, "score_mae": 0.30700000000000005, "ranked_probability_score": 0.069325 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.8963888888888888, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.8611111111111112, "flip_rate": 0.0, "covered_only_flip_rate": 0.0, "half_l1": 0.020833333333333332 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.6203636363636365, "share_at_0_9": 0.16363636363636364, "normalized_entropy": 0.7656918947082642 }, "covered_only_confidence_metrics": { "mean_max_p": 0.6203636363636365, "share_at_0_9": 0.16363636363636364, "normalized_entropy": 0.7656918947082642 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 103, "accuracy": 0.9363636363636364, "full_probability_metrics": { "n": 110, "nll": 0.14942226125159902, "acc": 0.9363636363636364, "ece": 0.05990909090909094, "brier": 0.08969454545454544, "mean_conf": 0.9233636363636362, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7818181818181819, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9545454545454546, "coverage_at_1pct_error": 0.7818181818181819, "aurc": 0.007315902581037094, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013000000000000234, "top_bins": { "0.9": { "n": 86, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 54, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6363636363636364, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.9886363636363636, "confidence_cutoff": 0.89 } }, "score_mae": 0.0955, "ranked_probability_score": 0.018065833333333337 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.14942226125159902, "acc": 0.9363636363636364, "ece": 0.05990909090909094, "brier": 0.08969454545454544, "mean_conf": 0.9233636363636362, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7818181818181819, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9545454545454546, "coverage_at_1pct_error": 0.7818181818181819, "aurc": 0.007315902581037094, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013000000000000234, "top_bins": { "0.9": { "n": 86, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 54, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6363636363636364, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.9886363636363636, "confidence_cutoff": 0.89 } }, "score_mae": 0.0955, "ranked_probability_score": 0.018065833333333337 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.303 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.3590294860631082, "paired_flip": { "pairs": 64, "flip_rate": 0.984375, "both_correct_rate": 0.9375, "invariant_pairs": 24, "invariance_rate": 0.9583333333333334, "invariant_both_correct_rate": 0.9166666666666666 }, "unknowable": { "n": 110, "mean_max_p": 0.6203636363636365, "share_at_0_9": 0.16363636363636364, "control_mean_max_p": 0.9233636363636362, "control_share_at_0_9": 0.7818181818181819, "control_acc": 0.9363636363636364, "paired_confidence_drop": 0.303, "share_less_confident_than_control": 0.9 }, "clean": { "n": 1046, "nll": 0.5742897471161112, "acc": 0.8718929254302104, "ece": 0.03352357224250148, "brier": 0.19121748011041134, "mean_conf": 0.8897186009231899, "confident_error_rate": 0.02581261950286807, "coverage_at_0_9": 0.7093690248565966, "accuracy_at_0_9": 0.9636118598382749, "coverage_at_5pct_error": 0.7695984703632888, "coverage_at_1pct_error": 0.31644359464627153, "aurc": 0.0345843507379907, "error_rate_at_0_9": 0.03638814016172507, "confidence_bias": 0.01782567549297953, "top_bins": { "0.9": { "n": 742, "errors": 27, "error_rate": 0.03638814016172507 }, "0.95": { "n": 649, "errors": 21, "error_rate": 0.032357473035439135 }, "0.99": { "n": 410, "errors": 6, "error_rate": 0.014634146341463415 } }, "selective": { "0.5": { "coverage": 0.5630975143403442, "accuracy": 0.9745331069609507, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8040152963671128, "accuracy": 0.9417360285374554, "confidence_cutoff": 0.79 } }, "score_mae": 0.11841919191919192, "ranked_probability_score": 0.024702299408223646 }, "tasks": { "buried_emotion": { "n": 20, "nll": 0.8467821724294178, "acc": 0.7, "ece": 0.14900000000000002, "brier": 0.42706999999999995, "mean_conf": 0.7550000000000001, "confident_error_rate": 0.05, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.16393868558342242, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.05500000000000016, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.81 }, "0.8": { "coverage": 0.9, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.53 } } }, "buried_paws": { "n": 20, "nll": 0.24546392070418083, "acc": 0.95, "ece": 0.1315, "brier": 0.13135000000000002, "mean_conf": 0.8185, "confident_error_rate": 0.0, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.75, "aurc": 0.013791709666322668, "error_rate_at_0_9": 0.0, "confidence_bias": -0.13149999999999995, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.85, "accuracy": 0.9411764705882353, "confidence_cutoff": 0.65 } } }, "buried_qnli": { "n": 20, "nll": 0.2431438312479735, "acc": 0.95, "ece": 0.0515000000000001, "brier": 0.12456999999999999, "mean_conf": 0.9155, "confident_error_rate": 0.05, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.4, "aurc": 0.04093456680917982, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": -0.034499999999999975, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.9166666666666666, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.85, "accuracy": 0.9411764705882353, "confidence_cutoff": 0.9 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.7003153021897492, "acc": 0.55, "ece": 0.221, "brier": 0.50352, "mean_conf": 0.6940000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3102403959989099, "error_rate_at_0_9": 0.0, "confidence_bias": 0.14400000000000002, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.71 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.56 } } }, "composition_final_combination": { "n": 32, "nll": 0.12990463418239556, "acc": 0.96875, "ece": 0.10812499999999997, "brier": 0.0653875, "mean_conf": 0.891875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "aurc": 0.0009765625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07687500000000003, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.76 } } }, "composition_final_exception": { "n": 32, "nll": 0.036098701740559194, "acc": 1.0, "ece": 0.032500000000000064, "brier": 0.0121875, "mean_conf": 0.9675, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03249999999999997, "top_bins": { "0.9": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6875, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "composition_final_negation": { "n": 32, "nll": 0.191794305363818, "acc": 0.90625, "ece": 0.0840625, "brier": 0.14094374999999998, "mean_conf": 0.9846875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.90625, "aurc": 0.006961945564516129, "error_rate_at_0_9": 0.0, "confidence_bias": 0.07843750000000005, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.71875, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.99 } } }, "contrastive_authorization": { "n": 40, "nll": 0.023792482376035658, "acc": 1.0, "ece": 0.023500000000000076, "brier": 0.0011500000000000013, "mean_conf": 0.9764999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.023500000000000076, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.65, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_deadline": { "n": 40, "nll": 0.21517298280454894, "acc": 0.925, "ece": 0.05920454545454551, "brier": 0.11905616671768186, "mean_conf": 0.8882045454545455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.7, "aurc": 0.013344366138263706, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03679545454545452, "top_bins": { "0.9": { "n": 27, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.81 } }, "score_mae": 0.1527979797979798, "ranked_probability_score": 0.03465699852055913 }, "emotion": { "n": 80, "nll": 3.465926158827294, "acc": 0.675, "ece": 0.21861742424242422, "brier": 0.5462414212835425, "mean_conf": 0.8728674242424243, "confident_error_rate": 0.125, "coverage_at_0_9": 0.6125, "accuracy_at_0_9": 0.7959183673469388, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.16764373290710635, "error_rate_at_0_9": 0.20408163265306123, "confidence_bias": 0.1978674242424242, "top_bins": { "0.9": { "n": 49, "errors": 10, "error_rate": 0.20408163265306123 }, "0.95": { "n": 47, "errors": 10, "error_rate": 0.2127659574468085 }, "0.99": { "n": 36, "errors": 5, "error_rate": 0.1388888888888889 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.8048780487804879, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.72 } } }, "mmlu": { "n": 80, "nll": 0.2850843907580966, "acc": 0.8875, "ece": 0.039125000000000125, "brier": 0.1542, "mean_conf": 0.9158750000000001, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9841269841269841, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.75, "aurc": 0.01330352725886717, "error_rate_at_0_9": 0.015873015873015872, "confidence_bias": 0.02837500000000015, "top_bins": { "0.9": { "n": 63, "errors": 1, "error_rate": 0.015873015873015872 }, "0.95": { "n": 58, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 45, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.89 } } }, "mmlu_pro": { "n": 200, "nll": 0.7427812144511485, "acc": 0.84, "ece": 0.10400959595959602, "brier": 0.2620076191204979, "mean_conf": 0.8334904040404041, "confident_error_rate": 0.05, "coverage_at_0_9": 0.64, "accuracy_at_0_9": 0.921875, "coverage_at_5pct_error": 0.515, "coverage_at_1pct_error": 0.325, "aurc": 0.052116153271370716, "error_rate_at_0_9": 0.078125, "confidence_bias": -0.006509595959595882, "top_bins": { "0.9": { "n": 128, "errors": 10, "error_rate": 0.078125 }, "0.95": { "n": 111, "errors": 7, "error_rate": 0.06306306306306306 }, "0.99": { "n": 85, "errors": 1, "error_rate": 0.011764705882352941 } }, "selective": { "0.5": { "coverage": 0.515, "accuracy": 0.9514563106796117, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.9125, "confidence_cutoff": 0.6 } } }, "paws": { "n": 80, "nll": 0.257969262004998, "acc": 0.875, "ece": 0.06662499999999996, "brier": 0.1524375, "mean_conf": 0.8696249999999999, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9807692307692307, "coverage_at_5pct_error": 0.85, "coverage_at_1pct_error": 0.15, "aurc": 0.03460551485249203, "error_rate_at_0_9": 0.019230769230769232, "confidence_bias": -0.00537500000000013, "top_bins": { "0.9": { "n": 52, "errors": 1, "error_rate": 0.019230769230769232 }, "0.95": { "n": 38, "errors": 1, "error_rate": 0.02631578947368421 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.975609756097561, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.75 } } }, "qnli": { "n": 80, "nll": 0.1977134395476539, "acc": 0.9125, "ece": 0.0483750000000001, "brier": 0.11659250000000002, "mean_conf": 0.914875, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9841269841269841, "coverage_at_5pct_error": 0.9125, "coverage_at_1pct_error": 0.5, "aurc": 0.015283246404822943, "error_rate_at_0_9": 0.015873015873015872, "confidence_bias": 0.002375000000000016, "top_bins": { "0.9": { "n": 63, "errors": 1, "error_rate": 0.015873015873015872 }, "0.95": { "n": 51, "errors": 1, "error_rate": 0.0196078431372549 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.89 } } }, "sciq": { "n": 80, "nll": 0.0002512583963375363, "acc": 1.0, "ece": 0.00024999999999997247, "brier": 5.000000000000005e-06, "mean_conf": 0.99975, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00024999999999997247, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 80, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.975, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.975, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "tweet_offensive": { "n": 80, "nll": 0.4679571929121237, "acc": 0.7625, "ece": 0.07849999999999999, "brier": 0.30820499999999995, "mean_conf": 0.841, "confident_error_rate": 0.025, "coverage_at_0_9": 0.4875, "accuracy_at_0_9": 0.9487179487179487, "coverage_at_5pct_error": 0.4625, "coverage_at_1pct_error": 0.1625, "aurc": 0.09418592802405981, "error_rate_at_0_9": 0.05128205128205128, "confidence_bias": 0.07850000000000001, "top_bins": { "0.9": { "n": 39, "errors": 2, "error_rate": 0.05128205128205128 }, "0.95": { "n": 27, "errors": 1, "error_rate": 0.037037037037037035 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.9024390243902439, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.67 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.7020952341644959, "acc": 0.5, "ece": 0.062000000000000055, "brier": 0.5088400000000001, "mean_conf": 0.562, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.062000000000000055, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.56 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.56 } } }, "unknowable_authorization": { "n": 10, "nll": 0.6897824907454839, "acc": 0.6, "ece": 0.08399999999999996, "brier": 0.4966399999999999, "mean_conf": 0.516, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.42333333333333334, "error_rate_at_0_9": null, "confidence_bias": -0.08399999999999996, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.52 }, "0.8": { "coverage": 0.9, "accuracy": 0.5555555555555556, "confidence_cutoff": 0.51 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.025330957401114018, "acc": 1.0, "ece": 0.025000000000000022, "brier": 0.0013000000000000021, "mean_conf": 0.975, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.025000000000000022, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.02225400735095729, "acc": 1.0, "ece": 0.02199999999999991, "brier": 0.0010000000000000013, "mean_conf": 0.9780000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02199999999999991, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "unknowable_control_deadline": { "n": 10, "nll": 0.14371680526465735, "acc": 1.0, "ece": 0.12199999999999996, "brier": 0.06044000000000001, "mean_conf": 0.8779999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.12200000000000011, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.81 } }, "score_mae": 0.13499999999999998, "ranked_probability_score": 0.017395 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.001005033585350145, "acc": 1.0, "ece": 0.0010000000000000009, "brier": 2.000000000000002e-05, "mean_conf": 0.999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0010000000000000009, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0010000000000000009, "ranked_probability_score": 5.000000000000009e-06 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.025012694561760796, "acc": 1.0, "ece": 0.02400000000000002, "brier": 0.003459999999999999, "mean_conf": 0.976, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02400000000000002, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.7682522471645241, "acc": 0.5, "ece": 0.378, "brier": 0.5646399999999999, "mean_conf": 0.836, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.20277777777777778, "error_rate_at_0_9": 0.0, "confidence_bias": 0.33599999999999997, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.82 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.14478249721081218, "acc": 0.9, "ece": 0.11100000000000004, "brier": 0.07976000000000001, "mean_conf": 0.901, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0010000000000000009, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.84 } }, "score_mae": 0.12800000000000003, "ranked_probability_score": 0.021660000000000002 }, "unknowable_control_sla_response": { "n": 10, "nll": 0.00201006717070029, "acc": 1.0, "ece": 0.0020000000000000018, "brier": 4.000000000000004e-05, "mean_conf": 0.998, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0020000000000000018, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.002000000000000001, "ranked_probability_score": 1.0000000000000018e-05 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0, "ranked_probability_score": 0.0 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.511280564057713, "acc": 0.9, "ece": 0.284, "brier": 0.27598, "mean_conf": 0.6160000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": null, "confidence_bias": -0.2839999999999999, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.65 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.56 } }, "score_mae": 0.30700000000000005, "ranked_probability_score": 0.069325 }, "unknowable_deadline": { "n": 10, "nll": 1.0379891843564246, "acc": 0.5, "ece": 0.03500000000000001, "brier": 0.62426, "mean_conf": 0.477, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.38603174603174606, "error_rate_at_0_9": null, "confidence_bias": -0.02300000000000002, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.48 }, "0.8": { "coverage": 0.9, "accuracy": 0.5555555555555556, "confidence_cutoff": 0.45 } }, "score_mae": 0.6799999999999999, "ranked_probability_score": 0.19992 }, "unknowable_late_fee": { "n": 10, "nll": 1.4916923313610093, "acc": 0.3, "ece": 0.36800000000000005, "brier": 0.9186799999999999, "mean_conf": 0.648, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7066666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.34800000000000003, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.67 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.61 } }, "score_mae": 0.812, "ranked_probability_score": 0.29361999999999994 }, "unknowable_quantity_limit": { "n": 10, "nll": 8.753874307711195, "acc": 0.5, "ece": 0.492, "brier": 0.9904, "mean_conf": 0.992, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.492, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 8, "errors": 4, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.99 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7030304948930253, "acc": 0.5, "ece": 0.06099999999999994, "brier": 0.5097400000000001, "mean_conf": 0.5609999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5166666666666666, "error_rate_at_0_9": null, "confidence_bias": 0.06099999999999994, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.56 }, "0.8": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.55 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.7120134783458076, "acc": 0.2, "ece": 0.531, "brier": 1.0713599999999999, "mean_conf": 0.7310000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7953174603174602, "error_rate_at_0_9": null, "confidence_bias": 0.5310000000000001, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.72 }, "0.8": { "coverage": 0.9, "accuracy": 0.2222222222222222, "confidence_cutoff": 0.7000000000000001 } }, "score_mae": 1.067, "ranked_probability_score": 0.42372499999999996 }, "unknowable_sla_response": { "n": 10, "nll": 1.1876603646604935, "acc": 0.3, "ece": 0.094, "brier": 0.7146800000000001, "mean_conf": 0.394, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6483333333333332, "error_rate_at_0_9": null, "confidence_bias": 0.09400000000000003, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.38 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.36 } }, "score_mae": 0.512, "ranked_probability_score": 0.19988999999999998 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.8197378832451736, "acc": 0.5, "ece": 0.03399999999999996, "brier": 0.52856, "mean_conf": 0.51, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.010000000000000009, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.52 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.49 } } }, "unknowable_volume_discount": { "n": 10, "nll": 2.4725824747586715, "acc": 0.4, "ece": 0.54, "brier": 1.1186399999999999, "mean_conf": 0.9400000000000002, "confident_error_rate": 0.5, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.375, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6116666666666666, "error_rate_at_0_9": 0.625, "confidence_bias": 0.5400000000000001, "top_bins": { "0.9": { "n": 8, "errors": 5, "error_rate": 0.625 }, "0.95": { "n": 6, "errors": 4, "error_rate": 0.6666666666666666 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.93 } }, "score_mae": 0.99, "ranked_probability_score": 0.46264000000000005 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.0996218785205143, "acc": 0.3, "ece": 0.193, "brier": 0.6849799999999999, "mean_conf": 0.49300000000000005, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6914285714285714, "error_rate_at_0_9": null, "confidence_bias": 0.19300000000000006, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.5 }, "0.8": { "coverage": 0.9, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.46 } }, "score_mae": 0.6, "ranked_probability_score": 0.19515 } }, "variants": { "clean": { "n": 1156, "nll": 0.6984497203383003, "acc": 0.828719723183391, "ece": 0.049260948586208116, "brier": 0.24366893096495698, "mean_conf": 0.8640879382055853, "confident_error_rate": 0.03200692041522491, "coverage_at_0_9": 0.657439446366782, "accuracy_at_0_9": 0.9513157894736842, "coverage_at_5pct_error": 0.657439446366782, "coverage_at_1pct_error": 0.0, "aurc": 0.054137673315579275, "error_rate_at_0_9": 0.04868421052631579, "confidence_bias": 0.03536821502219423, "top_bins": { "0.9": { "n": 760, "errors": 37, "error_rate": 0.04868421052631579 }, "0.95": { "n": 665, "errors": 30, "error_rate": 0.045112781954887216 }, "0.99": { "n": 418, "errors": 10, "error_rate": 0.023923444976076555 } }, "selective": { "0.5": { "coverage": 0.5198961937716263, "accuracy": 0.9650582362728786, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8019031141868512, "accuracy": 0.9072276159654801, "confidence_cutoff": 0.68 } }, "score_mae": 0.365324494949495, "ranked_probability_score": 0.12637299963013976 }, "none_absent": { "n": 36, "nll": 0.815433856299905, "acc": 0.6666666666666666, "ece": 0.18005050505050504, "brier": 0.43211627101542927, "mean_conf": 0.839949494949495, "confident_error_rate": 0.1111111111111111, "coverage_at_0_9": 0.5833333333333334, "accuracy_at_0_9": 0.8095238095238095, "coverage_at_5pct_error": 0.3333333333333333, "coverage_at_1pct_error": 0.3333333333333333, "aurc": 0.1274175654735333, "error_rate_at_0_9": 0.19047619047619047, "confidence_bias": 0.17328282828282837, "top_bins": { "0.9": { "n": 21, "errors": 4, "error_rate": 0.19047619047619047 }, "0.95": { "n": 16, "errors": 2, "error_rate": 0.125 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5277777777777778, "accuracy": 0.8947368421052632, "confidence_cutoff": 0.93 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7931034482758621, "confidence_cutoff": 0.71 } } }, "none_present": { "n": 36, "nll": 1.1107517878582784, "acc": 0.6944444444444444, "ece": 0.1898765432098766, "brier": 0.3452622861612761, "mean_conf": 0.8748765432098765, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6666666666666666, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6944444444444444, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.05870655728981203, "error_rate_at_0_9": 0.0, "confidence_bias": 0.1804320987654321, "top_bins": { "0.9": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5277777777777778, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8620689655172413, "confidence_cutoff": 0.69 } } }, "permuted": { "n": 36, "nll": 0.8638874849547608, "acc": 0.8611111111111112, "ece": 0.095, "brier": 0.21623333333333336, "mean_conf": 0.9233333333333332, "confident_error_rate": 0.027777777777777776, "coverage_at_0_9": 0.7777777777777778, "accuracy_at_0_9": 0.9642857142857143, "coverage_at_5pct_error": 0.8055555555555556, "coverage_at_1pct_error": 0.75, "aurc": 0.02347630045892958, "error_rate_at_0_9": 0.03571428571428571, "confidence_bias": 0.06222222222222207, "top_bins": { "0.9": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 }, "0.95": { "n": 27, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 24, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6388888888888888, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.9655172413793104, "confidence_cutoff": 0.89 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.02027777777777778, "flip_rate": 0.0 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.5742897471161112, "acc": 0.8718929254302104, "ece": 0.03352357224250148, "brier": 0.19121748011041134, "mean_conf": 0.8897186009231899, "confident_error_rate": 0.02581261950286807, "coverage_at_0_9": 0.7093690248565966, "accuracy_at_0_9": 0.9636118598382749, "coverage_at_5pct_error": 0.7695984703632888, "coverage_at_1pct_error": 0.31644359464627153, "aurc": 0.0345843507379907, "error_rate_at_0_9": 0.03638814016172507, "confidence_bias": 0.01782567549297953, "top_bins": { "0.9": { "n": 742, "errors": 27, "error_rate": 0.03638814016172507 }, "0.95": { "n": 649, "errors": 21, "error_rate": 0.032357473035439135 }, "0.99": { "n": 410, "errors": 6, "error_rate": 0.014634146341463415 } }, "selective": { "0.5": { "coverage": 0.5630975143403442, "accuracy": 0.9745331069609507, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8040152963671128, "accuracy": 0.9417360285374554, "confidence_cutoff": 0.79 } }, "score_mae": 0.11841919191919192, "ranked_probability_score": 0.024702299408223646 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 2338 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Lux": { "panels": { "old_core": { "point": 0.8438244047619048, "exact_fraction": "11341/13440", "ci95": [ 0.8172237723214285, 0.8705357142857142 ] }, "v3_core": { "point": 0.5275, "exact_fraction": "211/400", "ci95": [ 0.49375, 0.56125 ] }, "v4": { "point": 0.9015625, "exact_fraction": "577/640", "ci95": [ 0.8683564268867926, 0.9324785297746995 ] }, "v5": { "point": 0.9145833333333333, "exact_fraction": "439/480", "ci95": [ 0.8889180415782575, 0.9378879473136665 ] }, "transfer_v9_test": { "point": 0.777246653919694, "exact_fraction": "813/1046", "ci95": [ 0.7490309134919917, 0.8043278866211298 ] }, "original_four_panel_mean": { "point": 0.7968675595238095, "exact_fraction": "107099/134400", "ci95": [ 0.7818603628416357, 0.8112560296673489 ] }, "weighted_mean": { "point": 0.7740311945165256, "exact_fraction": "36271721/46860800", "ci95": [ 0.7601225402834362, 0.7877175850518867 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 114, "requested": 128 }, "boolean_constraints": { "correct": 64, "requested": 64 }, "dbpedia_14": { "correct": 108, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 70, "requested": 96 }, "ordinal_rubric": { "correct": 64, "requested": 64 }, "relational_composition": { "correct": 58, "requested": 96 }, "scoped_evidence": { "correct": 93, "requested": 96 }, "state_tracking": { "correct": 27, "requested": 96 }, "unknown_rejection": { "correct": 64, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 49, "requested": 80 }, "capacitated_assignment": { "correct": 59, "requested": 80 }, "constraint_assignment": { "correct": 26, "requested": 80 }, "massive_en": { "correct": 108, "requested": 120 }, "massive_zh": { "correct": 105, "requested": 120 }, "multiset_reconciliation": { "correct": 23, "requested": 80 }, "ordinal_service_loss": { "correct": 21, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 29, "requested": 80 }, "temporal_exclusion": { "correct": 30, "requested": 80 }, "transaction_recovery": { "correct": 43, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 143, "requested": 160 }, "belebele_en": { "correct": 148, "requested": 160 }, "belebele_zh": { "correct": 143, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 99, "requested": 120 }, "squad2_answerability": { "correct": 112, "requested": 120 }, "snli": { "correct": 111, "requested": 120 }, "qasc": { "correct": 117, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 813, "accuracy": 0.777246653919694, "full_probability_metrics": { "n": 1046, "nll": 0.5870477618581573, "acc": 0.777246653919694, "ece": 0.04794282897297868, "brier": 0.2910675481135292, "mean_conf": 0.8251894828926727, "confident_error_rate": 0.02198852772466539, "coverage_at_0_9": 0.5717017208413002, "accuracy_at_0_9": 0.9615384615384616, "coverage_at_5pct_error": 0.6338432122370937, "coverage_at_1pct_error": 0.29923518164435947, "aurc": 0.057865847250986556, "error_rate_at_0_9": 0.038461538461538464, "confidence_bias": 0.047942828972978635, "top_bins": { "0.9": { "n": 598, "errors": 23, "error_rate": 0.038461538461538464 }, "0.95": { "n": 491, "errors": 9, "error_rate": 0.018329938900203666 }, "0.99": { "n": 174, "errors": 3, "error_rate": 0.017241379310344827 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9751434034416826, "confidence_cutoff": 0.9408829808235168 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8793309438470729, "confidence_cutoff": 0.6510422047867755 } }, "score_mae": 0.18602619774583048, "ranked_probability_score": 0.050882127676710784 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.5870477618581573, "acc": 0.777246653919694, "ece": 0.04794282897297868, "brier": 0.2910675481135292, "mean_conf": 0.8251894828926727, "confident_error_rate": 0.02198852772466539, "coverage_at_0_9": 0.5717017208413002, "accuracy_at_0_9": 0.9615384615384616, "coverage_at_5pct_error": 0.6338432122370937, "coverage_at_1pct_error": 0.29923518164435947, "aurc": 0.057865847250986556, "error_rate_at_0_9": 0.038461538461538464, "confidence_bias": 0.047942828972978635, "top_bins": { "0.9": { "n": 598, "errors": 23, "error_rate": 0.038461538461538464 }, "0.95": { "n": 491, "errors": 9, "error_rate": 0.018329938900203666 }, "0.99": { "n": 174, "errors": 3, "error_rate": 0.017241379310344827 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9751434034416826, "confidence_cutoff": 0.9408829808235168 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8793309438470729, "confidence_cutoff": 0.6510422047867755 } }, "score_mae": 0.18602619774583048, "ranked_probability_score": 0.050882127676710784 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 9, "accuracy": 0.45, "full_probability_metrics": { "n": 20, "nll": 1.4638562023564279, "acc": 0.45, "ece": 0.22400769818984456, "brier": 0.685438069283081, "mean_conf": 0.6594526435269211, "confident_error_rate": 0.05, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3734132508358514, "error_rate_at_0_9": 0.25, "confidence_bias": 0.2094526435269211, "top_bins": { "0.9": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6973127263072573 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.483508165065625 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.4638562023564279, "acc": 0.45, "ece": 0.22400769818984456, "brier": 0.685438069283081, "mean_conf": 0.6594526435269211, "confident_error_rate": 0.05, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3734132508358514, "error_rate_at_0_9": 0.25, "confidence_bias": 0.2094526435269211, "top_bins": { "0.9": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6973127263072573 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.483508165065625 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 16, "accuracy": 0.8, "full_probability_metrics": { "n": 20, "nll": 0.36403248618468526, "acc": 0.8, "ece": 0.1295361487173242, "brier": 0.2257044216488208, "mean_conf": 0.8831790381288052, "confident_error_rate": 0.05, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9230769230769231, "coverage_at_5pct_error": 0.45, "coverage_at_1pct_error": 0.45, "aurc": 0.05397950583128602, "error_rate_at_0_9": 0.07692307692307693, "confidence_bias": 0.08317903812880512, "top_bins": { "0.9": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9546051514751633 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.7867595084452922 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.36403248618468526, "acc": 0.8, "ece": 0.1295361487173242, "brier": 0.2257044216488208, "mean_conf": 0.8831790381288052, "confident_error_rate": 0.05, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9230769230769231, "coverage_at_5pct_error": 0.45, "coverage_at_1pct_error": 0.45, "aurc": 0.05397950583128602, "error_rate_at_0_9": 0.07692307692307693, "confidence_bias": 0.08317903812880512, "top_bins": { "0.9": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9546051514751633 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.7867595084452922 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 18, "accuracy": 0.9, "full_probability_metrics": { "n": 20, "nll": 0.18031139864262885, "acc": 0.9, "ece": 0.0817012190955263, "brier": 0.11604628935827062, "mean_conf": 0.9409710011124263, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.0076315789473684215, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04097100111242624, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9736818072730682 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9485171543268541 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.18031139864262885, "acc": 0.9, "ece": 0.0817012190955263, "brier": 0.11604628935827062, "mean_conf": 0.9409710011124263, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.0076315789473684215, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04097100111242624, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9736818072730682 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9485171543268541 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 8, "accuracy": 0.4, "full_probability_metrics": { "n": 20, "nll": 0.9361317272289519, "acc": 0.4, "ece": 0.3467800262809543, "brier": 0.6890225027879284, "mean_conf": 0.7467800262809542, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.46809751974032154, "error_rate_at_0_9": 0.5, "confidence_bias": 0.34678002628095417, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7481542379764177 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.6360844184135074 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.9361317272289519, "acc": 0.4, "ece": 0.3467800262809543, "brier": 0.6890225027879284, "mean_conf": 0.7467800262809542, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.46809751974032154, "error_rate_at_0_9": 0.5, "confidence_bias": 0.34678002628095417, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7481542379764177 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.6360844184135074 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 31, "accuracy": 0.96875, "full_probability_metrics": { "n": 32, "nll": 0.12979322912048136, "acc": 0.96875, "ece": 0.05865632527071586, "brier": 0.06221598573997203, "mean_conf": 0.9181713626206109, "confident_error_rate": 0.0, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004070277739154617, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05057863737938906, "top_bins": { "0.9": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9449616248177756 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.8779250187145542 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.12979322912048136, "acc": 0.96875, "ece": 0.05865632527071586, "brier": 0.06221598573997203, "mean_conf": 0.9181713626206109, "confident_error_rate": 0.0, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004070277739154617, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05057863737938906, "top_bins": { "0.9": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9449616248177756 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.8779250187145542 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 29, "accuracy": 0.90625, "full_probability_metrics": { "n": 32, "nll": 0.24455502641087398, "acc": 0.90625, "ece": 0.079792014916701, "brier": 0.13577654408819648, "mean_conf": 0.9357865299742769, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9583333333333334, "coverage_at_5pct_error": 0.9375, "coverage_at_1pct_error": 0.375, "aurc": 0.033363772176546395, "error_rate_at_0_9": 0.041666666666666664, "confidence_bias": 0.029536529974276915, "top_bins": { "0.9": { "n": 24, "errors": 1, "error_rate": 0.041666666666666664 }, "0.95": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9375, "confidence_cutoff": 0.9711615479408262 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.8868939148352457 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.24455502641087398, "acc": 0.90625, "ece": 0.079792014916701, "brier": 0.13577654408819648, "mean_conf": 0.9357865299742769, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9583333333333334, "coverage_at_5pct_error": 0.9375, "coverage_at_1pct_error": 0.375, "aurc": 0.033363772176546395, "error_rate_at_0_9": 0.041666666666666664, "confidence_bias": 0.029536529974276915, "top_bins": { "0.9": { "n": 24, "errors": 1, "error_rate": 0.041666666666666664 }, "0.95": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9375, "confidence_cutoff": 0.9711615479408262 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.8868939148352457 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 31, "accuracy": 0.96875, "full_probability_metrics": { "n": 32, "nll": 0.1799321903230246, "acc": 0.96875, "ece": 0.10182879137977313, "brier": 0.09774414276889377, "mean_conf": 0.8675840743934221, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004072377873563219, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10116592560657789, "top_bins": { "0.9": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.9276751556299673 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.7040075273516038 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.1799321903230246, "acc": 0.96875, "ece": 0.10182879137977313, "brier": 0.09774414276889377, "mean_conf": 0.8675840743934221, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004072377873563219, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10116592560657789, "top_bins": { "0.9": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.9276751556299673 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.7040075273516038 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 40, "accuracy": 1.0, "full_probability_metrics": { "n": 40, "nll": 0.012461149245409892, "acc": 1.0, "ece": 0.012352988030602807, "brier": 0.0004269628768902182, "mean_conf": 0.9876470119693972, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.012352988030602807, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9926770167945137 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9794385287249308 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.012461149245409892, "acc": 1.0, "ece": 0.012352988030602807, "brier": 0.0004269628768902182, "mean_conf": 0.9876470119693972, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.012352988030602807, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9926770167945137 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9794385287249308 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 36, "accuracy": 0.9, "full_probability_metrics": { "n": 40, "nll": 0.26407278866821826, "acc": 0.9, "ece": 0.05245202539688848, "brier": 0.15778553734614284, "mean_conf": 0.8829211192449822, "confident_error_rate": 0.0, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.725, "coverage_at_1pct_error": 0.65, "aurc": 0.02308538024077973, "error_rate_at_0_9": 0.0, "confidence_bias": -0.017078880755017845, "top_bins": { "0.9": { "n": 23, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9200535815570202 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.8251958593609416 } }, "score_mae": 0.17291673608415475, "ranked_probability_score": 0.04024927146404126 }, "covered_only_probability_metrics": { "n": 40, "nll": 0.26407278866821826, "acc": 0.9, "ece": 0.05245202539688848, "brier": 0.15778553734614284, "mean_conf": 0.8829211192449822, "confident_error_rate": 0.0, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.725, "coverage_at_1pct_error": 0.65, "aurc": 0.02308538024077973, "error_rate_at_0_9": 0.0, "confidence_bias": -0.017078880755017845, "top_bins": { "0.9": { "n": 23, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9200535815570202 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.8251958593609416 } }, "score_mae": 0.17291673608415475, "ranked_probability_score": 0.04024927146404126 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 48, "accuracy": 0.6, "full_probability_metrics": { "n": 80, "nll": 1.2865095689254165, "acc": 0.6, "ece": 0.19167859824830136, "brier": 0.5660599536581878, "mean_conf": 0.7759389293425664, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.84375, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.24689075981896816, "error_rate_at_0_9": 0.15625, "confidence_bias": 0.17593892934256639, "top_bins": { "0.9": { "n": 32, "errors": 5, "error_rate": 0.15625 }, "0.95": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.99": { "n": 5, "errors": 2, "error_rate": 0.4 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8358595503583666 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.5660683528503451 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.2865095689254165, "acc": 0.6, "ece": 0.19167859824830136, "brier": 0.5660599536581878, "mean_conf": 0.7759389293425664, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.84375, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.24689075981896816, "error_rate_at_0_9": 0.15625, "confidence_bias": 0.17593892934256639, "top_bins": { "0.9": { "n": 32, "errors": 5, "error_rate": 0.15625 }, "0.95": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.99": { "n": 5, "errors": 2, "error_rate": 0.4 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8358595503583666 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.5660683528503451 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 60, "accuracy": 0.75, "full_probability_metrics": { "n": 80, "nll": 0.64319962297488, "acc": 0.75, "ece": 0.07583444934027661, "brier": 0.336846797840859, "mean_conf": 0.815404454455512, "confident_error_rate": 0.025, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.9512195121951219, "coverage_at_5pct_error": 0.575, "coverage_at_1pct_error": 0.425, "aurc": 0.06990146624747551, "error_rate_at_0_9": 0.04878048780487805, "confidence_bias": 0.06540445445551202, "top_bins": { "0.9": { "n": 41, "errors": 2, "error_rate": 0.04878048780487805 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9107307477881781 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.631165847975779 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.64319962297488, "acc": 0.75, "ece": 0.07583444934027661, "brier": 0.336846797840859, "mean_conf": 0.815404454455512, "confident_error_rate": 0.025, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.9512195121951219, "coverage_at_5pct_error": 0.575, "coverage_at_1pct_error": 0.425, "aurc": 0.06990146624747551, "error_rate_at_0_9": 0.04878048780487805, "confidence_bias": 0.06540445445551202, "top_bins": { "0.9": { "n": 41, "errors": 2, "error_rate": 0.04878048780487805 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9107307477881781 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.631165847975779 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 106, "accuracy": 0.53, "full_probability_metrics": { "n": 200, "nll": 1.3020212799561357, "acc": 0.53, "ece": 0.057886156442029546, "brier": 0.5699198585015737, "mean_conf": 0.5796763608829093, "confident_error_rate": 0.005, "coverage_at_0_9": 0.21, "accuracy_at_0_9": 0.9761904761904762, "coverage_at_5pct_error": 0.255, "coverage_at_1pct_error": 0.185, "aurc": 0.22101453807685856, "error_rate_at_0_9": 0.023809523809523808, "confidence_bias": 0.049676360882909276, "top_bins": { "0.9": { "n": 42, "errors": 1, "error_rate": 0.023809523809523808 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.76, "confidence_cutoff": 0.5622010178659016 }, "0.8": { "coverage": 0.8, "accuracy": 0.6125, "confidence_cutoff": 0.28172954890715934 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 1.3020212799561357, "acc": 0.53, "ece": 0.057886156442029546, "brier": 0.5699198585015737, "mean_conf": 0.5796763608829093, "confident_error_rate": 0.005, "coverage_at_0_9": 0.21, "accuracy_at_0_9": 0.9761904761904762, "coverage_at_5pct_error": 0.255, "coverage_at_1pct_error": 0.185, "aurc": 0.22101453807685856, "error_rate_at_0_9": 0.023809523809523808, "confidence_bias": 0.049676360882909276, "top_bins": { "0.9": { "n": 42, "errors": 1, "error_rate": 0.023809523809523808 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.76, "confidence_cutoff": 0.5622010178659016 }, "0.8": { "coverage": 0.8, "accuracy": 0.6125, "confidence_cutoff": 0.28172954890715934 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 73, "accuracy": 0.9125, "full_probability_metrics": { "n": 80, "nll": 0.2779227201992138, "acc": 0.9125, "ece": 0.05493591159180497, "brier": 0.15753943063635653, "mean_conf": 0.9357409581036927, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9384615384615385, "coverage_at_5pct_error": 0.7625, "coverage_at_1pct_error": 0.3, "aurc": 0.0308044058290765, "error_rate_at_0_9": 0.06153846153846154, "confidence_bias": 0.023240958103692755, "top_bins": { "0.9": { "n": 65, "errors": 4, "error_rate": 0.06153846153846154 }, "0.95": { "n": 58, "errors": 2, "error_rate": 0.034482758620689655 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9736310261741858 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9301026006058246 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.2779227201992138, "acc": 0.9125, "ece": 0.05493591159180497, "brier": 0.15753943063635653, "mean_conf": 0.9357409581036927, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9384615384615385, "coverage_at_5pct_error": 0.7625, "coverage_at_1pct_error": 0.3, "aurc": 0.0308044058290765, "error_rate_at_0_9": 0.06153846153846154, "confidence_bias": 0.023240958103692755, "top_bins": { "0.9": { "n": 65, "errors": 4, "error_rate": 0.06153846153846154 }, "0.95": { "n": 58, "errors": 2, "error_rate": 0.034482758620689655 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9736310261741858 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9301026006058246 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 73, "accuracy": 0.9125, "full_probability_metrics": { "n": 80, "nll": 0.21231177025039338, "acc": 0.9125, "ece": 0.026039786769557725, "brier": 0.11561259420716938, "mean_conf": 0.9322130681411511, "confident_error_rate": 0.025, "coverage_at_0_9": 0.825, "accuracy_at_0_9": 0.9696969696969697, "coverage_at_5pct_error": 0.925, "coverage_at_1pct_error": 0.1, "aurc": 0.03569781457299058, "error_rate_at_0_9": 0.030303030303030304, "confidence_bias": 0.019713068141151147, "top_bins": { "0.9": { "n": 66, "errors": 2, "error_rate": 0.030303030303030304 }, "0.95": { "n": 61, "errors": 1, "error_rate": 0.01639344262295082 }, "0.99": { "n": 14, "errors": 1, "error_rate": 0.07142857142857142 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9814442414444764 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9395185848998813 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.21231177025039338, "acc": 0.9125, "ece": 0.026039786769557725, "brier": 0.11561259420716938, "mean_conf": 0.9322130681411511, "confident_error_rate": 0.025, "coverage_at_0_9": 0.825, "accuracy_at_0_9": 0.9696969696969697, "coverage_at_5pct_error": 0.925, "coverage_at_1pct_error": 0.1, "aurc": 0.03569781457299058, "error_rate_at_0_9": 0.030303030303030304, "confidence_bias": 0.019713068141151147, "top_bins": { "0.9": { "n": 66, "errors": 2, "error_rate": 0.030303030303030304 }, "0.95": { "n": 61, "errors": 1, "error_rate": 0.01639344262295082 }, "0.99": { "n": 14, "errors": 1, "error_rate": 0.07142857142857142 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9814442414444764 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9395185848998813 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 80, "accuracy": 1.0, "full_probability_metrics": { "n": 80, "nll": 0.007801897547958209, "acc": 1.0, "ece": 0.007676312134330843, "brier": 0.0004369807984377848, "mean_conf": 0.9923236878656692, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007676312134330843, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 65, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9965300683221596 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9921908745945206 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.007801897547958209, "acc": 1.0, "ece": 0.007676312134330843, "brier": 0.0004369807984377848, "mean_conf": 0.9923236878656692, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007676312134330843, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 65, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9965300683221596 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9921908745945206 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 60, "accuracy": 0.75, "full_probability_metrics": { "n": 80, "nll": 0.4720489807336611, "acc": 0.75, "ece": 0.0761936963725907, "brier": 0.3177754348396661, "mean_conf": 0.8169291651727064, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 0.9090909090909091, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.3375, "aurc": 0.09282282377572246, "error_rate_at_0_9": 0.09090909090909091, "confidence_bias": 0.06692916517270642, "top_bins": { "0.9": { "n": 33, "errors": 3, "error_rate": 0.09090909090909091 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.8560822733652931 }, "0.8": { "coverage": 0.8, "accuracy": 0.828125, "confidence_cutoff": 0.6797098123940769 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4720489807336611, "acc": 0.75, "ece": 0.0761936963725907, "brier": 0.3177754348396661, "mean_conf": 0.8169291651727064, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 0.9090909090909091, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.3375, "aurc": 0.09282282377572246, "error_rate_at_0_9": 0.09090909090909091, "confidence_bias": 0.06692916517270642, "top_bins": { "0.9": { "n": 33, "errors": 3, "error_rate": 0.09090909090909091 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.8560822733652931 }, "0.8": { "coverage": 0.8, "accuracy": 0.828125, "confidence_cutoff": 0.6797098123940769 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.01010806429581184, "acc": 1.0, "ece": 0.010050520182894429, "brier": 0.00022828427851469854, "mean_conf": 0.9899494798171056, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.010050520182894429, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9920052497980926 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9867894292404178 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.01010806429581184, "acc": 1.0, "ece": 0.010050520182894429, "brier": 0.00022828427851469854, "mean_conf": 0.9899494798171056, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.010050520182894429, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9920052497980926 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9867894292404178 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.013336190464884707, "acc": 1.0, "ece": 0.013199482511126015, "brier": 0.0005383885458560314, "mean_conf": 0.986800517488874, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013199482511126015, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9957628189842604 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9774085096482822 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.013336190464884707, "acc": 1.0, "ece": 0.013199482511126015, "brier": 0.0005383885458560314, "mean_conf": 0.986800517488874, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013199482511126015, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9957628189842604 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9774085096482822 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.22694873211843464, "acc": 0.8, "ece": 0.16555386795989538, "brier": 0.13550345474895487, "mean_conf": 0.8535695538437231, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05356955384372308, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9367303412067545 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8305143644338201 } }, "score_mae": 0.16955440723819443, "ranked_probability_score": 0.034895014874681045 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.22694873211843464, "acc": 0.8, "ece": 0.16555386795989538, "brier": 0.13550345474895487, "mean_conf": 0.8535695538437231, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05356955384372308, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9367303412067545 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8305143644338201 } }, "score_mae": 0.16955440723819443, "ranked_probability_score": 0.034895014874681045 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.061294700563837144, "acc": 1.0, "ece": 0.04980564296050656, "brier": 0.03092495951104666, "mean_conf": 0.9501943570394934, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0498056429605066, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9916381797053619 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9777806624356902 } }, "score_mae": 0.04333117262198813, "ranked_probability_score": 0.007196645402378658 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.061294700563837144, "acc": 1.0, "ece": 0.04980564296050656, "brier": 0.03092495951104666, "mean_conf": 0.9501943570394934, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0498056429605066, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9916381797053619 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9777806624356902 } }, "score_mae": 0.04333117262198813, "ranked_probability_score": 0.007196645402378658 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.12423624782442941, "acc": 1.0, "ece": 0.10880820844844666, "brier": 0.04779494768609947, "mean_conf": 0.8911917915515533, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10880820844844674, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9558210808739257 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8866819137686461 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.12423624782442941, "acc": 1.0, "ece": 0.10880820844844666, "brier": 0.04779494768609947, "mean_conf": 0.8911917915515533, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10880820844844674, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9558210808739257 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8866819137686461 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 0.8828182838634889, "acc": 0.6, "ece": 0.3920235500266451, "brier": 0.6280874258674091, "mean_conf": 0.8410658551406985, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.288968253968254, "error_rate_at_0_9": 0.0, "confidence_bias": 0.24106585514069856, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8732116352500265 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8179686895920922 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.8828182838634889, "acc": 0.6, "ece": 0.3920235500266451, "brier": 0.6280874258674091, "mean_conf": 0.8410658551406985, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.288968253968254, "error_rate_at_0_9": 0.0, "confidence_bias": 0.24106585514069856, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8732116352500265 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8179686895920922 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.38549807963539046, "acc": 0.9, "ece": 0.20288794178646427, "brier": 0.23489325574634798, "mean_conf": 0.8015680467880623, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.06456349206349207, "error_rate_at_0_9": 0.0, "confidence_bias": -0.09843195321193776, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8456577690156466 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.6947201852727033 } }, "score_mae": 0.2564750859760038, "ranked_probability_score": 0.05882371330201126 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.38549807963539046, "acc": 0.9, "ece": 0.20288794178646427, "brier": 0.23489325574634798, "mean_conf": 0.8015680467880623, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.06456349206349207, "error_rate_at_0_9": 0.0, "confidence_bias": -0.09843195321193776, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8456577690156466 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.6947201852727033 } }, "score_mae": 0.2564750859760038, "ranked_probability_score": 0.05882371330201126 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.7292834955292916, "acc": 0.7, "ece": 0.21088673885292936, "brier": 0.4247531105085889, "mean_conf": 0.9108867388529293, "confident_error_rate": 0.1, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.8571428571428571, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.12817460317460316, "error_rate_at_0_9": 0.14285714285714285, "confidence_bias": 0.2108867388529293, "top_bins": { "0.9": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.95": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9757163386049921 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8745092806607159 } }, "score_mae": 0.27669455495743667, "ranked_probability_score": 0.1053871341351588 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7292834955292916, "acc": 0.7, "ece": 0.21088673885292936, "brier": 0.4247531105085889, "mean_conf": 0.9108867388529293, "confident_error_rate": 0.1, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.8571428571428571, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.12817460317460316, "error_rate_at_0_9": 0.14285714285714285, "confidence_bias": 0.2108867388529293, "top_bins": { "0.9": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.95": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9757163386049921 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8745092806607159 } }, "score_mae": 0.27669455495743667, "ranked_probability_score": 0.1053871341351588 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.005872511904691692, "acc": 1.0, "ece": 0.005852885976918243, "brier": 6.553204955281063e-05, "mean_conf": 0.9941471140230818, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.005852885976918243, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9956017537902697 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.992077102727452 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.005872511904691692, "acc": 1.0, "ece": 0.005852885976918243, "brier": 6.553204955281063e-05, "mean_conf": 0.9941471140230818, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.005852885976918243, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9956017537902697 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.992077102727452 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.009442501422597043, "acc": 1.0, "ece": 0.009390689401267194, "brier": 0.00017066799723336928, "mean_conf": 0.9906093105987328, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009390689401267194, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9912854619430778 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9857135990468268 } }, "score_mae": 0.010231745432055298, "ranked_probability_score": 4.863796909247521e-05 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.009442501422597043, "acc": 1.0, "ece": 0.009390689401267194, "brier": 0.00017066799723336928, "mean_conf": 0.9906093105987328, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009390689401267194, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9912854619430778 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9857135990468268 } }, "score_mae": 0.010231745432055298, "ranked_probability_score": 4.863796909247521e-05 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.9490781913515873, "acc": 0.5, "ece": 0.36923739486967955, "brier": 0.5785088804997336, "mean_conf": 0.710935585802415, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.356468253968254, "error_rate_at_0_9": 0.5, "confidence_bias": 0.210935585802415, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6925948236413493 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.586704671382904 } }, "score_mae": 0.41230806689600685, "ranked_probability_score": 0.14147304522762058 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.9490781913515873, "acc": 0.5, "ece": 0.36923739486967955, "brier": 0.5785088804997336, "mean_conf": 0.710935585802415, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.356468253968254, "error_rate_at_0_9": 0.5, "confidence_bias": 0.210935585802415, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6925948236413493 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.586704671382904 } }, "score_mae": 0.41230806689600685, "ranked_probability_score": 0.14147304522762058 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.8240277777777778, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.7777777777777778, "flip_rate": 0.08333333333333333, "covered_only_flip_rate": 0.08333333333333333, "half_l1": 0.08228615038271361 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.6359851141873526, "share_at_0_9": 0.20909090909090908, "normalized_entropy": 0.7358129257673091 }, "covered_only_confidence_metrics": { "mean_max_p": 0.6359851141873526, "share_at_0_9": 0.20909090909090908, "normalized_entropy": 0.7358129257673091 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 95, "accuracy": 0.8636363636363636, "full_probability_metrics": { "n": 110, "nll": 0.30890154536131303, "acc": 0.8636363636363636, "ece": 0.055763619767470476, "brier": 0.18922444613084882, "mean_conf": 0.9019016682678792, "confident_error_rate": 0.01818181818181818, "coverage_at_0_9": 0.6818181818181818, "accuracy_at_0_9": 0.9733333333333334, "coverage_at_5pct_error": 0.7545454545454545, "coverage_at_1pct_error": 0.5363636363636364, "aurc": 0.029053269688156625, "error_rate_at_0_9": 0.02666666666666667, "confidence_bias": 0.03826530463151556, "top_bins": { "0.9": { "n": 75, "errors": 2, "error_rate": 0.02666666666666667 }, "0.95": { "n": 66, "errors": 1, "error_rate": 0.015151515151515152 }, "0.99": { "n": 37, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9777806624356902 }, "0.8": { "coverage": 0.8, "accuracy": 0.9318181818181818, "confidence_cutoff": 0.8305143644338201 } }, "score_mae": 0.19476583885361418, "ranked_probability_score": 0.05797069848515713 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.30890154536131303, "acc": 0.8636363636363636, "ece": 0.055763619767470476, "brier": 0.18922444613084882, "mean_conf": 0.9019016682678792, "confident_error_rate": 0.01818181818181818, "coverage_at_0_9": 0.6818181818181818, "accuracy_at_0_9": 0.9733333333333334, "coverage_at_5pct_error": 0.7545454545454545, "coverage_at_1pct_error": 0.5363636363636364, "aurc": 0.029053269688156625, "error_rate_at_0_9": 0.02666666666666667, "confidence_bias": 0.03826530463151556, "top_bins": { "0.9": { "n": 75, "errors": 2, "error_rate": 0.02666666666666667 }, "0.95": { "n": 66, "errors": 1, "error_rate": 0.015151515151515152 }, "0.99": { "n": 37, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9777806624356902 }, "0.8": { "coverage": 0.8, "accuracy": 0.9318181818181818, "confidence_cutoff": 0.8305143644338201 } }, "score_mae": 0.19476583885361418, "ranked_probability_score": 0.05797069848515713 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.2659165540805263 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.4212918162126965, "paired_flip": { "pairs": 64, "flip_rate": 0.953125, "both_correct_rate": 0.890625, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.9583333333333334 }, "unknowable": { "n": 110, "mean_max_p": 0.6359851141873526, "share_at_0_9": 0.20909090909090908, "control_mean_max_p": 0.9019016682678792, "control_share_at_0_9": 0.6818181818181818, "control_acc": 0.8636363636363636, "paired_confidence_drop": 0.2659165540805263, "share_less_confident_than_control": 0.9090909090909091 }, "clean": { "n": 1046, "nll": 0.5870477618581573, "acc": 0.777246653919694, "ece": 0.04794282897297868, "brier": 0.2910675481135292, "mean_conf": 0.8251894828926727, "confident_error_rate": 0.02198852772466539, "coverage_at_0_9": 0.5717017208413002, "accuracy_at_0_9": 0.9615384615384616, "coverage_at_5pct_error": 0.6338432122370937, "coverage_at_1pct_error": 0.29923518164435947, "aurc": 0.057865847250986556, "error_rate_at_0_9": 0.038461538461538464, "confidence_bias": 0.047942828972978635, "top_bins": { "0.9": { "n": 598, "errors": 23, "error_rate": 0.038461538461538464 }, "0.95": { "n": 491, "errors": 9, "error_rate": 0.018329938900203666 }, "0.99": { "n": 174, "errors": 3, "error_rate": 0.017241379310344827 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9751434034416826, "confidence_cutoff": 0.9408829808235168 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8793309438470729, "confidence_cutoff": 0.6510422047867755 } }, "score_mae": 0.18602619774583048, "ranked_probability_score": 0.050882127676710784 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.4638562023564279, "acc": 0.45, "ece": 0.22400769818984456, "brier": 0.685438069283081, "mean_conf": 0.6594526435269211, "confident_error_rate": 0.05, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3734132508358514, "error_rate_at_0_9": 0.25, "confidence_bias": 0.2094526435269211, "top_bins": { "0.9": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6973127263072573 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.483508165065625 } } }, "buried_paws": { "n": 20, "nll": 0.36403248618468526, "acc": 0.8, "ece": 0.1295361487173242, "brier": 0.2257044216488208, "mean_conf": 0.8831790381288052, "confident_error_rate": 0.05, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9230769230769231, "coverage_at_5pct_error": 0.45, "coverage_at_1pct_error": 0.45, "aurc": 0.05397950583128602, "error_rate_at_0_9": 0.07692307692307693, "confidence_bias": 0.08317903812880512, "top_bins": { "0.9": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9546051514751633 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.7867595084452922 } } }, "buried_qnli": { "n": 20, "nll": 0.18031139864262885, "acc": 0.9, "ece": 0.0817012190955263, "brier": 0.11604628935827062, "mean_conf": 0.9409710011124263, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.0076315789473684215, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04097100111242624, "top_bins": { "0.9": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9736818072730682 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9485171543268541 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.9361317272289519, "acc": 0.4, "ece": 0.3467800262809543, "brier": 0.6890225027879284, "mean_conf": 0.7467800262809542, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.46809751974032154, "error_rate_at_0_9": 0.5, "confidence_bias": 0.34678002628095417, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7481542379764177 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.6360844184135074 } } }, "composition_final_combination": { "n": 32, "nll": 0.12979322912048136, "acc": 0.96875, "ece": 0.05865632527071586, "brier": 0.06221598573997203, "mean_conf": 0.9181713626206109, "confident_error_rate": 0.0, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004070277739154617, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05057863737938906, "top_bins": { "0.9": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9449616248177756 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.8779250187145542 } } }, "composition_final_exception": { "n": 32, "nll": 0.24455502641087398, "acc": 0.90625, "ece": 0.079792014916701, "brier": 0.13577654408819648, "mean_conf": 0.9357865299742769, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9583333333333334, "coverage_at_5pct_error": 0.9375, "coverage_at_1pct_error": 0.375, "aurc": 0.033363772176546395, "error_rate_at_0_9": 0.041666666666666664, "confidence_bias": 0.029536529974276915, "top_bins": { "0.9": { "n": 24, "errors": 1, "error_rate": 0.041666666666666664 }, "0.95": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9375, "confidence_cutoff": 0.9711615479408262 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.8868939148352457 } } }, "composition_final_negation": { "n": 32, "nll": 0.1799321903230246, "acc": 0.96875, "ece": 0.10182879137977313, "brier": 0.09774414276889377, "mean_conf": 0.8675840743934221, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004072377873563219, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10116592560657789, "top_bins": { "0.9": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.9276751556299673 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.7040075273516038 } } }, "contrastive_authorization": { "n": 40, "nll": 0.012461149245409892, "acc": 1.0, "ece": 0.012352988030602807, "brier": 0.0004269628768902182, "mean_conf": 0.9876470119693972, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.012352988030602807, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9926770167945137 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9794385287249308 } } }, "contrastive_deadline": { "n": 40, "nll": 0.26407278866821826, "acc": 0.9, "ece": 0.05245202539688848, "brier": 0.15778553734614284, "mean_conf": 0.8829211192449822, "confident_error_rate": 0.0, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.725, "coverage_at_1pct_error": 0.65, "aurc": 0.02308538024077973, "error_rate_at_0_9": 0.0, "confidence_bias": -0.017078880755017845, "top_bins": { "0.9": { "n": 23, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9200535815570202 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.8251958593609416 } }, "score_mae": 0.17291673608415475, "ranked_probability_score": 0.04024927146404126 }, "emotion": { "n": 80, "nll": 1.2865095689254165, "acc": 0.6, "ece": 0.19167859824830136, "brier": 0.5660599536581878, "mean_conf": 0.7759389293425664, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.84375, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.24689075981896816, "error_rate_at_0_9": 0.15625, "confidence_bias": 0.17593892934256639, "top_bins": { "0.9": { "n": 32, "errors": 5, "error_rate": 0.15625 }, "0.95": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.99": { "n": 5, "errors": 2, "error_rate": 0.4 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8358595503583666 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.5660683528503451 } } }, "mmlu": { "n": 80, "nll": 0.64319962297488, "acc": 0.75, "ece": 0.07583444934027661, "brier": 0.336846797840859, "mean_conf": 0.815404454455512, "confident_error_rate": 0.025, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.9512195121951219, "coverage_at_5pct_error": 0.575, "coverage_at_1pct_error": 0.425, "aurc": 0.06990146624747551, "error_rate_at_0_9": 0.04878048780487805, "confidence_bias": 0.06540445445551202, "top_bins": { "0.9": { "n": 41, "errors": 2, "error_rate": 0.04878048780487805 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9107307477881781 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.631165847975779 } } }, "mmlu_pro": { "n": 200, "nll": 1.3020212799561357, "acc": 0.53, "ece": 0.057886156442029546, "brier": 0.5699198585015737, "mean_conf": 0.5796763608829093, "confident_error_rate": 0.005, "coverage_at_0_9": 0.21, "accuracy_at_0_9": 0.9761904761904762, "coverage_at_5pct_error": 0.255, "coverage_at_1pct_error": 0.185, "aurc": 0.22101453807685856, "error_rate_at_0_9": 0.023809523809523808, "confidence_bias": 0.049676360882909276, "top_bins": { "0.9": { "n": 42, "errors": 1, "error_rate": 0.023809523809523808 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.76, "confidence_cutoff": 0.5622010178659016 }, "0.8": { "coverage": 0.8, "accuracy": 0.6125, "confidence_cutoff": 0.28172954890715934 } } }, "paws": { "n": 80, "nll": 0.2779227201992138, "acc": 0.9125, "ece": 0.05493591159180497, "brier": 0.15753943063635653, "mean_conf": 0.9357409581036927, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9384615384615385, "coverage_at_5pct_error": 0.7625, "coverage_at_1pct_error": 0.3, "aurc": 0.0308044058290765, "error_rate_at_0_9": 0.06153846153846154, "confidence_bias": 0.023240958103692755, "top_bins": { "0.9": { "n": 65, "errors": 4, "error_rate": 0.06153846153846154 }, "0.95": { "n": 58, "errors": 2, "error_rate": 0.034482758620689655 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9736310261741858 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9301026006058246 } } }, "qnli": { "n": 80, "nll": 0.21231177025039338, "acc": 0.9125, "ece": 0.026039786769557725, "brier": 0.11561259420716938, "mean_conf": 0.9322130681411511, "confident_error_rate": 0.025, "coverage_at_0_9": 0.825, "accuracy_at_0_9": 0.9696969696969697, "coverage_at_5pct_error": 0.925, "coverage_at_1pct_error": 0.1, "aurc": 0.03569781457299058, "error_rate_at_0_9": 0.030303030303030304, "confidence_bias": 0.019713068141151147, "top_bins": { "0.9": { "n": 66, "errors": 2, "error_rate": 0.030303030303030304 }, "0.95": { "n": 61, "errors": 1, "error_rate": 0.01639344262295082 }, "0.99": { "n": 14, "errors": 1, "error_rate": 0.07142857142857142 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9814442414444764 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9395185848998813 } } }, "sciq": { "n": 80, "nll": 0.007801897547958209, "acc": 1.0, "ece": 0.007676312134330843, "brier": 0.0004369807984377848, "mean_conf": 0.9923236878656692, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007676312134330843, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 65, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9965300683221596 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9921908745945206 } } }, "tweet_offensive": { "n": 80, "nll": 0.4720489807336611, "acc": 0.75, "ece": 0.0761936963725907, "brier": 0.3177754348396661, "mean_conf": 0.8169291651727064, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 0.9090909090909091, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.3375, "aurc": 0.09282282377572246, "error_rate_at_0_9": 0.09090909090909091, "confidence_bias": 0.06692916517270642, "top_bins": { "0.9": { "n": 33, "errors": 3, "error_rate": 0.09090909090909091 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.8560822733652931 }, "0.8": { "coverage": 0.8, "accuracy": 0.828125, "confidence_cutoff": 0.6797098123940769 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.8159140816405721, "acc": 0.5, "ece": 0.21421574323774287, "brier": 0.6032897524261119, "mean_conf": 0.7142157432377428, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.21421574323774284, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.7100501272097268 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6407661437988281 } } }, "unknowable_authorization": { "n": 10, "nll": 1.0676248278306069, "acc": 0.5, "ece": 0.35543382596706624, "brier": 0.7452622091449328, "mean_conf": 0.8554338259670662, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3481349206349207, "error_rate_at_0_9": 0.4, "confidence_bias": 0.3554338259670662, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9051045381673695 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7762378934586363 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.01010806429581184, "acc": 1.0, "ece": 0.010050520182894429, "brier": 0.00022828427851469854, "mean_conf": 0.9899494798171056, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.010050520182894429, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9920052497980926 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9867894292404178 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.013336190464884707, "acc": 1.0, "ece": 0.013199482511126015, "brier": 0.0005383885458560314, "mean_conf": 0.986800517488874, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013199482511126015, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9957628189842604 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9774085096482822 } } }, "unknowable_control_deadline": { "n": 10, "nll": 0.22694873211843464, "acc": 0.8, "ece": 0.16555386795989538, "brier": 0.13550345474895487, "mean_conf": 0.8535695538437231, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05356955384372308, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9367303412067545 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8305143644338201 } }, "score_mae": 0.16955440723819443, "ranked_probability_score": 0.034895014874681045 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.061294700563837144, "acc": 1.0, "ece": 0.04980564296050656, "brier": 0.03092495951104666, "mean_conf": 0.9501943570394934, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0498056429605066, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9916381797053619 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9777806624356902 } }, "score_mae": 0.04333117262198813, "ranked_probability_score": 0.007196645402378658 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.12423624782442941, "acc": 1.0, "ece": 0.10880820844844666, "brier": 0.04779494768609947, "mean_conf": 0.8911917915515533, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10880820844844674, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9558210808739257 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8866819137686461 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.8828182838634889, "acc": 0.6, "ece": 0.3920235500266451, "brier": 0.6280874258674091, "mean_conf": 0.8410658551406985, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.288968253968254, "error_rate_at_0_9": 0.0, "confidence_bias": 0.24106585514069856, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8732116352500265 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8179686895920922 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.38549807963539046, "acc": 0.9, "ece": 0.20288794178646427, "brier": 0.23489325574634798, "mean_conf": 0.8015680467880623, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.06456349206349207, "error_rate_at_0_9": 0.0, "confidence_bias": -0.09843195321193776, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8456577690156466 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.6947201852727033 } }, "score_mae": 0.2564750859760038, "ranked_probability_score": 0.05882371330201126 }, "unknowable_control_sla_response": { "n": 10, "nll": 0.7292834955292916, "acc": 0.7, "ece": 0.21088673885292936, "brier": 0.4247531105085889, "mean_conf": 0.9108867388529293, "confident_error_rate": 0.1, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.8571428571428571, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.12817460317460316, "error_rate_at_0_9": 0.14285714285714285, "confidence_bias": 0.2108867388529293, "top_bins": { "0.9": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.95": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9757163386049921 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8745092806607159 } }, "score_mae": 0.27669455495743667, "ranked_probability_score": 0.1053871341351588 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.005872511904691692, "acc": 1.0, "ece": 0.005852885976918243, "brier": 6.553204955281063e-05, "mean_conf": 0.9941471140230818, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.005852885976918243, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9956017537902697 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.992077102727452 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.009442501422597043, "acc": 1.0, "ece": 0.009390689401267194, "brier": 0.00017066799723336928, "mean_conf": 0.9906093105987328, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009390689401267194, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9912854619430778 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9857135990468268 } }, "score_mae": 0.010231745432055298, "ranked_probability_score": 4.863796909247521e-05 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.9490781913515873, "acc": 0.5, "ece": 0.36923739486967955, "brier": 0.5785088804997336, "mean_conf": 0.710935585802415, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.356468253968254, "error_rate_at_0_9": 0.5, "confidence_bias": 0.210935585802415, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6925948236413493 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.586704671382904 } }, "score_mae": 0.41230806689600685, "ranked_probability_score": 0.14147304522762058 }, "unknowable_deadline": { "n": 10, "nll": 1.1936892937872634, "acc": 0.3, "ece": 0.17490025011213214, "brier": 0.7253276456261111, "mean_conf": 0.4749002501121321, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7589285714285715, "error_rate_at_0_9": null, "confidence_bias": 0.17490025011213212, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.4989907294211009 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4357915260916039 } }, "score_mae": 0.7352555113184743, "ranked_probability_score": 0.2420866592785548 }, "unknowable_late_fee": { "n": 10, "nll": 1.2923453860166598, "acc": 0.3, "ece": 0.23038840334765942, "brier": 0.7937115661256419, "mean_conf": 0.5303884033476594, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7733333333333332, "error_rate_at_0_9": null, "confidence_bias": 0.23038840334765937, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.5093778740147706 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.44214211890736754 } }, "score_mae": 0.7422305550581385, "ranked_probability_score": 0.25270907837522644 }, "unknowable_quantity_limit": { "n": 10, "nll": 2.364100794867005, "acc": 0.5, "ece": 0.4827107892011071, "brier": 0.9737439215308642, "mean_conf": 0.9827107892011071, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4827107892011071, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.9826534113523934 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.98013697658372 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7914668987099229, "acc": 0.7, "ece": 0.286885017017804, "brier": 0.5690036879176795, "mean_conf": 0.585196890317801, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5203571428571429, "error_rate_at_0_9": null, "confidence_bias": -0.11480310968219898, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5657328367233276 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5336925983428955 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.2682712288665416, "acc": 0.2, "ece": 0.29121892684207396, "brier": 0.781447600806538, "mean_conf": 0.4462795720835988, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9309920634920636, "error_rate_at_0_9": null, "confidence_bias": 0.2462795720835988, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.4459783434867859 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.4014878571033478 } }, "score_mae": 0.8435188873044972, "ranked_probability_score": 0.27948214430957724 }, "unknowable_sla_response": { "n": 10, "nll": 1.200518862016904, "acc": 0.2, "ece": 0.25713645324930595, "brier": 0.7363382258751557, "mean_conf": 0.45713645324930585, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7266666666666668, "error_rate_at_0_9": null, "confidence_bias": 0.25713645324930584, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.4304346442222595 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.4151159699226627 } }, "score_mae": 0.5909151001699179, "ranked_probability_score": 0.21421444884056046 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.7380288693535819, "acc": 0.5, "ece": 0.07006097788326784, "brier": 0.520722008501961, "mean_conf": 0.5699825910724938, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.06998259107249383, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.5894361470891768 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.502282899723621 } } }, "unknowable_volume_discount": { "n": 10, "nll": 2.475799282190841, "acc": 0.4, "ece": 0.5247692963711266, "brier": 1.1101745871995399, "mean_conf": 0.9247692963711266, "confident_error_rate": 0.5, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.375, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5783333333333334, "error_rate_at_0_9": 0.625, "confidence_bias": 0.5247692963711266, "top_bins": { "0.9": { "n": 8, "errors": 5, "error_rate": 0.625 }, "0.95": { "n": 8, "errors": 5, "error_rate": 0.625 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.9584822208396057 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.9566530548759113 } }, "score_mae": 0.9890203760461705, "ranked_probability_score": 0.456901425188147 }, "unknowable_warranty_claim": { "n": 10, "nll": 0.9999682429218097, "acc": 0.4, "ece": 0.16837300755606652, "brier": 0.6134949708297965, "mean_conf": 0.4548224411008467, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5029761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.054822441100846664, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.42227191991424323 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4032192648416538 } }, "score_mae": 0.5399159280722049, "ranked_probability_score": 0.17294328544089552 } }, "variants": { "clean": { "n": 1156, "nll": 0.6540910351086936, "acc": 0.7422145328719724, "ece": 0.06497107410583432, "brier": 0.33406731581885374, "mean_conf": 0.8071856069778066, "confident_error_rate": 0.03027681660899654, "coverage_at_0_9": 0.53719723183391, "accuracy_at_0_9": 0.9436392914653784, "coverage_at_5pct_error": 0.5086505190311419, "coverage_at_1pct_error": 0.11245674740484429, "aurc": 0.08035729090618272, "error_rate_at_0_9": 0.05636070853462158, "confidence_bias": 0.06497107410583425, "top_bins": { "0.9": { "n": 621, "errors": 35, "error_rate": 0.05636070853462158 }, "0.95": { "n": 509, "errors": 19, "error_rate": 0.03732809430255403 }, "0.99": { "n": 174, "errors": 3, "error_rate": 0.017241379310344827 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9515570934256056, "confidence_cutoff": 0.9274184747088865 }, "0.8": { "coverage": 0.801038062283737, "accuracy": 0.8390928725701944, "confidence_cutoff": 0.5894361470891768 } }, "score_mae": 0.3938198959642317, "ranked_probability_score": 0.13294739488750432 }, "none_absent": { "n": 36, "nll": 1.2677850992926212, "acc": 0.5, "ece": 0.278053484436123, "brier": 0.7154228122314299, "mean_conf": 0.6718423502744483, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.08333333333333333, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5441880853688735, "error_rate_at_0_9": 1.0, "confidence_bias": 0.17184235027444827, "top_bins": { "0.9": { "n": 3, "errors": 3, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6111111111111112, "confidence_cutoff": 0.6822161181306214 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5172413793103449, "confidence_cutoff": 0.4968951628630185 } } }, "none_present": { "n": 36, "nll": 0.7499526141523112, "acc": 0.7777777777777778, "ece": 0.10875199149059422, "brier": 0.2872316962408804, "mean_conf": 0.7911516003199925, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5277777777777778, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7222222222222222, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.0406940559221325, "error_rate_at_0_9": 0.0, "confidence_bias": 0.013373822542214753, "top_bins": { "0.9": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9345858612432604 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.896551724137931, "confidence_cutoff": 0.5753147944995638 } } }, "permuted": { "n": 36, "nll": 0.7075736943294805, "acc": 0.7777777777777778, "ece": 0.11496218985382063, "brier": 0.2750490327493021, "mean_conf": 0.8292829467749867, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6111111111111112, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7777777777777778, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.037657461069206234, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05150516899720892, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9415673543396044 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.9310344827586207, "confidence_cutoff": 0.6202919952139035 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.08021796939627156, "flip_rate": 0.08333333333333333 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.5870477618581573, "acc": 0.777246653919694, "ece": 0.04794282897297868, "brier": 0.2910675481135292, "mean_conf": 0.8251894828926727, "confident_error_rate": 0.02198852772466539, "coverage_at_0_9": 0.5717017208413002, "accuracy_at_0_9": 0.9615384615384616, "coverage_at_5pct_error": 0.6338432122370937, "coverage_at_1pct_error": 0.29923518164435947, "aurc": 0.057865847250986556, "error_rate_at_0_9": 0.038461538461538464, "confidence_bias": 0.047942828972978635, "top_bins": { "0.9": { "n": 598, "errors": 23, "error_rate": 0.038461538461538464 }, "0.95": { "n": 491, "errors": 9, "error_rate": 0.018329938900203666 }, "0.99": { "n": 174, "errors": 3, "error_rate": 0.017241379310344827 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9751434034416826, "confidence_cutoff": 0.9408829808235168 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8793309438470729, "confidence_cutoff": 0.6510422047867755 } }, "score_mae": 0.18602619774583048, "ranked_probability_score": 0.050882127676710784 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Nox": { "panels": { "old_core": { "point": 0.8300223214285715, "exact_fraction": "7437/8960", "ci95": [ 0.7974693080357143, 0.862016369047619 ] }, "v3_core": { "point": 0.5179166666666667, "exact_fraction": "1243/2400", "ci95": [ 0.4920833333333333, 0.545 ] }, "v4": { "point": 0.790625, "exact_fraction": "253/320", "ci95": [ 0.7453125, 0.8331463068181818 ] }, "v5": { "point": 0.8625, "exact_fraction": "69/80", "ci95": [ 0.8321482316646345, 0.8915620903779563 ] }, "transfer_v9_test": { "point": 0.6959847036328872, "exact_fraction": "364/523", "ci95": [ 0.6660212501997199, 0.7249533151094627 ] }, "original_four_panel_mean": { "point": 0.7502659970238095, "exact_fraction": "403343/537600", "ci95": [ 0.7330469621683191, 0.7670605596068275 ] }, "weighted_mean": { "point": 0.7308523186401712, "exact_fraction": "102744973/140582400", "ci95": [ 0.7157461705500261, 0.7455873812383725 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 109, "requested": 128 }, "boolean_constraints": { "correct": 60, "requested": 64 }, "dbpedia_14": { "correct": 108, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 96, "requested": 96 }, "ordinal_rubric": { "correct": 57, "requested": 64 }, "relational_composition": { "correct": 50, "requested": 96 }, "scoped_evidence": { "correct": 75, "requested": 96 }, "state_tracking": { "correct": 34, "requested": 96 }, "unknown_rejection": { "correct": 64, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 43, "requested": 80 }, "capacitated_assignment": { "correct": 37, "requested": 80 }, "constraint_assignment": { "correct": 33, "requested": 80 }, "massive_en": { "correct": 110, "requested": 120 }, "massive_zh": { "correct": 105, "requested": 120 }, "multiset_reconciliation": { "correct": 23, "requested": 80 }, "ordinal_service_loss": { "correct": 24, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 27, "requested": 80 }, "temporal_exclusion": { "correct": 34, "requested": 80 }, "transaction_recovery": { "correct": 50, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 138, "requested": 160 }, "belebele_en": { "correct": 117, "requested": 160 }, "belebele_zh": { "correct": 113, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 85, "requested": 120 }, "squad2_answerability": { "correct": 106, "requested": 120 }, "snli": { "correct": 107, "requested": 120 }, "qasc": { "correct": 116, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 728, "accuracy": 0.6959847036328872, "full_probability_metrics": { "n": 1046, "nll": 0.9079076925806586, "acc": 0.6959847036328872, "ece": 0.11803573910263573, "brier": 0.41689550886050714, "mean_conf": 0.814020442735523, "confident_error_rate": 0.058317399617590825, "coverage_at_0_9": 0.5525812619502868, "accuracy_at_0_9": 0.8944636678200693, "coverage_at_5pct_error": 0.36424474187380496, "coverage_at_1pct_error": 0.14340344168260039, "aurc": 0.11088769274123257, "error_rate_at_0_9": 0.10553633217993079, "confidence_bias": 0.11803573910263576, "top_bins": { "0.9": { "n": 578, "errors": 61, "error_rate": 0.10553633217993079 }, "0.95": { "n": 491, "errors": 39, "error_rate": 0.07942973523421588 }, "0.99": { "n": 314, "errors": 11, "error_rate": 0.03503184713375796 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9101338432122371, "confidence_cutoff": 0.9301925551074537 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7921146953405018, "confidence_cutoff": 0.5957019462220806 } }, "score_mae": 0.35059676214309027, "ranked_probability_score": 0.1358512450448682 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.9079076925806586, "acc": 0.6959847036328872, "ece": 0.11803573910263573, "brier": 0.41689550886050714, "mean_conf": 0.814020442735523, "confident_error_rate": 0.058317399617590825, "coverage_at_0_9": 0.5525812619502868, "accuracy_at_0_9": 0.8944636678200693, "coverage_at_5pct_error": 0.36424474187380496, "coverage_at_1pct_error": 0.14340344168260039, "aurc": 0.11088769274123257, "error_rate_at_0_9": 0.10553633217993079, "confidence_bias": 0.11803573910263576, "top_bins": { "0.9": { "n": 578, "errors": 61, "error_rate": 0.10553633217993079 }, "0.95": { "n": 491, "errors": 39, "error_rate": 0.07942973523421588 }, "0.99": { "n": 314, "errors": 11, "error_rate": 0.03503184713375796 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9101338432122371, "confidence_cutoff": 0.9301925551074537 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7921146953405018, "confidence_cutoff": 0.5957019462220806 } }, "score_mae": 0.35059676214309027, "ranked_probability_score": 0.1358512450448682 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 5, "accuracy": 0.25, "full_probability_metrics": { "n": 20, "nll": 1.4541223366826688, "acc": 0.25, "ece": 0.33647159228575807, "brier": 0.784234954627584, "mean_conf": 0.5836932166148608, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5585412761902699, "error_rate_at_0_9": 0.0, "confidence_bias": 0.33369321661486084, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.5472205811051023 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.4314323265237658 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.4541223366826688, "acc": 0.25, "ece": 0.33647159228575807, "brier": 0.784234954627584, "mean_conf": 0.5836932166148608, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5585412761902699, "error_rate_at_0_9": 0.0, "confidence_bias": 0.33369321661486084, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.5472205811051023 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.4314323265237658 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 15, "accuracy": 0.75, "full_probability_metrics": { "n": 20, "nll": 0.5510082641235248, "acc": 0.75, "ece": 0.19237663660270798, "brier": 0.328035527127882, "mean_conf": 0.8736262843517905, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.1673632207146139, "error_rate_at_0_9": 0.2, "confidence_bias": 0.12362628435179046, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9014872178213713 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8605996242076215 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5510082641235248, "acc": 0.75, "ece": 0.19237663660270798, "brier": 0.328035527127882, "mean_conf": 0.8736262843517905, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.1673632207146139, "error_rate_at_0_9": 0.2, "confidence_bias": 0.12362628435179046, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9014872178213713 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8605996242076215 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 18, "accuracy": 0.9, "full_probability_metrics": { "n": 20, "nll": 0.5526630617678594, "acc": 0.9, "ece": 0.08451549724807916, "brier": 0.16462432865214055, "mean_conf": 0.9563843565810446, "confident_error_rate": 0.05, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.13238698285718406, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.05638435658104457, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 14, "errors": 1, "error_rate": 0.07142857142857142 }, "0.99": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9881610401010019 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9184301749456608 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5526630617678594, "acc": 0.9, "ece": 0.08451549724807916, "brier": 0.16462432865214055, "mean_conf": 0.9563843565810446, "confident_error_rate": 0.05, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.13238698285718406, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.05638435658104457, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 14, "errors": 1, "error_rate": 0.07142857142857142 }, "0.99": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9881610401010019 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9184301749456608 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 9, "accuracy": 0.45, "full_probability_metrics": { "n": 20, "nll": 1.2065227096255264, "acc": 0.45, "ece": 0.39769566114049026, "brier": 0.8321185752741209, "mean_conf": 0.8113218215459973, "confident_error_rate": 0.25, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.2857142857142857, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5495693812121831, "error_rate_at_0_9": 0.7142857142857143, "confidence_bias": 0.3613218215459973, "top_bins": { "0.9": { "n": 7, "errors": 5, "error_rate": 0.7142857142857143 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.8285751095838813 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.7590371257210207 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.2065227096255264, "acc": 0.45, "ece": 0.39769566114049026, "brier": 0.8321185752741209, "mean_conf": 0.8113218215459973, "confident_error_rate": 0.25, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.2857142857142857, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5495693812121831, "error_rate_at_0_9": 0.7142857142857143, "confidence_bias": 0.3613218215459973, "top_bins": { "0.9": { "n": 7, "errors": 5, "error_rate": 0.7142857142857143 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.8285751095838813 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.7590371257210207 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 24, "accuracy": 0.75, "full_probability_metrics": { "n": 32, "nll": 0.5256356689781225, "acc": 0.75, "ece": 0.1529710399930555, "brier": 0.35279263462074684, "mean_conf": 0.9029710399930555, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.46875, "coverage_at_1pct_error": 0.46875, "aurc": 0.08660865357022784, "error_rate_at_0_9": 0.1, "confidence_bias": 0.1529710399930555, "top_bins": { "0.9": { "n": 20, "errors": 2, "error_rate": 0.1 }, "0.95": { "n": 17, "errors": 2, "error_rate": 0.11764705882352941 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.9523231812860004 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8076923076923077, "confidence_cutoff": 0.7949666858469444 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.5256356689781225, "acc": 0.75, "ece": 0.1529710399930555, "brier": 0.35279263462074684, "mean_conf": 0.9029710399930555, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.46875, "coverage_at_1pct_error": 0.46875, "aurc": 0.08660865357022784, "error_rate_at_0_9": 0.1, "confidence_bias": 0.1529710399930555, "top_bins": { "0.9": { "n": 20, "errors": 2, "error_rate": 0.1 }, "0.95": { "n": 17, "errors": 2, "error_rate": 0.11764705882352941 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.9523231812860004 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8076923076923077, "confidence_cutoff": 0.7949666858469444 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 31, "accuracy": 0.96875, "full_probability_metrics": { "n": 32, "nll": 0.0645410427439187, "acc": 0.96875, "ece": 0.01263240392017407, "brier": 0.04288259191537224, "mean_conf": 0.9561175960798258, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "aurc": 0.0009765625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.012632403920174173, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 28, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.998177071995936 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9916606813279537 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.0645410427439187, "acc": 0.96875, "ece": 0.01263240392017407, "brier": 0.04288259191537224, "mean_conf": 0.9561175960798258, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "aurc": 0.0009765625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.012632403920174173, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 28, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.998177071995936 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9916606813279537 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 28, "accuracy": 0.875, "full_probability_metrics": { "n": 32, "nll": 0.22726442327949534, "acc": 0.875, "ece": 0.07386328589545743, "brier": 0.14503486400352358, "mean_conf": 0.9488632858954573, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 0.9655172413793104, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.65625, "aurc": 0.01880742997747248, "error_rate_at_0_9": 0.034482758620689655, "confidence_bias": 0.07386328589545732, "top_bins": { "0.9": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 }, "0.95": { "n": 23, "errors": 1, "error_rate": 0.043478260869565216 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.984567030580252 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.9467094611131359 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.22726442327949534, "acc": 0.875, "ece": 0.07386328589545743, "brier": 0.14503486400352358, "mean_conf": 0.9488632858954573, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 0.9655172413793104, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.65625, "aurc": 0.01880742997747248, "error_rate_at_0_9": 0.034482758620689655, "confidence_bias": 0.07386328589545732, "top_bins": { "0.9": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 }, "0.95": { "n": 23, "errors": 1, "error_rate": 0.043478260869565216 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.984567030580252 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.9467094611131359 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 39, "accuracy": 0.975, "full_probability_metrics": { "n": 40, "nll": 0.0243362292210797, "acc": 0.975, "ece": 0.017858790014640084, "brier": 0.015699629076925452, "mean_conf": 0.9849901174605661, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.009990117460566106, "top_bins": { "0.9": { "n": 39, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 38, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 35, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9993653730592746 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9921725591053999 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.0243362292210797, "acc": 0.975, "ece": 0.017858790014640084, "brier": 0.015699629076925452, "mean_conf": 0.9849901174605661, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.009990117460566106, "top_bins": { "0.9": { "n": 39, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 38, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 35, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9993653730592746 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9921725591053999 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 28, "accuracy": 0.7, "full_probability_metrics": { "n": 40, "nll": 1.047432121686454, "acc": 0.7, "ece": 0.20673562497432707, "brier": 0.4802232259217643, "mean_conf": 0.8813141453076305, "confident_error_rate": 0.175, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.7407407407407407, "coverage_at_5pct_error": 0.175, "coverage_at_1pct_error": 0.175, "aurc": 0.16984685102162406, "error_rate_at_0_9": 0.25925925925925924, "confidence_bias": 0.18131414530763057, "top_bins": { "0.9": { "n": 27, "errors": 7, "error_rate": 0.25925925925925924 }, "0.95": { "n": 25, "errors": 5, "error_rate": 0.2 }, "0.99": { "n": 15, "errors": 2, "error_rate": 0.13333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.9854354943941819 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.7120645231919425 } }, "score_mae": 0.32940381663734525, "ranked_probability_score": 0.12282431450581155 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.047432121686454, "acc": 0.7, "ece": 0.20673562497432707, "brier": 0.4802232259217643, "mean_conf": 0.8813141453076305, "confident_error_rate": 0.175, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.7407407407407407, "coverage_at_5pct_error": 0.175, "coverage_at_1pct_error": 0.175, "aurc": 0.16984685102162406, "error_rate_at_0_9": 0.25925925925925924, "confidence_bias": 0.18131414530763057, "top_bins": { "0.9": { "n": 27, "errors": 7, "error_rate": 0.25925925925925924 }, "0.95": { "n": 25, "errors": 5, "error_rate": 0.2 }, "0.99": { "n": 15, "errors": 2, "error_rate": 0.13333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.9854354943941819 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.7120645231919425 } }, "score_mae": 0.32940381663734525, "ranked_probability_score": 0.12282431450581155 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 48, "accuracy": 0.6, "full_probability_metrics": { "n": 80, "nll": 1.3247643428213003, "acc": 0.6, "ece": 0.15895568320390185, "brier": 0.5466111456851124, "mean_conf": 0.7589556832039019, "confident_error_rate": 0.05, "coverage_at_0_9": 0.3625, "accuracy_at_0_9": 0.8620689655172413, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.0375, "aurc": 0.20182699408158983, "error_rate_at_0_9": 0.13793103448275862, "confidence_bias": 0.15895568320390197, "top_bins": { "0.9": { "n": 29, "errors": 4, "error_rate": 0.13793103448275862 }, "0.95": { "n": 19, "errors": 1, "error_rate": 0.05263157894736842 }, "0.99": { "n": 11, "errors": 1, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.7988864004860878 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.5632915364363126 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.3247643428213003, "acc": 0.6, "ece": 0.15895568320390185, "brier": 0.5466111456851124, "mean_conf": 0.7589556832039019, "confident_error_rate": 0.05, "coverage_at_0_9": 0.3625, "accuracy_at_0_9": 0.8620689655172413, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.0375, "aurc": 0.20182699408158983, "error_rate_at_0_9": 0.13793103448275862, "confidence_bias": 0.15895568320390197, "top_bins": { "0.9": { "n": 29, "errors": 4, "error_rate": 0.13793103448275862 }, "0.95": { "n": 19, "errors": 1, "error_rate": 0.05263157894736842 }, "0.99": { "n": 11, "errors": 1, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.7988864004860878 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.5632915364363126 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 49, "accuracy": 0.6125, "full_probability_metrics": { "n": 80, "nll": 1.0127333401419472, "acc": 0.6125, "ece": 0.1377193577863906, "brier": 0.5312447700385823, "mean_conf": 0.7247140272548329, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.8695652173913043, "coverage_at_5pct_error": 0.1625, "coverage_at_1pct_error": 0.1625, "aurc": 0.20307145976919885, "error_rate_at_0_9": 0.13043478260869565, "confidence_bias": 0.11221402725483287, "top_bins": { "0.9": { "n": 23, "errors": 3, "error_rate": 0.13043478260869565 }, "0.95": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.7549804106161881 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.5332086761773925 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.0127333401419472, "acc": 0.6125, "ece": 0.1377193577863906, "brier": 0.5312447700385823, "mean_conf": 0.7247140272548329, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.8695652173913043, "coverage_at_5pct_error": 0.1625, "coverage_at_1pct_error": 0.1625, "aurc": 0.20307145976919885, "error_rate_at_0_9": 0.13043478260869565, "confidence_bias": 0.11221402725483287, "top_bins": { "0.9": { "n": 23, "errors": 3, "error_rate": 0.13043478260869565 }, "0.95": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.7549804106161881 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.5332086761773925 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 74, "accuracy": 0.37, "full_probability_metrics": { "n": 200, "nll": 2.1097418028458126, "acc": 0.37, "ece": 0.18888035546451648, "brier": 0.816754409398974, "mean_conf": 0.5588803554645164, "confident_error_rate": 0.04, "coverage_at_0_9": 0.13, "accuracy_at_0_9": 0.6923076923076923, "coverage_at_5pct_error": 0.005, "coverage_at_1pct_error": 0.005, "aurc": 0.4837549628516244, "error_rate_at_0_9": 0.3076923076923077, "confidence_bias": 0.18888035546451643, "top_bins": { "0.9": { "n": 26, "errors": 8, "error_rate": 0.3076923076923077 }, "0.95": { "n": 18, "errors": 7, "error_rate": 0.3888888888888889 }, "0.99": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.51, "confidence_cutoff": 0.5444299474723507 }, "0.8": { "coverage": 0.8, "accuracy": 0.425, "confidence_cutoff": 0.31696554014754635 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.1097418028458126, "acc": 0.37, "ece": 0.18888035546451648, "brier": 0.816754409398974, "mean_conf": 0.5588803554645164, "confident_error_rate": 0.04, "coverage_at_0_9": 0.13, "accuracy_at_0_9": 0.6923076923076923, "coverage_at_5pct_error": 0.005, "coverage_at_1pct_error": 0.005, "aurc": 0.4837549628516244, "error_rate_at_0_9": 0.3076923076923077, "confidence_bias": 0.18888035546451643, "top_bins": { "0.9": { "n": 26, "errors": 8, "error_rate": 0.3076923076923077 }, "0.95": { "n": 18, "errors": 7, "error_rate": 0.3888888888888889 }, "0.99": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.51, "confidence_cutoff": 0.5444299474723507 }, "0.8": { "coverage": 0.8, "accuracy": 0.425, "confidence_cutoff": 0.31696554014754635 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 68, "accuracy": 0.85, "full_probability_metrics": { "n": 80, "nll": 0.4305324059103368, "acc": 0.85, "ece": 0.0743918537090755, "brier": 0.2500832419802905, "mean_conf": 0.895736388780324, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.9074074074074074, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.2125, "aurc": 0.07989816308768842, "error_rate_at_0_9": 0.09259259259259259, "confidence_bias": 0.045736388780324044, "top_bins": { "0.9": { "n": 54, "errors": 5, "error_rate": 0.09259259259259259 }, "0.95": { "n": 40, "errors": 3, "error_rate": 0.075 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9553241765324927 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.8361291885375977 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4305324059103368, "acc": 0.85, "ece": 0.0743918537090755, "brier": 0.2500832419802905, "mean_conf": 0.895736388780324, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.9074074074074074, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.2125, "aurc": 0.07989816308768842, "error_rate_at_0_9": 0.09259259259259259, "confidence_bias": 0.045736388780324044, "top_bins": { "0.9": { "n": 54, "errors": 5, "error_rate": 0.09259259259259259 }, "0.95": { "n": 40, "errors": 3, "error_rate": 0.075 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9553241765324927 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.8361291885375977 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 73, "accuracy": 0.9125, "full_probability_metrics": { "n": 80, "nll": 0.27741353511539624, "acc": 0.9125, "ece": 0.05199690323487452, "brier": 0.1368487492129699, "mean_conf": 0.924803918063375, "confident_error_rate": 0.025, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9682539682539683, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.1125, "aurc": 0.0427126670435451, "error_rate_at_0_9": 0.031746031746031744, "confidence_bias": 0.012303918063375052, "top_bins": { "0.9": { "n": 63, "errors": 2, "error_rate": 0.031746031746031744 }, "0.95": { "n": 60, "errors": 2, "error_rate": 0.03333333333333333 }, "0.99": { "n": 34, "errors": 1, "error_rate": 0.029411764705882353 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9856478809935345 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8547324879547212 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.27741353511539624, "acc": 0.9125, "ece": 0.05199690323487452, "brier": 0.1368487492129699, "mean_conf": 0.924803918063375, "confident_error_rate": 0.025, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9682539682539683, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.1125, "aurc": 0.0427126670435451, "error_rate_at_0_9": 0.031746031746031744, "confidence_bias": 0.012303918063375052, "top_bins": { "0.9": { "n": 63, "errors": 2, "error_rate": 0.031746031746031744 }, "0.95": { "n": 60, "errors": 2, "error_rate": 0.03333333333333333 }, "0.99": { "n": 34, "errors": 1, "error_rate": 0.029411764705882353 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9856478809935345 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8547324879547212 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 79, "accuracy": 0.9875, "full_probability_metrics": { "n": 80, "nll": 0.019254124790419755, "acc": 0.9875, "ece": 0.011027601759015163, "brier": 0.010074870709088662, "mean_conf": 0.9893015647961718, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.001801564796171773, "top_bins": { "0.9": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 70, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997252360553303 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9978996236173687 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.019254124790419755, "acc": 0.9875, "ece": 0.011027601759015163, "brier": 0.010074870709088662, "mean_conf": 0.9893015647961718, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.001801564796171773, "top_bins": { "0.9": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 70, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997252360553303 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9978996236173687 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 60, "accuracy": 0.75, "full_probability_metrics": { "n": 80, "nll": 0.49964852338886434, "acc": 0.75, "ece": 0.10966613449862916, "brier": 0.3391735900372638, "mean_conf": 0.8348346937547854, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.35, "aurc": 0.10212708785291205, "error_rate_at_0_9": 0.1, "confidence_bias": 0.08483469375478536, "top_bins": { "0.9": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.95": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9073231693451269 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.6563300890824172 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.49964852338886434, "acc": 0.75, "ece": 0.10966613449862916, "brier": 0.3391735900372638, "mean_conf": 0.8348346937547854, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.35, "aurc": 0.10212708785291205, "error_rate_at_0_9": 0.1, "confidence_bias": 0.08483469375478536, "top_bins": { "0.9": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.95": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9073231693451269 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.6563300890824172 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 4.349007159296205e-05, "acc": 1.0, "ece": 4.348866768721926e-05, "brier": 5.6553185151565385e-09, "mean_conf": 0.9999565113323128, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -4.348866768721926e-05, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999768015957586 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.99992955359586 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 4.349007159296205e-05, "acc": 1.0, "ece": 4.348866768721926e-05, "brier": 5.6553185151565385e-09, "mean_conf": 0.9999565113323128, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -4.348866768721926e-05, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999768015957586 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.99992955359586 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.03647909789739452, "acc": 1.0, "ece": 0.032862005560244975, "brier": 0.012276567746343992, "mean_conf": 0.967137994439755, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03286200556024499, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9995014876372317 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98595429285109 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.03647909789739452, "acc": 1.0, "ece": 0.032862005560244975, "brier": 0.012276567746343992, "mean_conf": 0.967137994439755, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03286200556024499, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9995014876372317 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98595429285109 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 1.2630390287312345, "acc": 0.6, "ece": 0.26514049358253106, "brier": 0.5611240389725733, "mean_conf": 0.8651404935825312, "confident_error_rate": 0.2, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.7142857142857143, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.20107142857142857, "error_rate_at_0_9": 0.2857142857142857, "confidence_bias": 0.2651404935825312, "top_bins": { "0.9": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 }, "0.95": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 }, "0.99": { "n": 5, "errors": 1, "error_rate": 0.2 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9909728772409536 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6022991268799099 } }, "score_mae": 0.37338389562981816, "ranked_probability_score": 0.14337139808857724 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.2630390287312345, "acc": 0.6, "ece": 0.26514049358253106, "brier": 0.5611240389725733, "mean_conf": 0.8651404935825312, "confident_error_rate": 0.2, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.7142857142857143, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.20107142857142857, "error_rate_at_0_9": 0.2857142857142857, "confidence_bias": 0.2651404935825312, "top_bins": { "0.9": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 }, "0.95": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 }, "0.99": { "n": 5, "errors": 1, "error_rate": 0.2 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9909728772409536 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6022991268799099 } }, "score_mae": 0.37338389562981816, "ranked_probability_score": 0.14337139808857724 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.6507463501618063, "acc": 0.9, "ece": 0.09800439727918331, "brier": 0.1994140697537365, "mean_conf": 0.9980043972791833, "confident_error_rate": 0.1, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": 0.1, "confidence_bias": 0.09800439727918331, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998285766342422 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9993923232675862 } }, "score_mae": 0.10134625759777652, "ranked_probability_score": 0.049845160786496916 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.6507463501618063, "acc": 0.9, "ece": 0.09800439727918331, "brier": 0.1994140697537365, "mean_conf": 0.9980043972791833, "confident_error_rate": 0.1, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": 0.1, "confidence_bias": 0.09800439727918331, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998285766342422 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9993923232675862 } }, "score_mae": 0.10134625759777652, "ranked_probability_score": 0.049845160786496916 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.9715825778240935, "acc": 0.8, "ece": 0.2432005764573813, "brier": 0.4398285824448429, "mean_conf": 0.9098426142854367, "confident_error_rate": 0.2, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.19412698412698412, "error_rate_at_0_9": 0.25, "confidence_bias": 0.10984261428543662, "top_bins": { "0.9": { "n": 8, "errors": 2, "error_rate": 0.25 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9859771448659403 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.9012285932715447 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.9715825778240935, "acc": 0.8, "ece": 0.2432005764573813, "brier": 0.4398285824448429, "mean_conf": 0.9098426142854367, "confident_error_rate": 0.2, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.19412698412698412, "error_rate_at_0_9": 0.25, "confidence_bias": 0.10984261428543662, "top_bins": { "0.9": { "n": 8, "errors": 2, "error_rate": 0.25 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9859771448659403 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.9012285932715447 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.9239412352173808, "acc": 0.4, "ece": 0.5315281234264875, "brier": 1.0096672946342693, "mean_conf": 0.9315281234264875, "confident_error_rate": 0.5, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.4444444444444444, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.48269841269841257, "error_rate_at_0_9": 0.5555555555555556, "confidence_bias": 0.5315281234264875, "top_bins": { "0.9": { "n": 9, "errors": 5, "error_rate": 0.5555555555555556 }, "0.95": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9695129466765593 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9335679094066592 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.9239412352173808, "acc": 0.4, "ece": 0.5315281234264875, "brier": 1.0096672946342693, "mean_conf": 0.9315281234264875, "confident_error_rate": 0.5, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.4444444444444444, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.48269841269841257, "error_rate_at_0_9": 0.5555555555555556, "confidence_bias": 0.5315281234264875, "top_bins": { "0.9": { "n": 9, "errors": 5, "error_rate": 0.5555555555555556 }, "0.95": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9695129466765593 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9335679094066592 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.1869636949597817, "acc": 0.4, "ece": 0.28965910171157766, "brier": 0.677862274253051, "mean_conf": 0.6896591017115776, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.48964285714285716, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.2896591017115776, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6117203497692347 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5315867145595955 } }, "score_mae": 0.5688200975075368, "ranked_probability_score": 0.18981618397857178 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.1869636949597817, "acc": 0.4, "ece": 0.28965910171157766, "brier": 0.677862274253051, "mean_conf": 0.6896591017115776, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.48964285714285716, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.2896591017115776, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6117203497692347 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5315867145595955 } }, "score_mae": 0.5688200975075368, "ranked_probability_score": 0.18981618397857178 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.9409500994437856, "acc": 0.4, "ece": 0.5167113937294245, "brier": 0.9527987845490602, "mean_conf": 0.9167113937294245, "confident_error_rate": 0.3, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.28674603174603175, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.5167113937294244, "top_bins": { "0.9": { "n": 7, "errors": 3, "error_rate": 0.42857142857142855 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9957422325457661 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8366278480083222 } }, "score_mae": 0.5618519024688775, "ranked_probability_score": 0.24776248927800112 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.9409500994437856, "acc": 0.4, "ece": 0.5167113937294245, "brier": 0.9527987845490602, "mean_conf": 0.9167113937294245, "confident_error_rate": 0.3, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.28674603174603175, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.5167113937294244, "top_bins": { "0.9": { "n": 7, "errors": 3, "error_rate": 0.42857142857142855 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9957422325457661 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8366278480083222 } }, "score_mae": 0.5618519024688775, "ranked_probability_score": 0.24776248927800112 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.06761394060100764, "acc": 1.0, "ece": 0.05198777471829483, "brier": 0.04068130154894793, "mean_conf": 0.948012225281705, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.051987774718294966, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9994738448135313 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9859824999329407 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.06761394060100764, "acc": 1.0, "ece": 0.05198777471829483, "brier": 0.04068130154894793, "mean_conf": 0.948012225281705, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.051987774718294966, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9994738448135313 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9859824999329407 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.6072396838101906, "acc": 0.8, "ece": 0.1778739442829313, "brier": 0.33885839627718745, "mean_conf": 0.9778739442829313, "confident_error_rate": 0.1, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.043611111111111114, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.17787394428293124, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997642181523415 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9834781398440434 } }, "score_mae": 0.1891973355789936, "ranked_probability_score": 0.08472185151513073 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.6072396838101906, "acc": 0.8, "ece": 0.1778739442829313, "brier": 0.33885839627718745, "mean_conf": 0.9778739442829313, "confident_error_rate": 0.1, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.043611111111111114, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.17787394428293124, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997642181523415 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9834781398440434 } }, "score_mae": 0.1891973355789936, "ranked_probability_score": 0.08472185151513073 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 1.1774214309220379, "acc": 0.7, "ece": 0.37936253899799444, "brier": 0.5953893006027079, "mean_conf": 0.8375796351080373, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.36702380952380953, "error_rate_at_0_9": 0.4, "confidence_bias": 0.1375796351080374, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9087024673250745 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7741340651127437 } }, "score_mae": 0.3937528660985191, "ranked_probability_score": 0.1516981087786578 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.1774214309220379, "acc": 0.7, "ece": 0.37936253899799444, "brier": 0.5953893006027079, "mean_conf": 0.8375796351080373, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.36702380952380953, "error_rate_at_0_9": 0.4, "confidence_bias": 0.1375796351080374, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9087024673250745 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7741340651127437 } }, "score_mae": 0.3937528660985191, "ranked_probability_score": 0.1516981087786578 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.729675925925926, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.6388888888888888, "flip_rate": 0.16666666666666666, "covered_only_flip_rate": 0.16666666666666666, "half_l1": 0.07949225370274392 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.7864845684290255, "share_at_0_9": 0.2727272727272727, "normalized_entropy": 0.5072453987195353 }, "covered_only_confidence_metrics": { "mean_max_p": 0.7864845684290255, "share_at_0_9": 0.2727272727272727, "normalized_entropy": 0.5072453987195353 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 80, "accuracy": 0.7272727272727273, "full_probability_metrics": { "n": 110, "nll": 0.8932746026945734, "acc": 0.7272727272727273, "ece": 0.18558603949630761, "brier": 0.43890005603982174, "mean_conf": 0.9128587667690348, "confident_error_rate": 0.15454545454545454, "coverage_at_0_9": 0.7818181818181819, "accuracy_at_0_9": 0.8023255813953488, "coverage_at_5pct_error": 0.44545454545454544, "coverage_at_1pct_error": 0.34545454545454546, "aurc": 0.10233847009618001, "error_rate_at_0_9": 0.19767441860465115, "confidence_bias": 0.18558603949630748, "top_bins": { "0.9": { "n": 86, "errors": 17, "error_rate": 0.19767441860465115 }, "0.95": { "n": 76, "errors": 13, "error_rate": 0.17105263157894737 }, "0.99": { "n": 54, "errors": 4, "error_rate": 0.07407407407407407 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9090909090909091, "confidence_cutoff": 0.9896074880665148 }, "0.8": { "coverage": 0.8, "accuracy": 0.7840909090909091, "confidence_cutoff": 0.8514560045658981 } }, "score_mae": 0.3647253924802536, "ranked_probability_score": 0.14453586540423927 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.8932746026945734, "acc": 0.7272727272727273, "ece": 0.18558603949630761, "brier": 0.43890005603982174, "mean_conf": 0.9128587667690348, "confident_error_rate": 0.15454545454545454, "coverage_at_0_9": 0.7818181818181819, "accuracy_at_0_9": 0.8023255813953488, "coverage_at_5pct_error": 0.44545454545454544, "coverage_at_1pct_error": 0.34545454545454546, "aurc": 0.10233847009618001, "error_rate_at_0_9": 0.19767441860465115, "confidence_bias": 0.18558603949630748, "top_bins": { "0.9": { "n": 86, "errors": 17, "error_rate": 0.19767441860465115 }, "0.95": { "n": 76, "errors": 13, "error_rate": 0.17105263157894737 }, "0.99": { "n": 54, "errors": 4, "error_rate": 0.07407407407407407 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9090909090909091, "confidence_cutoff": 0.9896074880665148 }, "0.8": { "coverage": 0.8, "accuracy": 0.7840909090909091, "confidence_cutoff": 0.8514560045658981 } }, "score_mae": 0.3647253924802536, "ranked_probability_score": 0.14453586540423927 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.12637419834000913 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.7834679467690011, "paired_flip": { "pairs": 64, "flip_rate": 0.828125, "both_correct_rate": 0.671875, "invariant_pairs": 24, "invariance_rate": 0.9583333333333334, "invariant_both_correct_rate": 0.875 }, "unknowable": { "n": 110, "mean_max_p": 0.7864845684290255, "share_at_0_9": 0.2727272727272727, "control_mean_max_p": 0.9128587667690348, "control_share_at_0_9": 0.7818181818181819, "control_acc": 0.7272727272727273, "paired_confidence_drop": 0.12637419834000913, "share_less_confident_than_control": 0.7727272727272727 }, "clean": { "n": 1046, "nll": 0.9079076925806586, "acc": 0.6959847036328872, "ece": 0.11803573910263573, "brier": 0.41689550886050714, "mean_conf": 0.814020442735523, "confident_error_rate": 0.058317399617590825, "coverage_at_0_9": 0.5525812619502868, "accuracy_at_0_9": 0.8944636678200693, "coverage_at_5pct_error": 0.36424474187380496, "coverage_at_1pct_error": 0.14340344168260039, "aurc": 0.11088769274123257, "error_rate_at_0_9": 0.10553633217993079, "confidence_bias": 0.11803573910263576, "top_bins": { "0.9": { "n": 578, "errors": 61, "error_rate": 0.10553633217993079 }, "0.95": { "n": 491, "errors": 39, "error_rate": 0.07942973523421588 }, "0.99": { "n": 314, "errors": 11, "error_rate": 0.03503184713375796 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9101338432122371, "confidence_cutoff": 0.9301925551074537 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7921146953405018, "confidence_cutoff": 0.5957019462220806 } }, "score_mae": 0.35059676214309027, "ranked_probability_score": 0.1358512450448682 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.4541223366826688, "acc": 0.25, "ece": 0.33647159228575807, "brier": 0.784234954627584, "mean_conf": 0.5836932166148608, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5585412761902699, "error_rate_at_0_9": 0.0, "confidence_bias": 0.33369321661486084, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.5472205811051023 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.4314323265237658 } } }, "buried_paws": { "n": 20, "nll": 0.5510082641235248, "acc": 0.75, "ece": 0.19237663660270798, "brier": 0.328035527127882, "mean_conf": 0.8736262843517905, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.1673632207146139, "error_rate_at_0_9": 0.2, "confidence_bias": 0.12362628435179046, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9014872178213713 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8605996242076215 } } }, "buried_qnli": { "n": 20, "nll": 0.5526630617678594, "acc": 0.9, "ece": 0.08451549724807916, "brier": 0.16462432865214055, "mean_conf": 0.9563843565810446, "confident_error_rate": 0.05, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.13238698285718406, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.05638435658104457, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 14, "errors": 1, "error_rate": 0.07142857142857142 }, "0.99": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9881610401010019 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9184301749456608 } } }, "buried_tweet_offensive": { "n": 20, "nll": 1.2065227096255264, "acc": 0.45, "ece": 0.39769566114049026, "brier": 0.8321185752741209, "mean_conf": 0.8113218215459973, "confident_error_rate": 0.25, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.2857142857142857, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5495693812121831, "error_rate_at_0_9": 0.7142857142857143, "confidence_bias": 0.3613218215459973, "top_bins": { "0.9": { "n": 7, "errors": 5, "error_rate": 0.7142857142857143 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.8285751095838813 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.7590371257210207 } } }, "composition_final_combination": { "n": 32, "nll": 0.5256356689781225, "acc": 0.75, "ece": 0.1529710399930555, "brier": 0.35279263462074684, "mean_conf": 0.9029710399930555, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.46875, "coverage_at_1pct_error": 0.46875, "aurc": 0.08660865357022784, "error_rate_at_0_9": 0.1, "confidence_bias": 0.1529710399930555, "top_bins": { "0.9": { "n": 20, "errors": 2, "error_rate": 0.1 }, "0.95": { "n": 17, "errors": 2, "error_rate": 0.11764705882352941 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.9523231812860004 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8076923076923077, "confidence_cutoff": 0.7949666858469444 } } }, "composition_final_exception": { "n": 32, "nll": 0.0645410427439187, "acc": 0.96875, "ece": 0.01263240392017407, "brier": 0.04288259191537224, "mean_conf": 0.9561175960798258, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "aurc": 0.0009765625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.012632403920174173, "top_bins": { "0.9": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 28, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.998177071995936 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9916606813279537 } } }, "composition_final_negation": { "n": 32, "nll": 0.22726442327949534, "acc": 0.875, "ece": 0.07386328589545743, "brier": 0.14503486400352358, "mean_conf": 0.9488632858954573, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 0.9655172413793104, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.65625, "aurc": 0.01880742997747248, "error_rate_at_0_9": 0.034482758620689655, "confidence_bias": 0.07386328589545732, "top_bins": { "0.9": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 }, "0.95": { "n": 23, "errors": 1, "error_rate": 0.043478260869565216 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 0.984567030580252 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.9467094611131359 } } }, "contrastive_authorization": { "n": 40, "nll": 0.0243362292210797, "acc": 0.975, "ece": 0.017858790014640084, "brier": 0.015699629076925452, "mean_conf": 0.9849901174605661, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.009990117460566106, "top_bins": { "0.9": { "n": 39, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 38, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 35, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9993653730592746 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9921725591053999 } } }, "contrastive_deadline": { "n": 40, "nll": 1.047432121686454, "acc": 0.7, "ece": 0.20673562497432707, "brier": 0.4802232259217643, "mean_conf": 0.8813141453076305, "confident_error_rate": 0.175, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.7407407407407407, "coverage_at_5pct_error": 0.175, "coverage_at_1pct_error": 0.175, "aurc": 0.16984685102162406, "error_rate_at_0_9": 0.25925925925925924, "confidence_bias": 0.18131414530763057, "top_bins": { "0.9": { "n": 27, "errors": 7, "error_rate": 0.25925925925925924 }, "0.95": { "n": 25, "errors": 5, "error_rate": 0.2 }, "0.99": { "n": 15, "errors": 2, "error_rate": 0.13333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.9854354943941819 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.7120645231919425 } }, "score_mae": 0.32940381663734525, "ranked_probability_score": 0.12282431450581155 }, "emotion": { "n": 80, "nll": 1.3247643428213003, "acc": 0.6, "ece": 0.15895568320390185, "brier": 0.5466111456851124, "mean_conf": 0.7589556832039019, "confident_error_rate": 0.05, "coverage_at_0_9": 0.3625, "accuracy_at_0_9": 0.8620689655172413, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.0375, "aurc": 0.20182699408158983, "error_rate_at_0_9": 0.13793103448275862, "confidence_bias": 0.15895568320390197, "top_bins": { "0.9": { "n": 29, "errors": 4, "error_rate": 0.13793103448275862 }, "0.95": { "n": 19, "errors": 1, "error_rate": 0.05263157894736842 }, "0.99": { "n": 11, "errors": 1, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.7988864004860878 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.5632915364363126 } } }, "mmlu": { "n": 80, "nll": 1.0127333401419472, "acc": 0.6125, "ece": 0.1377193577863906, "brier": 0.5312447700385823, "mean_conf": 0.7247140272548329, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.8695652173913043, "coverage_at_5pct_error": 0.1625, "coverage_at_1pct_error": 0.1625, "aurc": 0.20307145976919885, "error_rate_at_0_9": 0.13043478260869565, "confidence_bias": 0.11221402725483287, "top_bins": { "0.9": { "n": 23, "errors": 3, "error_rate": 0.13043478260869565 }, "0.95": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.7549804106161881 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.5332086761773925 } } }, "mmlu_pro": { "n": 200, "nll": 2.1097418028458126, "acc": 0.37, "ece": 0.18888035546451648, "brier": 0.816754409398974, "mean_conf": 0.5588803554645164, "confident_error_rate": 0.04, "coverage_at_0_9": 0.13, "accuracy_at_0_9": 0.6923076923076923, "coverage_at_5pct_error": 0.005, "coverage_at_1pct_error": 0.005, "aurc": 0.4837549628516244, "error_rate_at_0_9": 0.3076923076923077, "confidence_bias": 0.18888035546451643, "top_bins": { "0.9": { "n": 26, "errors": 8, "error_rate": 0.3076923076923077 }, "0.95": { "n": 18, "errors": 7, "error_rate": 0.3888888888888889 }, "0.99": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.51, "confidence_cutoff": 0.5444299474723507 }, "0.8": { "coverage": 0.8, "accuracy": 0.425, "confidence_cutoff": 0.31696554014754635 } } }, "paws": { "n": 80, "nll": 0.4305324059103368, "acc": 0.85, "ece": 0.0743918537090755, "brier": 0.2500832419802905, "mean_conf": 0.895736388780324, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.9074074074074074, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.2125, "aurc": 0.07989816308768842, "error_rate_at_0_9": 0.09259259259259259, "confidence_bias": 0.045736388780324044, "top_bins": { "0.9": { "n": 54, "errors": 5, "error_rate": 0.09259259259259259 }, "0.95": { "n": 40, "errors": 3, "error_rate": 0.075 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9553241765324927 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.8361291885375977 } } }, "qnli": { "n": 80, "nll": 0.27741353511539624, "acc": 0.9125, "ece": 0.05199690323487452, "brier": 0.1368487492129699, "mean_conf": 0.924803918063375, "confident_error_rate": 0.025, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9682539682539683, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.1125, "aurc": 0.0427126670435451, "error_rate_at_0_9": 0.031746031746031744, "confidence_bias": 0.012303918063375052, "top_bins": { "0.9": { "n": 63, "errors": 2, "error_rate": 0.031746031746031744 }, "0.95": { "n": 60, "errors": 2, "error_rate": 0.03333333333333333 }, "0.99": { "n": 34, "errors": 1, "error_rate": 0.029411764705882353 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9856478809935345 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8547324879547212 } } }, "sciq": { "n": 80, "nll": 0.019254124790419755, "acc": 0.9875, "ece": 0.011027601759015163, "brier": 0.010074870709088662, "mean_conf": 0.9893015647961718, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.001801564796171773, "top_bins": { "0.9": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 70, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997252360553303 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9978996236173687 } } }, "tweet_offensive": { "n": 80, "nll": 0.49964852338886434, "acc": 0.75, "ece": 0.10966613449862916, "brier": 0.3391735900372638, "mean_conf": 0.8348346937547854, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.35, "aurc": 0.10212708785291205, "error_rate_at_0_9": 0.1, "confidence_bias": 0.08483469375478536, "top_bins": { "0.9": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.95": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9073231693451269 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.6563300890824172 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 1.048844192772443, "acc": 0.5, "ece": 0.2748845499033215, "brier": 0.7066621742701622, "mean_conf": 0.7748845499033215, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.27488454990332145, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8608284592628479 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.656721611521292 } } }, "unknowable_authorization": { "n": 10, "nll": 2.933123364057356, "acc": 0.5, "ece": 0.4971052379489367, "brier": 0.9940740011137097, "mean_conf": 0.9971052379489367, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4273015873015873, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4971052379489367, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 10, "errors": 5, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9973643354783845 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9969000554062178 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 4.349007159296205e-05, "acc": 1.0, "ece": 4.348866768721926e-05, "brier": 5.6553185151565385e-09, "mean_conf": 0.9999565113323128, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -4.348866768721926e-05, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999768015957586 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.99992955359586 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.03647909789739452, "acc": 1.0, "ece": 0.032862005560244975, "brier": 0.012276567746343992, "mean_conf": 0.967137994439755, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03286200556024499, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9995014876372317 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.98595429285109 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.2630390287312345, "acc": 0.6, "ece": 0.26514049358253106, "brier": 0.5611240389725733, "mean_conf": 0.8651404935825312, "confident_error_rate": 0.2, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.7142857142857143, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.20107142857142857, "error_rate_at_0_9": 0.2857142857142857, "confidence_bias": 0.2651404935825312, "top_bins": { "0.9": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 }, "0.95": { "n": 7, "errors": 2, "error_rate": 0.2857142857142857 }, "0.99": { "n": 5, "errors": 1, "error_rate": 0.2 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9909728772409536 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6022991268799099 } }, "score_mae": 0.37338389562981816, "ranked_probability_score": 0.14337139808857724 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.6507463501618063, "acc": 0.9, "ece": 0.09800439727918331, "brier": 0.1994140697537365, "mean_conf": 0.9980043972791833, "confident_error_rate": 0.1, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": 0.1, "confidence_bias": 0.09800439727918331, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998285766342422 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9993923232675862 } }, "score_mae": 0.10134625759777652, "ranked_probability_score": 0.049845160786496916 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.9715825778240935, "acc": 0.8, "ece": 0.2432005764573813, "brier": 0.4398285824448429, "mean_conf": 0.9098426142854367, "confident_error_rate": 0.2, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.19412698412698412, "error_rate_at_0_9": 0.25, "confidence_bias": 0.10984261428543662, "top_bins": { "0.9": { "n": 8, "errors": 2, "error_rate": 0.25 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9859771448659403 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.9012285932715447 } } }, "unknowable_control_return_window": { "n": 10, "nll": 1.9239412352173808, "acc": 0.4, "ece": 0.5315281234264875, "brier": 1.0096672946342693, "mean_conf": 0.9315281234264875, "confident_error_rate": 0.5, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.4444444444444444, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.48269841269841257, "error_rate_at_0_9": 0.5555555555555556, "confidence_bias": 0.5315281234264875, "top_bins": { "0.9": { "n": 9, "errors": 5, "error_rate": 0.5555555555555556 }, "0.95": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9695129466765593 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9335679094066592 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 1.1869636949597817, "acc": 0.4, "ece": 0.28965910171157766, "brier": 0.677862274253051, "mean_conf": 0.6896591017115776, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.48964285714285716, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.2896591017115776, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6117203497692347 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5315867145595955 } }, "score_mae": 0.5688200975075368, "ranked_probability_score": 0.18981618397857178 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.9409500994437856, "acc": 0.4, "ece": 0.5167113937294245, "brier": 0.9527987845490602, "mean_conf": 0.9167113937294245, "confident_error_rate": 0.3, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.28674603174603175, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.5167113937294244, "top_bins": { "0.9": { "n": 7, "errors": 3, "error_rate": 0.42857142857142855 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9957422325457661 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8366278480083222 } }, "score_mae": 0.5618519024688775, "ranked_probability_score": 0.24776248927800112 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.06761394060100764, "acc": 1.0, "ece": 0.05198777471829483, "brier": 0.04068130154894793, "mean_conf": 0.948012225281705, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.051987774718294966, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9994738448135313 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9859824999329407 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.6072396838101906, "acc": 0.8, "ece": 0.1778739442829313, "brier": 0.33885839627718745, "mean_conf": 0.9778739442829313, "confident_error_rate": 0.1, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.043611111111111114, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.17787394428293124, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997642181523415 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9834781398440434 } }, "score_mae": 0.1891973355789936, "ranked_probability_score": 0.08472185151513073 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 1.1774214309220379, "acc": 0.7, "ece": 0.37936253899799444, "brier": 0.5953893006027079, "mean_conf": 0.8375796351080373, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.36702380952380953, "error_rate_at_0_9": 0.4, "confidence_bias": 0.1375796351080374, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9087024673250745 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7741340651127437 } }, "score_mae": 0.3937528660985191, "ranked_probability_score": 0.1516981087786578 }, "unknowable_deadline": { "n": 10, "nll": 1.451992889687925, "acc": 0.4, "ece": 0.34564668776773744, "brier": 0.815814715912046, "mean_conf": 0.7456466877677375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5793650793650793, "error_rate_at_0_9": null, "confidence_bias": 0.34564668776773744, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8033998249441463 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6559939604399564 } }, "score_mae": 0.7223433763625182, "ranked_probability_score": 0.26679270147301143 }, "unknowable_late_fee": { "n": 10, "nll": 1.6350510250869283, "acc": 0.3, "ece": 0.49918152841229224, "brier": 0.9915824820904847, "mean_conf": 0.7991815284122922, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5983333333333333, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4991815284122922, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.8371454130752508 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7872959756659258 } }, "score_mae": 0.7592312577137725, "ranked_probability_score": 0.3053086724900892 }, "unknowable_quantity_limit": { "n": 10, "nll": 2.215500054724211, "acc": 0.5, "ece": 0.4131784353073209, "brier": 0.9137340699579747, "mean_conf": 0.9131784353073208, "confident_error_rate": 0.3, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4131784353073208, "top_bins": { "0.9": { "n": 6, "errors": 3, "error_rate": 0.5 }, "0.95": { "n": 6, "errors": 3, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.9693512316217116 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8960489251625464 } } }, "unknowable_return_window": { "n": 10, "nll": 0.935932150336164, "acc": 0.5, "ece": 0.41106899006508046, "brier": 0.6799470868566309, "mean_conf": 0.6367636701634779, "confident_error_rate": 0.1, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7778174603174604, "error_rate_at_0_9": 1.0, "confidence_bias": 0.1367636701634779, "top_bins": { "0.9": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6251804828643799 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5921892523765564 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.6409938561098243, "acc": 0.4, "ece": 0.4714669256893661, "brier": 0.8378632494001279, "mean_conf": 0.7316861702430386, "confident_error_rate": 0.1, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.666031746031746, "error_rate_at_0_9": 1.0, "confidence_bias": 0.3316861702430386, "top_bins": { "0.9": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.8034843191439578 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.646818995475769 } }, "score_mae": 0.8047054475567172, "ranked_probability_score": 0.3074525740343078 }, "unknowable_sla_response": { "n": 10, "nll": 1.4978023119580108, "acc": 0.2, "ece": 0.37600319889442363, "brier": 0.8886656536072085, "mean_conf": 0.5760031988944236, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9016666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.37600319889442363, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.6007340833161877 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.49394236118962676 } }, "score_mae": 0.8244604582950565, "ranked_probability_score": 0.3032582677441629 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.9853018065474901, "acc": 0.5, "ece": 0.3209044890657235, "brier": 0.7119714317253193, "mean_conf": 0.8209044890657236, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.32090448906572355, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8048679300060602 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8044344265005536 } } }, "unknowable_volume_discount": { "n": 10, "nll": 4.9847859094202045, "acc": 0.3, "ece": 0.6499662984749598, "brier": 1.3382543181507045, "mean_conf": 0.9499662984749598, "confident_error_rate": 0.6, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6733333333333332, "error_rate_at_0_9": 0.75, "confidence_bias": 0.6499662984749599, "top_bins": { "0.9": { "n": 8, "errors": 6, "error_rate": 0.75 }, "0.95": { "n": 6, "errors": 4, "error_rate": 0.6666666666666666 }, "0.99": { "n": 4, "errors": 3, "error_rate": 0.75 } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.95663632137295 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.9461429723576554 } }, "score_mae": 0.8085405434948656, "ranked_probability_score": 0.3819673400973467 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.3914526232361453, "acc": 0.6, "ece": 0.2606947510173806, "brier": 0.6082191341878626, "mean_conf": 0.7060099865380499, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.4010714285714285, "error_rate_at_0_9": null, "confidence_bias": 0.10600998653804994, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7133394083009867 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6673887968063354 } }, "score_mae": 0.5449903009813266, "ranked_probability_score": 0.18497319330259748 } }, "variants": { "clean": { "n": 1156, "nll": 1.0007605953968302, "acc": 0.6704152249134948, "ece": 0.1409850221700258, "brier": 0.45929116387613567, "mean_conf": 0.8114002470835204, "confident_error_rate": 0.06833910034602077, "coverage_at_0_9": 0.5259515570934256, "accuracy_at_0_9": 0.8700657894736842, "coverage_at_5pct_error": 0.20242214532871972, "coverage_at_1pct_error": 0.12975778546712802, "aurc": 0.1369722577685153, "error_rate_at_0_9": 0.1299342105263158, "confidence_bias": 0.14098502217002562, "top_bins": { "0.9": { "n": 608, "errors": 79, "error_rate": 0.1299342105263158 }, "0.95": { "n": 515, "errors": 52, "error_rate": 0.10097087378640776 }, "0.99": { "n": 328, "errors": 19, "error_rate": 0.057926829268292686 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8788927335640139, "confidence_cutoff": 0.9157804353063602 }, "0.8": { "coverage": 0.801038062283737, "accuracy": 0.7591792656587473, "confidence_cutoff": 0.6007340833161877 } }, "score_mae": 0.49813993786469746, "ranked_probability_score": 0.19426657497438732 }, "none_absent": { "n": 36, "nll": 2.016479899399725, "acc": 0.2777777777777778, "ece": 0.43639357499087156, "brier": 1.0193378435955023, "mean_conf": 0.7141713527686493, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.1111111111111111, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6622061759903008, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4363935749908715, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3888888888888889, "confidence_cutoff": 0.7284490118427092 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.3448275862068966, "confidence_cutoff": 0.5772219386282852 } } }, "none_present": { "n": 36, "nll": 0.9834989953062324, "acc": 0.6666666666666666, "ece": 0.16246789063665687, "brier": 0.42631332906431735, "mean_conf": 0.7475449294183114, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3888888888888889, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.10954859179448669, "error_rate_at_0_9": 0.0, "confidence_bias": 0.08087826275164478, "top_bins": { "0.9": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8038808995404794 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7586206896551724, "confidence_cutoff": 0.514763438695478 } } }, "permuted": { "n": 36, "nll": 0.9187906169822195, "acc": 0.6666666666666666, "ece": 0.13776171917514016, "brier": 0.4066078577529032, "mean_conf": 0.7543481885836296, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4166666666666667, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5555555555555556, "coverage_at_1pct_error": 0.5, "aurc": 0.09826261699381092, "error_rate_at_0_9": 0.0, "confidence_bias": 0.08768152191696299, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8030261229089575 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7931034482758621, "confidence_cutoff": 0.5168578524128333 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.07510801427468833, "flip_rate": 0.16666666666666666 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.9079076925806586, "acc": 0.6959847036328872, "ece": 0.11803573910263573, "brier": 0.41689550886050714, "mean_conf": 0.814020442735523, "confident_error_rate": 0.058317399617590825, "coverage_at_0_9": 0.5525812619502868, "accuracy_at_0_9": 0.8944636678200693, "coverage_at_5pct_error": 0.36424474187380496, "coverage_at_1pct_error": 0.14340344168260039, "aurc": 0.11088769274123257, "error_rate_at_0_9": 0.10553633217993079, "confidence_bias": 0.11803573910263576, "top_bins": { "0.9": { "n": 578, "errors": 61, "error_rate": 0.10553633217993079 }, "0.95": { "n": 491, "errors": 39, "error_rate": 0.07942973523421588 }, "0.99": { "n": 314, "errors": 11, "error_rate": 0.03503184713375796 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9101338432122371, "confidence_cutoff": 0.9301925551074537 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7921146953405018, "confidence_cutoff": 0.5957019462220806 } }, "score_mae": 0.35059676214309027, "ranked_probability_score": 0.1358512450448682 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "kev-9b": { "panels": { "old_core": { "point": 0.7674851190476191, "exact_fraction": "2063/2688", "ci95": [ 0.7324767485119047, 0.80234375 ] }, "v3_core": { "point": 0.4575, "exact_fraction": "183/400", "ci95": [ 0.4325, 0.4820833333333333 ] }, "v4": { "point": 0.8671875, "exact_fraction": "111/128", "ci95": [ 0.8296875, 0.901812798566879 ] }, "v5": { "point": 0.8354166666666667, "exact_fraction": "401/480", "ci95": [ 0.8041802520135837, 0.8659082981709634 ] }, "transfer_v9_test": { "point": 0.7925430210325047, "exact_fraction": "829/1046", "ci95": [ 0.7643116165500827, 0.8199802176063303 ] }, "original_four_panel_mean": { "point": 0.7318973214285714, "exact_fraction": "32789/44800", "ci95": [ 0.7156388195610951, 0.7476547801749678 ] }, "weighted_mean": { "point": 0.7188926138691615, "exact_fraction": "33687883/46860800", "ci95": [ 0.7041827343835357, 0.7334576867498865 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 112, "requested": 128 }, "boolean_constraints": { "correct": 61, "requested": 64 }, "dbpedia_14": { "correct": 110, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 48, "requested": 96 }, "ordinal_rubric": { "correct": 62, "requested": 64 }, "relational_composition": { "correct": 49, "requested": 96 }, "scoped_evidence": { "correct": 63, "requested": 96 }, "state_tracking": { "correct": 37, "requested": 96 }, "unknown_rejection": { "correct": 54, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 36, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 22, "requested": 80 }, "massive_en": { "correct": 104, "requested": 120 }, "massive_zh": { "correct": 103, "requested": 120 }, "multiset_reconciliation": { "correct": 30, "requested": 80 }, "ordinal_service_loss": { "correct": 20, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 22, "requested": 80 }, "temporal_exclusion": { "correct": 23, "requested": 80 }, "transaction_recovery": { "correct": 35, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 146, "requested": 160 }, "belebele_en": { "correct": 133, "requested": 160 }, "belebele_zh": { "correct": 130, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 82, "requested": 120 }, "squad2_answerability": { "correct": 94, "requested": 120 }, "snli": { "correct": 109, "requested": 120 }, "qasc": { "correct": 116, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 829, "accuracy": 0.7925430210325047, "full_probability_metrics": { "n": 1046, "nll": 0.6027548590263215, "acc": 0.7925430210325047, "ece": 0.044502507431275706, "brier": 0.294775757633525, "mean_conf": 0.7761167229324795, "confident_error_rate": 0.0248565965583174, "coverage_at_0_9": 0.5296367112810707, "accuracy_at_0_9": 0.9530685920577617, "coverage_at_5pct_error": 0.5803059273422562, "coverage_at_1pct_error": 0.29732313575525815, "aurc": 0.05863375703305322, "error_rate_at_0_9": 0.04693140794223827, "confidence_bias": -0.01642629810002527, "top_bins": { "0.9": { "n": 554, "errors": 26, "error_rate": 0.04693140794223827 }, "0.95": { "n": 437, "errors": 10, "error_rate": 0.02288329519450801 }, "0.99": { "n": 119, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9521988527724665, "confidence_cutoff": 0.9187699097211646 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8864994026284349, "confidence_cutoff": 0.5194740121242921 } }, "score_mae": 0.18132269690566039, "ranked_probability_score": 0.04791018680601314 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.6027548590263215, "acc": 0.7925430210325047, "ece": 0.044502507431275706, "brier": 0.294775757633525, "mean_conf": 0.7761167229324795, "confident_error_rate": 0.0248565965583174, "coverage_at_0_9": 0.5296367112810707, "accuracy_at_0_9": 0.9530685920577617, "coverage_at_5pct_error": 0.5803059273422562, "coverage_at_1pct_error": 0.29732313575525815, "aurc": 0.05863375703305322, "error_rate_at_0_9": 0.04693140794223827, "confidence_bias": -0.01642629810002527, "top_bins": { "0.9": { "n": 554, "errors": 26, "error_rate": 0.04693140794223827 }, "0.95": { "n": 437, "errors": 10, "error_rate": 0.02288329519450801 }, "0.99": { "n": 119, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9521988527724665, "confidence_cutoff": 0.9187699097211646 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8864994026284349, "confidence_cutoff": 0.5194740121242921 } }, "score_mae": 0.18132269690566039, "ranked_probability_score": 0.04791018680601314 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 13, "accuracy": 0.65, "full_probability_metrics": { "n": 20, "nll": 1.0527257481257462, "acc": 0.65, "ece": 0.3051233916894593, "brier": 0.5023087907915027, "mean_conf": 0.4819157184465988, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.18057358555617067, "error_rate_at_0_9": null, "confidence_bias": -0.16808428155340122, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.4279851929395839 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.2728929499321211 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.0527257481257462, "acc": 0.65, "ece": 0.3051233916894593, "brier": 0.5023087907915027, "mean_conf": 0.4819157184465988, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.18057358555617067, "error_rate_at_0_9": null, "confidence_bias": -0.16808428155340122, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.4279851929395839 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.2728929499321211 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 16, "accuracy": 0.8, "full_probability_metrics": { "n": 20, "nll": 0.6833261160573135, "acc": 0.8, "ece": 0.16396340710521495, "brier": 0.371614784794417, "mean_conf": 0.963963407105215, "confident_error_rate": 0.2, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.11906431782012276, "error_rate_at_0_9": 0.2, "confidence_bias": 0.16396340710521495, "top_bins": { "0.9": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.95": { "n": 14, "errors": 2, "error_rate": 0.14285714285714285 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9716955314759664 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.9488051361801536 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.6833261160573135, "acc": 0.8, "ece": 0.16396340710521495, "brier": 0.371614784794417, "mean_conf": 0.963963407105215, "confident_error_rate": 0.2, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.11906431782012276, "error_rate_at_0_9": 0.2, "confidence_bias": 0.16396340710521495, "top_bins": { "0.9": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.95": { "n": 14, "errors": 2, "error_rate": 0.14285714285714285 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9716955314759664 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.9488051361801536 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 19, "accuracy": 0.95, "full_probability_metrics": { "n": 20, "nll": 0.148984242944029, "acc": 0.95, "ece": 0.07618705591020539, "brier": 0.08005192124027319, "mean_conf": 0.9056351007639526, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9, "aurc": 0.005131578947368421, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04436489923604736, "top_bins": { "0.9": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9484042101321722 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8695504794947875 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.148984242944029, "acc": 0.95, "ece": 0.07618705591020539, "brier": 0.08005192124027319, "mean_conf": 0.9056351007639526, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9, "aurc": 0.005131578947368421, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04436489923604736, "top_bins": { "0.9": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9484042101321722 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8695504794947875 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 16, "accuracy": 0.8, "full_probability_metrics": { "n": 20, "nll": 0.4566841691053221, "acc": 0.8, "ece": 0.11810294600060224, "brier": 0.29841595844660157, "mean_conf": 0.7647957278678895, "confident_error_rate": 0.0, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.101723631545613, "error_rate_at_0_9": 0.0, "confidence_bias": -0.035204272132110526, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7540354245012317 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.666028181730861 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.4566841691053221, "acc": 0.8, "ece": 0.11810294600060224, "brier": 0.29841595844660157, "mean_conf": 0.7647957278678895, "confident_error_rate": 0.0, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.101723631545613, "error_rate_at_0_9": 0.0, "confidence_bias": -0.035204272132110526, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7540354245012317 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.666028181730861 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 22, "accuracy": 0.6875, "full_probability_metrics": { "n": 32, "nll": 0.5866928427487809, "acc": 0.6875, "ece": 0.2387653374086254, "brier": 0.41343064573118143, "mean_conf": 0.9262653374086254, "confident_error_rate": 0.125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.6875, "aurc": 0.06261277252248122, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.23876533740862538, "top_bins": { "0.9": { "n": 26, "errors": 4, "error_rate": 0.15384615384615385 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9836031674848541 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8461538461538461, "confidence_cutoff": 0.9233322143554688 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.5866928427487809, "acc": 0.6875, "ece": 0.2387653374086254, "brier": 0.41343064573118143, "mean_conf": 0.9262653374086254, "confident_error_rate": 0.125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.6875, "aurc": 0.06261277252248122, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.23876533740862538, "top_bins": { "0.9": { "n": 26, "errors": 4, "error_rate": 0.15384615384615385 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9836031674848541 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8461538461538461, "confidence_cutoff": 0.9233322143554688 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 28, "accuracy": 0.875, "full_probability_metrics": { "n": 32, "nll": 0.30002121819543187, "acc": 0.875, "ece": 0.09059456263848523, "brier": 0.19589500998782167, "mean_conf": 0.9655945626384852, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.96875, "accuracy_at_0_9": 0.9032258064516129, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.875, "aurc": 0.011097110215053763, "error_rate_at_0_9": 0.0967741935483871, "confidence_bias": 0.0905945626384852, "top_bins": { "0.9": { "n": 31, "errors": 3, "error_rate": 0.0967741935483871 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9773114432839093 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9668880032979845 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.30002121819543187, "acc": 0.875, "ece": 0.09059456263848523, "brier": 0.19589500998782167, "mean_conf": 0.9655945626384852, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.96875, "accuracy_at_0_9": 0.9032258064516129, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.875, "aurc": 0.011097110215053763, "error_rate_at_0_9": 0.0967741935483871, "confidence_bias": 0.0905945626384852, "top_bins": { "0.9": { "n": 31, "errors": 3, "error_rate": 0.0967741935483871 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9773114432839093 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9668880032979845 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 29, "accuracy": 0.90625, "full_probability_metrics": { "n": 32, "nll": 0.32524130332020473, "acc": 0.90625, "ece": 0.10976979965216396, "brier": 0.1938474981101241, "mean_conf": 0.9339476677914791, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.8928571428571429, "coverage_at_5pct_error": 0.78125, "coverage_at_1pct_error": 0.78125, "aurc": 0.020194977536033568, "error_rate_at_0_9": 0.10714285714285714, "confidence_bias": 0.027697667791479086, "top_bins": { "0.9": { "n": 28, "errors": 3, "error_rate": 0.10714285714285714 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.972606823435371 }, "0.8": { "coverage": 0.84375, "accuracy": 0.9259259259259259, "confidence_cutoff": 0.934568040560424 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.32524130332020473, "acc": 0.90625, "ece": 0.10976979965216396, "brier": 0.1938474981101241, "mean_conf": 0.9339476677914791, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.8928571428571429, "coverage_at_5pct_error": 0.78125, "coverage_at_1pct_error": 0.78125, "aurc": 0.020194977536033568, "error_rate_at_0_9": 0.10714285714285714, "confidence_bias": 0.027697667791479086, "top_bins": { "0.9": { "n": 28, "errors": 3, "error_rate": 0.10714285714285714 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.972606823435371 }, "0.8": { "coverage": 0.84375, "accuracy": 0.9259259259259259, "confidence_cutoff": 0.934568040560424 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 40, "accuracy": 1.0, "full_probability_metrics": { "n": 40, "nll": 0.008969445416060125, "acc": 1.0, "ece": 0.008926449902319056, "brier": 0.00017080060193278867, "mean_conf": 0.9910735500976809, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.008926449902319056, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 33, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9912844970172978 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9903233743032935 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.008969445416060125, "acc": 1.0, "ece": 0.008926449902319056, "brier": 0.00017080060193278867, "mean_conf": 0.9910735500976809, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.008926449902319056, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 33, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9912844970172978 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9903233743032935 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 35, "accuracy": 0.875, "full_probability_metrics": { "n": 40, "nll": 0.48724767585244866, "acc": 0.875, "ece": 0.1684988546735438, "brier": 0.25020945043990783, "mean_conf": 0.8603903077286162, "confident_error_rate": 0.075, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8846153846153846, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.225, "aurc": 0.07716789950736613, "error_rate_at_0_9": 0.11538461538461539, "confidence_bias": -0.014609692271383756, "top_bins": { "0.9": { "n": 26, "errors": 3, "error_rate": 0.11538461538461539 }, "0.95": { "n": 21, "errors": 2, "error_rate": 0.09523809523809523 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9530614596734541 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7113426393760265 } }, "score_mae": 0.2585023571663795, "ranked_probability_score": 0.07135054511501865 }, "covered_only_probability_metrics": { "n": 40, "nll": 0.48724767585244866, "acc": 0.875, "ece": 0.1684988546735438, "brier": 0.25020945043990783, "mean_conf": 0.8603903077286162, "confident_error_rate": 0.075, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8846153846153846, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.225, "aurc": 0.07716789950736613, "error_rate_at_0_9": 0.11538461538461539, "confidence_bias": -0.014609692271383756, "top_bins": { "0.9": { "n": 26, "errors": 3, "error_rate": 0.11538461538461539 }, "0.95": { "n": 21, "errors": 2, "error_rate": 0.09523809523809523 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9530614596734541 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7113426393760265 } }, "score_mae": 0.2585023571663795, "ranked_probability_score": 0.07135054511501865 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 54, "accuracy": 0.675, "full_probability_metrics": { "n": 80, "nll": 1.0358750495293694, "acc": 0.675, "ece": 0.16340709671926676, "brier": 0.489098687652309, "mean_conf": 0.5466186040709095, "confident_error_rate": 0.0, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18662413706260517, "error_rate_at_0_9": 0.0, "confidence_bias": -0.1283813959290906, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.54093244915798 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.3296945491342112 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.0358750495293694, "acc": 0.675, "ece": 0.16340709671926676, "brier": 0.489098687652309, "mean_conf": 0.5466186040709095, "confident_error_rate": 0.0, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18662413706260517, "error_rate_at_0_9": 0.0, "confidence_bias": -0.1283813959290906, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.54093244915798 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.3296945491342112 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 60, "accuracy": 0.75, "full_probability_metrics": { "n": 80, "nll": 0.6861770583254359, "acc": 0.75, "ece": 0.1217068504481862, "brier": 0.347020698808706, "mean_conf": 0.6895968895124733, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 0.9545454545454546, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.1625, "aurc": 0.0857126877554564, "error_rate_at_0_9": 0.045454545454545456, "confidence_bias": -0.06040311048752667, "top_bins": { "0.9": { "n": 22, "errors": 1, "error_rate": 0.045454545454545456 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.7578185059536526 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.4477054772574507 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.6861770583254359, "acc": 0.75, "ece": 0.1217068504481862, "brier": 0.347020698808706, "mean_conf": 0.6895968895124733, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 0.9545454545454546, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.1625, "aurc": 0.0857126877554564, "error_rate_at_0_9": 0.045454545454545456, "confidence_bias": -0.06040311048752667, "top_bins": { "0.9": { "n": 22, "errors": 1, "error_rate": 0.045454545454545456 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.7578185059536526 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.4477054772574507 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 106, "accuracy": 0.53, "full_probability_metrics": { "n": 200, "nll": 1.4591827161479933, "acc": 0.53, "ece": 0.12623940287807733, "brier": 0.6295569349100498, "mean_conf": 0.46737060291822385, "confident_error_rate": 0.0, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.23, "coverage_at_1pct_error": 0.215, "aurc": 0.26761545126804087, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06262939708177617, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.66, "confidence_cutoff": 0.4061193826617392 }, "0.8": { "coverage": 0.8, "accuracy": 0.5875, "confidence_cutoff": 0.24139014837018813 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 1.4591827161479933, "acc": 0.53, "ece": 0.12623940287807733, "brier": 0.6295569349100498, "mean_conf": 0.46737060291822385, "confident_error_rate": 0.0, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.23, "coverage_at_1pct_error": 0.215, "aurc": 0.26761545126804087, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06262939708177617, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.66, "confidence_cutoff": 0.4061193826617392 }, "0.8": { "coverage": 0.8, "accuracy": 0.5875, "confidence_cutoff": 0.24139014837018813 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 65, "accuracy": 0.8125, "full_probability_metrics": { "n": 80, "nll": 0.45012460945133626, "acc": 0.8125, "ece": 0.10990939739597044, "brier": 0.2687832669143296, "mean_conf": 0.9206235941007703, "confident_error_rate": 0.075, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9047619047619048, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.3875, "aurc": 0.05975568138683405, "error_rate_at_0_9": 0.09523809523809523, "confidence_bias": 0.10812359410077033, "top_bins": { "0.9": { "n": 63, "errors": 6, "error_rate": 0.09523809523809523 }, "0.95": { "n": 54, "errors": 5, "error_rate": 0.09259259259259259 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9730505331188758 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.8913617994128987 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.45012460945133626, "acc": 0.8125, "ece": 0.10990939739597044, "brier": 0.2687832669143296, "mean_conf": 0.9206235941007703, "confident_error_rate": 0.075, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9047619047619048, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.3875, "aurc": 0.05975568138683405, "error_rate_at_0_9": 0.09523809523809523, "confidence_bias": 0.10812359410077033, "top_bins": { "0.9": { "n": 63, "errors": 6, "error_rate": 0.09523809523809523 }, "0.95": { "n": 54, "errors": 5, "error_rate": 0.09259259259259259 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9730505331188758 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.8913617994128987 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 76, "accuracy": 0.95, "full_probability_metrics": { "n": 80, "nll": 0.13767493715682097, "acc": 0.95, "ece": 0.04288842138989113, "brier": 0.07799499282917079, "mean_conf": 0.926202802533366, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.85, "aurc": 0.004593511081296997, "error_rate_at_0_9": 0.0, "confidence_bias": -0.023797197466634, "top_bins": { "0.9": { "n": 64, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 52, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9693666425507508 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9018282630230584 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.13767493715682097, "acc": 0.95, "ece": 0.04288842138989113, "brier": 0.07799499282917079, "mean_conf": 0.926202802533366, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.85, "aurc": 0.004593511081296997, "error_rate_at_0_9": 0.0, "confidence_bias": -0.023797197466634, "top_bins": { "0.9": { "n": 64, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 52, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9693666425507508 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9018282630230584 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 80, "accuracy": 1.0, "full_probability_metrics": { "n": 80, "nll": 0.044030358287843394, "acc": 1.0, "ece": 0.040828241440827244, "brier": 0.009274412683663524, "mean_conf": 0.9591717585591726, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04082824144082742, "top_bins": { "0.9": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 62, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9799394210545673 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9393920038797857 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.044030358287843394, "acc": 1.0, "ece": 0.040828241440827244, "brier": 0.009274412683663524, "mean_conf": 0.9591717585591726, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04082824144082742, "top_bins": { "0.9": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 62, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9799394210545673 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9393920038797857 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 69, "accuracy": 0.8625, "full_probability_metrics": { "n": 80, "nll": 0.33117936803701575, "acc": 0.8625, "ece": 0.05436399874561858, "brier": 0.19776349471623236, "mean_conf": 0.8258637096555989, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 0.9696969696969697, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.275, "aurc": 0.0435435591607148, "error_rate_at_0_9": 0.030303030303030304, "confidence_bias": -0.03663629034440119, "top_bins": { "0.9": { "n": 33, "errors": 1, "error_rate": 0.030303030303030304 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.8626082025403856 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.7113036685404975 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.33117936803701575, "acc": 0.8625, "ece": 0.05436399874561858, "brier": 0.19776349471623236, "mean_conf": 0.8258637096555989, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 0.9696969696969697, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.275, "aurc": 0.0435435591607148, "error_rate_at_0_9": 0.030303030303030304, "confidence_bias": -0.03663629034440119, "top_bins": { "0.9": { "n": 33, "errors": 1, "error_rate": 0.030303030303030304 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.8626082025403856 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.7113036685404975 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.007953143720346688, "acc": 1.0, "ece": 0.007920164821361153, "brier": 0.0001311540982699086, "mean_conf": 0.9920798351786388, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007920164821361153, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9932902696475102 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9906916446721327 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.007953143720346688, "acc": 1.0, "ece": 0.007920164821361153, "brier": 0.0001311540982699086, "mean_conf": 0.9920798351786388, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007920164821361153, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9932902696475102 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9906916446721327 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.00916049398305022, "acc": 1.0, "ece": 0.009115478091194618, "brier": 0.0001787908571993623, "mean_conf": 0.9908845219088054, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009115478091194618, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9918931399320625 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9901469063995834 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.00916049398305022, "acc": 1.0, "ece": 0.009115478091194618, "brier": 0.0001787908571993623, "mean_conf": 0.9908845219088054, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009115478091194618, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9918931399320625 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9901469063995834 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.7024896484857039, "acc": 0.7, "ece": 0.17400345592562788, "brier": 0.3785387189770012, "mean_conf": 0.7679596335799816, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.18650793650793648, "error_rate_at_0_9": 0.2, "confidence_bias": 0.0679596335799817, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9067403044300515 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5076514037743629 } }, "score_mae": 0.36186168608722724, "ranked_probability_score": 0.1065096872204793 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7024896484857039, "acc": 0.7, "ece": 0.17400345592562788, "brier": 0.3785387189770012, "mean_conf": 0.7679596335799816, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.18650793650793648, "error_rate_at_0_9": 0.2, "confidence_bias": 0.0679596335799817, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9067403044300515 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5076514037743629 } }, "score_mae": 0.36186168608722724, "ranked_probability_score": 0.1065096872204793 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.024287043856752667, "acc": 1.0, "ece": 0.023175149207907343, "brier": 0.0031940845407984683, "mean_conf": 0.9768248507920927, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02317514920790731, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9886598232543066 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9857301340232879 } }, "score_mae": 0.014873371840116872, "ranked_probability_score": 0.0006047375405274235 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.024287043856752667, "acc": 1.0, "ece": 0.023175149207907343, "brier": 0.0031940845407984683, "mean_conf": 0.9768248507920927, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02317514920790731, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9886598232543066 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9857301340232879 } }, "score_mae": 0.014873371840116872, "ranked_probability_score": 0.0006047375405274235 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.020701088110845985, "acc": 1.0, "ece": 0.02045118600935203, "brier": 0.0007459655558159184, "mean_conf": 0.979548813990648, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02045118600935203, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9819328792463708 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9754537565548328 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.020701088110845985, "acc": 1.0, "ece": 0.02045118600935203, "brier": 0.0007459655558159184, "mean_conf": 0.979548813990648, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02045118600935203, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9819328792463708 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9754537565548328 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.4513697339011308, "acc": 0.7, "ece": 0.18409695581172933, "brier": 0.32078285339100915, "mean_conf": 0.8399471880958155, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.0790079365079365, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13994718809581552, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9500897945819461 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5924374704100209 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.4513697339011308, "acc": 0.7, "ece": 0.18409695581172933, "brier": 0.32078285339100915, "mean_conf": 0.8399471880958155, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.0790079365079365, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13994718809581552, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9500897945819461 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5924374704100209 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.2376608685283057, "acc": 0.9, "ece": 0.13385397331210808, "brier": 0.14308312468801188, "mean_conf": 0.9183269867497545, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": 0.0, "confidence_bias": 0.018326986749754526, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9726140002708453 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9332098203343255 } }, "score_mae": 0.13395495166501625, "ranked_probability_score": 0.034552147729470475 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.2376608685283057, "acc": 0.9, "ece": 0.13385397331210808, "brier": 0.14308312468801188, "mean_conf": 0.9183269867497545, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": 0.0, "confidence_bias": 0.018326986749754526, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9726140002708453 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9332098203343255 } }, "score_mae": 0.13395495166501625, "ranked_probability_score": 0.034552147729470475 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.013168227347695117, "acc": 1.0, "ece": 0.01305669366324802, "brier": 0.00033543059842657696, "mean_conf": 0.986943306336752, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01305669366324802, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.990669763308131 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9805133070873305 } }, "score_mae": 0.009074522139395101, "ranked_probability_score": 8.093111202506926e-05 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.013168227347695117, "acc": 1.0, "ece": 0.01305669366324802, "brier": 0.00033543059842657696, "mean_conf": 0.986943306336752, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01305669366324802, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.990669763308131 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9805133070873305 } }, "score_mae": 0.009074522139395101, "ranked_probability_score": 8.093111202506926e-05 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.009421911521444424, "acc": 1.0, "ece": 0.009375639073634923, "brier": 0.0001399225658629515, "mean_conf": 0.9906243609263651, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009375639073634923, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9902797336735392 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9892708284244871 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.009421911521444424, "acc": 1.0, "ece": 0.009375639073634923, "brier": 0.0001399225658629515, "mean_conf": 0.9906243609263651, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009375639073634923, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9902797336735392 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9892708284244871 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.007415668719870068, "acc": 1.0, "ece": 0.007387384683748666, "brier": 8.48199771890235e-05, "mean_conf": 0.9926126153162513, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007387384683748666, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9933185676840829 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9914980720036761 } }, "score_mae": 0.009117160051502043, "ranked_probability_score": 3.107045891868192e-05 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.007415668719870068, "acc": 1.0, "ece": 0.007387384683748666, "brier": 8.48199771890235e-05, "mean_conf": 0.9926126153162513, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007387384683748666, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9933185676840829 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9914980720036761 } }, "score_mae": 0.009117160051502043, "ranked_probability_score": 3.107045891868192e-05 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.3538188535306609, "acc": 0.8, "ece": 0.2529433155026392, "brier": 0.1992482492146836, "mean_conf": 0.7729666232834627, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027033376716537383, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.861579127956784 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.6303436207051845 } }, "score_mae": 0.2503358486078282, "ranked_probability_score": 0.05192111353863584 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.3538188535306609, "acc": 0.8, "ece": 0.2529433155026392, "brier": 0.1992482492146836, "mean_conf": 0.7729666232834627, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027033376716537383, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.861579127956784 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.6303436207051845 } }, "score_mae": 0.2503358486078282, "ranked_probability_score": 0.05192111353863584 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.8601388888888889, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.8055555555555556, "flip_rate": 0.027777777777777776, "covered_only_flip_rate": 0.027777777777777776, "half_l1": 0.06150700612994839 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.39611662801464104, "share_at_0_9": 0.0, "normalized_entropy": 0.9981201857275759 }, "covered_only_confidence_metrics": { "mean_max_p": 0.39611662801464104, "share_at_0_9": 0.0, "normalized_entropy": 0.9981201857275759 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 101, "accuracy": 0.9181818181818182, "full_probability_metrics": { "n": 110, "nll": 0.1670406074278006, "acc": 0.9181818181818182, "ece": 0.040263403454622974, "brier": 0.09513301040584257, "mean_conf": 0.9280653396507789, "confident_error_rate": 0.00909090909090909, "coverage_at_0_9": 0.8181818181818182, "accuracy_at_0_9": 0.9888888888888889, "coverage_at_5pct_error": 0.9545454545454546, "coverage_at_1pct_error": 0.7181818181818181, "aurc": 0.007938948084780633, "error_rate_at_0_9": 0.011111111111111112, "confidence_bias": 0.009883521468960654, "top_bins": { "0.9": { "n": 90, "errors": 1, "error_rate": 0.011111111111111112 }, "0.95": { "n": 84, "errors": 1, "error_rate": 0.011904761904761904 }, "0.99": { "n": 41, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9867862470873447 }, "0.8": { "coverage": 0.8, "accuracy": 0.9886363636363636, "confidence_cutoff": 0.9257932753686331 } }, "score_mae": 0.12986959006518095, "ranked_probability_score": 0.032283281266676137 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.1670406074278006, "acc": 0.9181818181818182, "ece": 0.040263403454622974, "brier": 0.09513301040584257, "mean_conf": 0.9280653396507789, "confident_error_rate": 0.00909090909090909, "coverage_at_0_9": 0.8181818181818182, "accuracy_at_0_9": 0.9888888888888889, "coverage_at_5pct_error": 0.9545454545454546, "coverage_at_1pct_error": 0.7181818181818181, "aurc": 0.007938948084780633, "error_rate_at_0_9": 0.011111111111111112, "confidence_bias": 0.009883521468960654, "top_bins": { "0.9": { "n": 90, "errors": 1, "error_rate": 0.011111111111111112 }, "0.95": { "n": 84, "errors": 1, "error_rate": 0.011904761904761904 }, "0.99": { "n": 41, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9867862470873447 }, "0.8": { "coverage": 0.8, "accuracy": 0.9886363636363636, "confidence_cutoff": 0.9257932753686331 } }, "score_mae": 0.12986959006518095, "ranked_probability_score": 0.032283281266676137 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.5319487116361378 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.37154013112618367, "paired_flip": { "pairs": 64, "flip_rate": 0.890625, "both_correct_rate": 0.8125, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.7916666666666666 }, "unknowable": { "n": 110, "mean_max_p": 0.39611662801464104, "share_at_0_9": 0.0, "control_mean_max_p": 0.9280653396507789, "control_share_at_0_9": 0.8181818181818182, "control_acc": 0.9181818181818182, "paired_confidence_drop": 0.5319487116361378, "share_less_confident_than_control": 1.0 }, "clean": { "n": 1046, "nll": 0.6027548590263215, "acc": 0.7925430210325047, "ece": 0.044502507431275706, "brier": 0.294775757633525, "mean_conf": 0.7761167229324795, "confident_error_rate": 0.0248565965583174, "coverage_at_0_9": 0.5296367112810707, "accuracy_at_0_9": 0.9530685920577617, "coverage_at_5pct_error": 0.5803059273422562, "coverage_at_1pct_error": 0.29732313575525815, "aurc": 0.05863375703305322, "error_rate_at_0_9": 0.04693140794223827, "confidence_bias": -0.01642629810002527, "top_bins": { "0.9": { "n": 554, "errors": 26, "error_rate": 0.04693140794223827 }, "0.95": { "n": 437, "errors": 10, "error_rate": 0.02288329519450801 }, "0.99": { "n": 119, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9521988527724665, "confidence_cutoff": 0.9187699097211646 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8864994026284349, "confidence_cutoff": 0.5194740121242921 } }, "score_mae": 0.18132269690566039, "ranked_probability_score": 0.04791018680601314 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.0527257481257462, "acc": 0.65, "ece": 0.3051233916894593, "brier": 0.5023087907915027, "mean_conf": 0.4819157184465988, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.18057358555617067, "error_rate_at_0_9": null, "confidence_bias": -0.16808428155340122, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.4279851929395839 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.2728929499321211 } } }, "buried_paws": { "n": 20, "nll": 0.6833261160573135, "acc": 0.8, "ece": 0.16396340710521495, "brier": 0.371614784794417, "mean_conf": 0.963963407105215, "confident_error_rate": 0.2, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.11906431782012276, "error_rate_at_0_9": 0.2, "confidence_bias": 0.16396340710521495, "top_bins": { "0.9": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.95": { "n": 14, "errors": 2, "error_rate": 0.14285714285714285 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9716955314759664 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.9488051361801536 } } }, "buried_qnli": { "n": 20, "nll": 0.148984242944029, "acc": 0.95, "ece": 0.07618705591020539, "brier": 0.08005192124027319, "mean_conf": 0.9056351007639526, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9, "aurc": 0.005131578947368421, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04436489923604736, "top_bins": { "0.9": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9484042101321722 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8695504794947875 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.4566841691053221, "acc": 0.8, "ece": 0.11810294600060224, "brier": 0.29841595844660157, "mean_conf": 0.7647957278678895, "confident_error_rate": 0.0, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.35, "aurc": 0.101723631545613, "error_rate_at_0_9": 0.0, "confidence_bias": -0.035204272132110526, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7540354245012317 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.666028181730861 } } }, "composition_final_combination": { "n": 32, "nll": 0.5866928427487809, "acc": 0.6875, "ece": 0.2387653374086254, "brier": 0.41343064573118143, "mean_conf": 0.9262653374086254, "confident_error_rate": 0.125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.6875, "aurc": 0.06261277252248122, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.23876533740862538, "top_bins": { "0.9": { "n": 26, "errors": 4, "error_rate": 0.15384615384615385 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9836031674848541 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8461538461538461, "confidence_cutoff": 0.9233322143554688 } } }, "composition_final_exception": { "n": 32, "nll": 0.30002121819543187, "acc": 0.875, "ece": 0.09059456263848523, "brier": 0.19589500998782167, "mean_conf": 0.9655945626384852, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.96875, "accuracy_at_0_9": 0.9032258064516129, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.875, "aurc": 0.011097110215053763, "error_rate_at_0_9": 0.0967741935483871, "confidence_bias": 0.0905945626384852, "top_bins": { "0.9": { "n": 31, "errors": 3, "error_rate": 0.0967741935483871 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9773114432839093 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9668880032979845 } } }, "composition_final_negation": { "n": 32, "nll": 0.32524130332020473, "acc": 0.90625, "ece": 0.10976979965216396, "brier": 0.1938474981101241, "mean_conf": 0.9339476677914791, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.8928571428571429, "coverage_at_5pct_error": 0.78125, "coverage_at_1pct_error": 0.78125, "aurc": 0.020194977536033568, "error_rate_at_0_9": 0.10714285714285714, "confidence_bias": 0.027697667791479086, "top_bins": { "0.9": { "n": 28, "errors": 3, "error_rate": 0.10714285714285714 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.972606823435371 }, "0.8": { "coverage": 0.84375, "accuracy": 0.9259259259259259, "confidence_cutoff": 0.934568040560424 } } }, "contrastive_authorization": { "n": 40, "nll": 0.008969445416060125, "acc": 1.0, "ece": 0.008926449902319056, "brier": 0.00017080060193278867, "mean_conf": 0.9910735500976809, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.008926449902319056, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 33, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9912844970172978 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9903233743032935 } } }, "contrastive_deadline": { "n": 40, "nll": 0.48724767585244866, "acc": 0.875, "ece": 0.1684988546735438, "brier": 0.25020945043990783, "mean_conf": 0.8603903077286162, "confident_error_rate": 0.075, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8846153846153846, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.225, "aurc": 0.07716789950736613, "error_rate_at_0_9": 0.11538461538461539, "confidence_bias": -0.014609692271383756, "top_bins": { "0.9": { "n": 26, "errors": 3, "error_rate": 0.11538461538461539 }, "0.95": { "n": 21, "errors": 2, "error_rate": 0.09523809523809523 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9530614596734541 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7113426393760265 } }, "score_mae": 0.2585023571663795, "ranked_probability_score": 0.07135054511501865 }, "emotion": { "n": 80, "nll": 1.0358750495293694, "acc": 0.675, "ece": 0.16340709671926676, "brier": 0.489098687652309, "mean_conf": 0.5466186040709095, "confident_error_rate": 0.0, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18662413706260517, "error_rate_at_0_9": 0.0, "confidence_bias": -0.1283813959290906, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.54093244915798 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.3296945491342112 } } }, "mmlu": { "n": 80, "nll": 0.6861770583254359, "acc": 0.75, "ece": 0.1217068504481862, "brier": 0.347020698808706, "mean_conf": 0.6895968895124733, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 0.9545454545454546, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.1625, "aurc": 0.0857126877554564, "error_rate_at_0_9": 0.045454545454545456, "confidence_bias": -0.06040311048752667, "top_bins": { "0.9": { "n": 22, "errors": 1, "error_rate": 0.045454545454545456 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.7578185059536526 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.4477054772574507 } } }, "mmlu_pro": { "n": 200, "nll": 1.4591827161479933, "acc": 0.53, "ece": 0.12623940287807733, "brier": 0.6295569349100498, "mean_conf": 0.46737060291822385, "confident_error_rate": 0.0, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.23, "coverage_at_1pct_error": 0.215, "aurc": 0.26761545126804087, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06262939708177617, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.66, "confidence_cutoff": 0.4061193826617392 }, "0.8": { "coverage": 0.8, "accuracy": 0.5875, "confidence_cutoff": 0.24139014837018813 } } }, "paws": { "n": 80, "nll": 0.45012460945133626, "acc": 0.8125, "ece": 0.10990939739597044, "brier": 0.2687832669143296, "mean_conf": 0.9206235941007703, "confident_error_rate": 0.075, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9047619047619048, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.3875, "aurc": 0.05975568138683405, "error_rate_at_0_9": 0.09523809523809523, "confidence_bias": 0.10812359410077033, "top_bins": { "0.9": { "n": 63, "errors": 6, "error_rate": 0.09523809523809523 }, "0.95": { "n": 54, "errors": 5, "error_rate": 0.09259259259259259 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9730505331188758 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.8913617994128987 } } }, "qnli": { "n": 80, "nll": 0.13767493715682097, "acc": 0.95, "ece": 0.04288842138989113, "brier": 0.07799499282917079, "mean_conf": 0.926202802533366, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.85, "aurc": 0.004593511081296997, "error_rate_at_0_9": 0.0, "confidence_bias": -0.023797197466634, "top_bins": { "0.9": { "n": 64, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 52, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9693666425507508 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9018282630230584 } } }, "sciq": { "n": 80, "nll": 0.044030358287843394, "acc": 1.0, "ece": 0.040828241440827244, "brier": 0.009274412683663524, "mean_conf": 0.9591717585591726, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04082824144082742, "top_bins": { "0.9": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 62, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9799394210545673 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9393920038797857 } } }, "tweet_offensive": { "n": 80, "nll": 0.33117936803701575, "acc": 0.8625, "ece": 0.05436399874561858, "brier": 0.19776349471623236, "mean_conf": 0.8258637096555989, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 0.9696969696969697, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.275, "aurc": 0.0435435591607148, "error_rate_at_0_9": 0.030303030303030304, "confidence_bias": -0.03663629034440119, "top_bins": { "0.9": { "n": 33, "errors": 1, "error_rate": 0.030303030303030304 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.8626082025403856 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.7113036685404975 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.6955067280403423, "acc": 0.5, "ece": 0.029877868310791977, "brier": 0.5023504735950214, "mean_conf": 0.529877868310792, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.029877868310791977, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.5258282423019409 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.525545597076416 } } }, "unknowable_authorization": { "n": 10, "nll": 0.6689670185238051, "acc": 0.7, "ece": 0.14989575784128065, "brier": 0.4761430291744775, "mean_conf": 0.5501042421587192, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3370238095238095, "error_rate_at_0_9": null, "confidence_bias": -0.14989575784128073, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5434738993644714 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5347487330436707 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.007953143720346688, "acc": 1.0, "ece": 0.007920164821361153, "brier": 0.0001311540982699086, "mean_conf": 0.9920798351786388, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007920164821361153, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9932902696475102 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9906916446721327 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.00916049398305022, "acc": 1.0, "ece": 0.009115478091194618, "brier": 0.0001787908571993623, "mean_conf": 0.9908845219088054, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009115478091194618, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9918931399320625 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9901469063995834 } } }, "unknowable_control_deadline": { "n": 10, "nll": 0.7024896484857039, "acc": 0.7, "ece": 0.17400345592562788, "brier": 0.3785387189770012, "mean_conf": 0.7679596335799816, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.18650793650793648, "error_rate_at_0_9": 0.2, "confidence_bias": 0.0679596335799817, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9067403044300515 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5076514037743629 } }, "score_mae": 0.36186168608722724, "ranked_probability_score": 0.1065096872204793 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.024287043856752667, "acc": 1.0, "ece": 0.023175149207907343, "brier": 0.0031940845407984683, "mean_conf": 0.9768248507920927, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02317514920790731, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9886598232543066 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9857301340232879 } }, "score_mae": 0.014873371840116872, "ranked_probability_score": 0.0006047375405274235 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.020701088110845985, "acc": 1.0, "ece": 0.02045118600935203, "brier": 0.0007459655558159184, "mean_conf": 0.979548813990648, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02045118600935203, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9819328792463708 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9754537565548328 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.4513697339011308, "acc": 0.7, "ece": 0.18409695581172933, "brier": 0.32078285339100915, "mean_conf": 0.8399471880958155, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.0790079365079365, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13994718809581552, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9500897945819461 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5924374704100209 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.2376608685283057, "acc": 0.9, "ece": 0.13385397331210808, "brier": 0.14308312468801188, "mean_conf": 0.9183269867497545, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.021111111111111112, "error_rate_at_0_9": 0.0, "confidence_bias": 0.018326986749754526, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9726140002708453 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9332098203343255 } }, "score_mae": 0.13395495166501625, "ranked_probability_score": 0.034552147729470475 }, "unknowable_control_sla_response": { "n": 10, "nll": 0.013168227347695117, "acc": 1.0, "ece": 0.01305669366324802, "brier": 0.00033543059842657696, "mean_conf": 0.986943306336752, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01305669366324802, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.990669763308131 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9805133070873305 } }, "score_mae": 0.009074522139395101, "ranked_probability_score": 8.093111202506926e-05 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.009421911521444424, "acc": 1.0, "ece": 0.009375639073634923, "brier": 0.0001399225658629515, "mean_conf": 0.9906243609263651, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009375639073634923, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9902797336735392 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9892708284244871 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.007415668719870068, "acc": 1.0, "ece": 0.007387384683748666, "brier": 8.48199771890235e-05, "mean_conf": 0.9926126153162513, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007387384683748666, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9933185676840829 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9914980720036761 } }, "score_mae": 0.009117160051502043, "ranked_probability_score": 3.107045891868192e-05 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.3538188535306609, "acc": 0.8, "ece": 0.2529433155026392, "brier": 0.1992482492146836, "mean_conf": 0.7729666232834627, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027033376716537383, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.861579127956784 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.6303436207051845 } }, "score_mae": 0.2503358486078282, "ranked_probability_score": 0.05192111353863584 }, "unknowable_deadline": { "n": 10, "nll": 1.0839526191432045, "acc": 0.5, "ece": 0.14876459838577705, "brier": 0.6569700509478682, "mean_conf": 0.35123540161422295, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.46158730158730155, "error_rate_at_0_9": null, "confidence_bias": -0.14876459838577705, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.3517669439315796 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3487052619457245 } }, "score_mae": 0.6933661192584812, "ranked_probability_score": 0.22234902130598524 }, "unknowable_late_fee": { "n": 10, "nll": 1.0994462290702525, "acc": 0.3, "ece": 0.04611502093152603, "brier": 0.6671563545934535, "mean_conf": 0.346115020931526, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5650000000000001, "error_rate_at_0_9": null, "confidence_bias": 0.04611502093152603, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.34381222818603174 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3421555161476135 } }, "score_mae": 0.6025732934744976, "ranked_probability_score": 0.21061688921073612 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.1209844830291982, "acc": 0.3, "ece": 0.05200887956882155, "brier": 0.6810367441354941, "mean_conf": 0.35200887956882154, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6233333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.05200887956882155, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.35450633493103595 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.35305941104888916 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7016430391720105, "acc": 0.3, "ece": 0.209163076979136, "brier": 0.5084854155401464, "mean_conf": 0.509163076979136, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7246428571428571, "error_rate_at_0_9": null, "confidence_bias": 0.209163076979136, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5071807352548618 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5012782216072083 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.106494665503828, "acc": 0.2, "ece": 0.1452527433583194, "brier": 0.6719672024580847, "mean_conf": 0.3452527433583194, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7734920634920635, "error_rate_at_0_9": null, "confidence_bias": 0.1452527433583194, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.34633246064186096 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3425494032338439 } }, "score_mae": 0.7124151795369602, "ranked_probability_score": 0.23031017392822192 }, "unknowable_sla_response": { "n": 10, "nll": 1.0950565808380595, "acc": 0.5, "ece": 0.15060654150368558, "brier": 0.6640765470100547, "mean_conf": 0.3493934584963144, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": -0.15060654150368558, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.35209455036835224 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3462310628226083 } }, "score_mae": 0.5190795531566852, "ranked_probability_score": 0.1955841654354815 }, "unknowable_spend_threshold": { "n": 10, "nll": 1.0808490129113162, "acc": 0.5, "ece": 0.1582491218607846, "brier": 0.6549524864389306, "mean_conf": 0.3417508781392154, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": -0.1582491218607846, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.342225442595048 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33939048647880554 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.09538972843237, "acc": 0.3, "ece": 0.03939684026033935, "brier": 0.6645374999431917, "mean_conf": 0.33939684026033934, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6233333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.03939684026033935, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.33896157145500183 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.33875938240410736 } }, "score_mae": 0.8, "ranked_probability_score": 0.24348741996187467 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.0940072410377701, "acc": 0.5, "ece": 0.15701550165635397, "brier": 0.6636906024729206, "mean_conf": 0.342984498343646, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5384126984126983, "error_rate_at_0_9": null, "confidence_bias": -0.15701550165635397, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.34084099531173706 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33769776410765157 } }, "score_mae": 0.5018685075488395, "ranked_probability_score": 0.19362353046796416 } }, "variants": { "clean": { "n": 1156, "nll": 0.6391907923862923, "acc": 0.7569204152249135, "ece": 0.04236746859126635, "brier": 0.32564801604477817, "mean_conf": 0.7399575443503323, "confident_error_rate": 0.02249134948096886, "coverage_at_0_9": 0.47923875432525953, "accuracy_at_0_9": 0.9530685920577617, "coverage_at_5pct_error": 0.5250865051903114, "coverage_at_1pct_error": 0.2690311418685121, "aurc": 0.07403444878978717, "error_rate_at_0_9": 0.04693140794223827, "confidence_bias": -0.01696287087458115, "top_bins": { "0.9": { "n": 554, "errors": 26, "error_rate": 0.04693140794223827 }, "0.95": { "n": 437, "errors": 10, "error_rate": 0.02288329519450801 }, "0.99": { "n": 119, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9532871972318339, "confidence_cutoff": 0.886536639597194 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.8432432432432433, "confidence_cutoff": 0.417062088211548 } }, "score_mae": 0.3526581013770042, "ranked_probability_score": 0.11094206677314969 }, "none_absent": { "n": 36, "nll": 0.9034340304805742, "acc": 0.7222222222222222, "ece": 0.2118736635019033, "brier": 0.46550660911781794, "mean_conf": 0.591709442911497, "confident_error_rate": 0.027777777777777776, "coverage_at_0_9": 0.027777777777777776, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.2767347201233771, "error_rate_at_0_9": 1.0, "confidence_bias": -0.13051277931072525, "top_bins": { "0.9": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7777777777777778, "confidence_cutoff": 0.5841955259645661 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7586206896551724, "confidence_cutoff": 0.448009729146305 } } }, "none_present": { "n": 36, "nll": 0.8169456340423473, "acc": 0.7222222222222222, "ece": 0.1391879778669812, "brier": 0.37826277110654566, "mean_conf": 0.6744604564707586, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3611111111111111, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5833333333333334, "coverage_at_1pct_error": 0.5555555555555556, "aurc": 0.09292503597745425, "error_rate_at_0_9": 0.0, "confidence_bias": -0.047761765751463625, "top_bins": { "0.9": { "n": 13, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.6926343544220083 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7586206896551724, "confidence_cutoff": 0.39649764589587555 } } }, "permuted": { "n": 36, "nll": 0.6133940215041882, "acc": 0.8055555555555556, "ece": 0.14809608224812643, "brier": 0.28852756310317884, "mean_conf": 0.717483898589816, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3611111111111111, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6111111111111112, "coverage_at_1pct_error": 0.5833333333333334, "aurc": 0.050074281815927595, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0880716569657396, "top_bins": { "0.9": { "n": 13, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7403727800486908 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.896551724137931, "confidence_cutoff": 0.4945882113400635 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.05806784792778933, "flip_rate": 0.027777777777777776 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.6027548590263215, "acc": 0.7925430210325047, "ece": 0.044502507431275706, "brier": 0.294775757633525, "mean_conf": 0.7761167229324795, "confident_error_rate": 0.0248565965583174, "coverage_at_0_9": 0.5296367112810707, "accuracy_at_0_9": 0.9530685920577617, "coverage_at_5pct_error": 0.5803059273422562, "coverage_at_1pct_error": 0.29732313575525815, "aurc": 0.05863375703305322, "error_rate_at_0_9": 0.04693140794223827, "confidence_bias": -0.01642629810002527, "top_bins": { "0.9": { "n": 554, "errors": 26, "error_rate": 0.04693140794223827 }, "0.95": { "n": 437, "errors": 10, "error_rate": 0.02288329519450801 }, "0.99": { "n": 119, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9521988527724665, "confidence_cutoff": 0.9187699097211646 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8864994026284349, "confidence_cutoff": 0.5194740121242921 } }, "score_mae": 0.18132269690566039, "ranked_probability_score": 0.04791018680601314 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "kev-4b": { "panels": { "old_core": { "point": 0.719047619047619, "exact_fraction": "151/210", "ci95": [ 0.6805059523809524, 0.756547619047619 ] }, "v3_core": { "point": 0.48541666666666666, "exact_fraction": "233/480", "ci95": [ 0.45791666666666664, 0.5116666666666667 ] }, "v4": { "point": 0.81875, "exact_fraction": "131/160", "ci95": [ 0.7759307065217391, 0.8595646994134898 ] }, "v5": { "point": 0.8458333333333333, "exact_fraction": "203/240", "ci95": [ 0.8154801504313007, 0.8750837906504066 ] }, "transfer_v9_test": { "point": 0.7609942638623327, "exact_fraction": "398/523", "ci95": [ 0.7319867357345924, 0.7883735478309257 ] }, "original_four_panel_mean": { "point": 0.7172619047619048, "exact_fraction": "241/336", "ci95": [ 0.6995049548368345, 0.7340168058629347 ] }, "weighted_mean": { "point": 0.7009050919603023, "exact_fraction": "2463373/3514560", "ci95": [ 0.6845181833150294, 0.7163497835168438 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 112, "requested": 128 }, "boolean_constraints": { "correct": 46, "requested": 64 }, "dbpedia_14": { "correct": 110, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 40, "requested": 96 }, "ordinal_rubric": { "correct": 64, "requested": 64 }, "relational_composition": { "correct": 60, "requested": 96 }, "scoped_evidence": { "correct": 46, "requested": 96 }, "state_tracking": { "correct": 33, "requested": 96 }, "unknown_rejection": { "correct": 48, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 40, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 26, "requested": 80 }, "massive_en": { "correct": 105, "requested": 120 }, "massive_zh": { "correct": 104, "requested": 120 }, "multiset_reconciliation": { "correct": 23, "requested": 80 }, "ordinal_service_loss": { "correct": 16, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 27, "requested": 80 }, "temporal_exclusion": { "correct": 36, "requested": 80 }, "transaction_recovery": { "correct": 41, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 142, "requested": 160 }, "belebele_en": { "correct": 121, "requested": 160 }, "belebele_zh": { "correct": 119, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 85, "requested": 120 }, "squad2_answerability": { "correct": 98, "requested": 120 }, "snli": { "correct": 107, "requested": 120 }, "qasc": { "correct": 116, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 796, "accuracy": 0.7609942638623327, "full_probability_metrics": { "n": 1046, "nll": 0.64340205208523, "acc": 0.7609942638623327, "ece": 0.02607917633618737, "brier": 0.3145325747542575, "mean_conf": 0.776584127219792, "confident_error_rate": 0.015296367112810707, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9694072657743786, "coverage_at_5pct_error": 0.5678776290630975, "coverage_at_1pct_error": 0.2418738049713193, "aurc": 0.0659154401199933, "error_rate_at_0_9": 0.030592734225621414, "confidence_bias": 0.015589863357459266, "top_bins": { "0.9": { "n": 523, "errors": 16, "error_rate": 0.030592734225621414 }, "0.95": { "n": 428, "errors": 10, "error_rate": 0.02336448598130841 }, "0.99": { "n": 136, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9694072657743786, "confidence_cutoff": 0.900898774733367 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8685782556750299, "confidence_cutoff": 0.5414887070655823 } }, "score_mae": 0.18542259338155553, "ranked_probability_score": 0.05309659577015893 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.64340205208523, "acc": 0.7609942638623327, "ece": 0.02607917633618737, "brier": 0.3145325747542575, "mean_conf": 0.776584127219792, "confident_error_rate": 0.015296367112810707, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9694072657743786, "coverage_at_5pct_error": 0.5678776290630975, "coverage_at_1pct_error": 0.2418738049713193, "aurc": 0.0659154401199933, "error_rate_at_0_9": 0.030592734225621414, "confidence_bias": 0.015589863357459266, "top_bins": { "0.9": { "n": 523, "errors": 16, "error_rate": 0.030592734225621414 }, "0.95": { "n": 428, "errors": 10, "error_rate": 0.02336448598130841 }, "0.99": { "n": 136, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9694072657743786, "confidence_cutoff": 0.900898774733367 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8685782556750299, "confidence_cutoff": 0.5414887070655823 } }, "score_mae": 0.18542259338155553, "ranked_probability_score": 0.05309659577015893 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 11, "accuracy": 0.55, "full_probability_metrics": { "n": 20, "nll": 1.2001095097427736, "acc": 0.55, "ece": 0.15707177801291555, "brier": 0.5423712656116211, "mean_conf": 0.5839834204481658, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.22846596579375214, "error_rate_at_0_9": 0.0, "confidence_bias": 0.03398342044816571, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.5908694179226545 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.41954513748428623 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.2001095097427736, "acc": 0.55, "ece": 0.15707177801291555, "brier": 0.5423712656116211, "mean_conf": 0.5839834204481658, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.22846596579375214, "error_rate_at_0_9": 0.0, "confidence_bias": 0.03398342044816571, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.5908694179226545 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.41954513748428623 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 15, "accuracy": 0.75, "full_probability_metrics": { "n": 20, "nll": 0.7583959032791606, "acc": 0.75, "ece": 0.1962823095026524, "brier": 0.44253604470947644, "mean_conf": 0.894629996826019, "confident_error_rate": 0.15, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.161849374825381, "error_rate_at_0_9": 0.2, "confidence_bias": 0.14462999682601896, "top_bins": { "0.9": { "n": 15, "errors": 3, "error_rate": 0.2 }, "0.95": { "n": 13, "errors": 2, "error_rate": 0.15384615384615385 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9630329120498103 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8593514305546454 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.7583959032791606, "acc": 0.75, "ece": 0.1962823095026524, "brier": 0.44253604470947644, "mean_conf": 0.894629996826019, "confident_error_rate": 0.15, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.161849374825381, "error_rate_at_0_9": 0.2, "confidence_bias": 0.14462999682601896, "top_bins": { "0.9": { "n": 15, "errors": 3, "error_rate": 0.2 }, "0.95": { "n": 13, "errors": 2, "error_rate": 0.15384615384615385 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9630329120498103 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8593514305546454 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 18, "accuracy": 0.9, "full_probability_metrics": { "n": 20, "nll": 0.2951763106727876, "acc": 0.9, "ece": 0.10106320366043044, "brier": 0.18245509323688672, "mean_conf": 0.8800315839835061, "confident_error_rate": 0.05, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9230769230769231, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.032635805671796386, "error_rate_at_0_9": 0.07692307692307693, "confidence_bias": -0.019968416016493906, "top_bins": { "0.9": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9382989932723418 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.7824955712261455 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.2951763106727876, "acc": 0.9, "ece": 0.10106320366043044, "brier": 0.18245509323688672, "mean_conf": 0.8800315839835061, "confident_error_rate": 0.05, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9230769230769231, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.032635805671796386, "error_rate_at_0_9": 0.07692307692307693, "confidence_bias": -0.019968416016493906, "top_bins": { "0.9": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9382989932723418 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.7824955712261455 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 13, "accuracy": 0.65, "full_probability_metrics": { "n": 20, "nll": 0.5691632942022599, "acc": 0.65, "ece": 0.09037131558307589, "brier": 0.39247233447016044, "mean_conf": 0.6960255894462387, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.1926055119630971, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04602558944623869, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7124768495559692 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5802226066589355 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5691632942022599, "acc": 0.65, "ece": 0.09037131558307589, "brier": 0.39247233447016044, "mean_conf": 0.6960255894462387, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.1926055119630971, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04602558944623869, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7124768495559692 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5802226066589355 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 25, "accuracy": 0.78125, "full_probability_metrics": { "n": 32, "nll": 0.5780628525520849, "acc": 0.78125, "ece": 0.18448002890132353, "brier": 0.35513312120830176, "mean_conf": 0.9436765872113124, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.78125, "accuracy_at_0_9": 0.88, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.59375, "aurc": 0.05964396990758747, "error_rate_at_0_9": 0.12, "confidence_bias": 0.16242658721131242, "top_bins": { "0.9": { "n": 25, "errors": 3, "error_rate": 0.12 }, "0.95": { "n": 25, "errors": 3, "error_rate": 0.12 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9751177862732483 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8846153846153846, "confidence_cutoff": 0.8956495523452759 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.5780628525520849, "acc": 0.78125, "ece": 0.18448002890132353, "brier": 0.35513312120830176, "mean_conf": 0.9436765872113124, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.78125, "accuracy_at_0_9": 0.88, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.59375, "aurc": 0.05964396990758747, "error_rate_at_0_9": 0.12, "confidence_bias": 0.16242658721131242, "top_bins": { "0.9": { "n": 25, "errors": 3, "error_rate": 0.12 }, "0.95": { "n": 25, "errors": 3, "error_rate": 0.12 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9751177862732483 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8846153846153846, "confidence_cutoff": 0.8956495523452759 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 31, "accuracy": 0.96875, "full_probability_metrics": { "n": 32, "nll": 0.1291452177610849, "acc": 0.96875, "ece": 0.03135938784471088, "brier": 0.06433444963615564, "mean_conf": 0.9620390965967792, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 0.9655172413793104, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004072377873563219, "error_rate_at_0_9": 0.034482758620689655, "confidence_bias": -0.00671090340322078, "top_bins": { "0.9": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9812241902339182 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9633261726799828 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.1291452177610849, "acc": 0.96875, "ece": 0.03135938784471088, "brier": 0.06433444963615564, "mean_conf": 0.9620390965967792, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 0.9655172413793104, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004072377873563219, "error_rate_at_0_9": 0.034482758620689655, "confidence_bias": -0.00671090340322078, "top_bins": { "0.9": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9812241902339182 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9633261726799828 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 28, "accuracy": 0.875, "full_probability_metrics": { "n": 32, "nll": 0.17987896070468248, "acc": 0.875, "ece": 0.10201081131864481, "brier": 0.12239407244814024, "mean_conf": 0.9252330939193277, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.84375, "aurc": 0.012148294732242174, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05023309391932773, "top_bins": { "0.9": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9799535676072882 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9359505702856885 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.17987896070468248, "acc": 0.875, "ece": 0.10201081131864481, "brier": 0.12239407244814024, "mean_conf": 0.9252330939193277, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.84375, "aurc": 0.012148294732242174, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05023309391932773, "top_bins": { "0.9": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9799535676072882 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9359505702856885 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 40, "accuracy": 1.0, "full_probability_metrics": { "n": 40, "nll": 0.009886734443521155, "acc": 1.0, "ece": 0.00983711614968219, "brier": 0.0001971224341842575, "mean_conf": 0.9901628838503178, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00983711614968219, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9901536448259697 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9892038988557839 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.009886734443521155, "acc": 1.0, "ece": 0.00983711614968219, "brier": 0.0001971224341842575, "mean_conf": 0.9901628838503178, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00983711614968219, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9901536448259697 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9892038988557839 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 30, "accuracy": 0.75, "full_probability_metrics": { "n": 40, "nll": 0.47269915336494506, "acc": 0.75, "ece": 0.11366225941886313, "brier": 0.30185000988919575, "mean_conf": 0.8610888519756067, "confident_error_rate": 0.025, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.96, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.575, "aurc": 0.060192217951763195, "error_rate_at_0_9": 0.04, "confidence_bias": 0.1110888519756067, "top_bins": { "0.9": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.95": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9618168777802716 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7242188075964775 } }, "score_mae": 0.2723265825254887, "ranked_probability_score": 0.08370646847307771 }, "covered_only_probability_metrics": { "n": 40, "nll": 0.47269915336494506, "acc": 0.75, "ece": 0.11366225941886313, "brier": 0.30185000988919575, "mean_conf": 0.8610888519756067, "confident_error_rate": 0.025, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.96, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.575, "aurc": 0.060192217951763195, "error_rate_at_0_9": 0.04, "confidence_bias": 0.1110888519756067, "top_bins": { "0.9": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.95": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9618168777802716 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7242188075964775 } }, "score_mae": 0.2723265825254887, "ranked_probability_score": 0.08370646847307771 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 52, "accuracy": 0.65, "full_probability_metrics": { "n": 80, "nll": 0.9985265977595695, "acc": 0.65, "ece": 0.13719284197874734, "brier": 0.4738362030380367, "mean_conf": 0.5921262888964118, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18732507638516857, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05787371110358819, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.775, "confidence_cutoff": 0.5825281619880166 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.3873917359421322 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.9985265977595695, "acc": 0.65, "ece": 0.13719284197874734, "brier": 0.4738362030380367, "mean_conf": 0.5921262888964118, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18732507638516857, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05787371110358819, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.775, "confidence_cutoff": 0.5825281619880166 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.3873917359421322 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 55, "accuracy": 0.6875, "full_probability_metrics": { "n": 80, "nll": 0.7777836570002284, "acc": 0.6875, "ece": 0.13366762585626352, "brier": 0.40587483543235975, "mean_conf": 0.6636962556940373, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.225, "aurc": 0.1236239585461083, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02380374430596266, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.6687477777246551 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.45085220564241135 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.7777836570002284, "acc": 0.6875, "ece": 0.13366762585626352, "brier": 0.40587483543235975, "mean_conf": 0.6636962556940373, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.225, "aurc": 0.1236239585461083, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02380374430596266, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.6687477777246551 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.45085220564241135 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 91, "accuracy": 0.455, "full_probability_metrics": { "n": 200, "nll": 1.601810692776627, "acc": 0.455, "ece": 0.056005765405377654, "brier": 0.6769671023476735, "mean_conf": 0.4724915706205288, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.105, "aurc": 0.3099459714151796, "error_rate_at_0_9": 0.0, "confidence_bias": 0.017491570620528807, "top_bins": { "0.9": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.65, "confidence_cutoff": 0.4038586624038567 }, "0.8": { "coverage": 0.8, "accuracy": 0.51875, "confidence_cutoff": 0.21069876298990906 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 1.601810692776627, "acc": 0.455, "ece": 0.056005765405377654, "brier": 0.6769671023476735, "mean_conf": 0.4724915706205288, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.105, "aurc": 0.3099459714151796, "error_rate_at_0_9": 0.0, "confidence_bias": 0.017491570620528807, "top_bins": { "0.9": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.65, "confidence_cutoff": 0.4038586624038567 }, "0.8": { "coverage": 0.8, "accuracy": 0.51875, "confidence_cutoff": 0.21069876298990906 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 65, "accuracy": 0.8125, "full_probability_metrics": { "n": 80, "nll": 0.5108312670021606, "acc": 0.8125, "ece": 0.10584698482403496, "brier": 0.29074335575111687, "mean_conf": 0.9094995046506804, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 0.8852459016393442, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1875, "aurc": 0.09380572263354736, "error_rate_at_0_9": 0.11475409836065574, "confidence_bias": 0.09699950465068041, "top_bins": { "0.9": { "n": 61, "errors": 7, "error_rate": 0.11475409836065574 }, "0.95": { "n": 51, "errors": 5, "error_rate": 0.09803921568627451 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.9752455175493786 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.863691483029977 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.5108312670021606, "acc": 0.8125, "ece": 0.10584698482403496, "brier": 0.29074335575111687, "mean_conf": 0.9094995046506804, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 0.8852459016393442, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1875, "aurc": 0.09380572263354736, "error_rate_at_0_9": 0.11475409836065574, "confidence_bias": 0.09699950465068041, "top_bins": { "0.9": { "n": 61, "errors": 7, "error_rate": 0.11475409836065574 }, "0.95": { "n": 51, "errors": 5, "error_rate": 0.09803921568627451 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.9752455175493786 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.863691483029977 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 73, "accuracy": 0.9125, "full_probability_metrics": { "n": 80, "nll": 0.20791893083223725, "acc": 0.9125, "ece": 0.05459867399837923, "brier": 0.12776983331632913, "mean_conf": 0.9090606154386556, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8875, "coverage_at_1pct_error": 0.7125, "aurc": 0.014063367113830239, "error_rate_at_0_9": 0.0, "confidence_bias": -0.003439384561344405, "top_bins": { "0.9": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 46, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9612046730832484 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8546129210408652 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.20791893083223725, "acc": 0.9125, "ece": 0.05459867399837923, "brier": 0.12776983331632913, "mean_conf": 0.9090606154386556, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8875, "coverage_at_1pct_error": 0.7125, "aurc": 0.014063367113830239, "error_rate_at_0_9": 0.0, "confidence_bias": -0.003439384561344405, "top_bins": { "0.9": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 46, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9612046730832484 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8546129210408652 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 80, "accuracy": 1.0, "full_probability_metrics": { "n": 80, "nll": 0.04200688131108375, "acc": 1.0, "ece": 0.0377444109725753, "brier": 0.010801956650448623, "mean_conf": 0.9622555890274246, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.037744410972575415, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 36, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9885163832978595 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9714295452328383 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.04200688131108375, "acc": 1.0, "ece": 0.0377444109725753, "brier": 0.010801956650448623, "mean_conf": 0.9622555890274246, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.037744410972575415, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 36, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9885163832978595 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9714295452328383 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 68, "accuracy": 0.85, "full_probability_metrics": { "n": 80, "nll": 0.3550170518878505, "acc": 0.85, "ece": 0.06284948647596125, "brier": 0.22255604167511941, "mean_conf": 0.8349219569479723, "confident_error_rate": 0.0, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.525, "aurc": 0.04598201886121128, "error_rate_at_0_9": 0.0, "confidence_bias": -0.015078043052027645, "top_bins": { "0.9": { "n": 34, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8773225676614429 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.7503990679741461 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.3550170518878505, "acc": 0.85, "ece": 0.06284948647596125, "brier": 0.22255604167511941, "mean_conf": 0.8349219569479723, "confident_error_rate": 0.0, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.525, "aurc": 0.04598201886121128, "error_rate_at_0_9": 0.0, "confidence_bias": -0.015078043052027645, "top_bins": { "0.9": { "n": 34, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8773225676614429 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.7503990679741461 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0072341360015546075, "acc": 1.0, "ece": 0.007206565143181853, "brier": 0.00010969391392247062, "mean_conf": 0.9927934348568181, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007206565143181853, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9934249644496728 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9910678212098712 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0072341360015546075, "acc": 1.0, "ece": 0.007206565143181853, "brier": 0.00010969391392247062, "mean_conf": 0.9927934348568181, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007206565143181853, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9934249644496728 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9910678212098712 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.010829823238942816, "acc": 1.0, "ece": 0.01077059059766794, "brier": 0.00023517846224729413, "mean_conf": 0.9892294094023321, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01077059059766794, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9898583938275348 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9880541353470401 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.010829823238942816, "acc": 1.0, "ece": 0.01077059059766794, "brier": 0.00023517846224729413, "mean_conf": 0.9892294094023321, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01077059059766794, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9898583938275348 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9880541353470401 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 0.6220650317484174, "acc": 0.6, "ece": 0.2598616210654444, "brier": 0.4259277938499115, "mean_conf": 0.8165019808339021, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.11261904761904762, "error_rate_at_0_9": 0.0, "confidence_bias": 0.21650198083390215, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8638689517974854 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7093246526321422 } }, "score_mae": 0.39276232386391496, "ranked_probability_score": 0.12211607208292019 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.6220650317484174, "acc": 0.6, "ece": 0.2598616210654444, "brier": 0.4259277938499115, "mean_conf": 0.8165019808339021, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.11261904761904762, "error_rate_at_0_9": 0.0, "confidence_bias": 0.21650198083390215, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8638689517974854 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7093246526321422 } }, "score_mae": 0.39276232386391496, "ranked_probability_score": 0.12211607208292019 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.06131828267827373, "acc": 1.0, "ece": 0.048899604664545876, "brier": 0.026489029544736947, "mean_conf": 0.9511003953354541, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.048899604664545904, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9905512932556068 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9884751059386553 } }, "score_mae": 0.011149562571177114, "ranked_probability_score": 0.004455729704082858 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.06131828267827373, "acc": 1.0, "ece": 0.048899604664545876, "brier": 0.026489029544736947, "mean_conf": 0.9511003953354541, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.048899604664545904, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9905512932556068 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9884751059386553 } }, "score_mae": 0.011149562571177114, "ranked_probability_score": 0.004455729704082858 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.01155051575443739, "acc": 1.0, "ece": 0.011472845159145706, "brier": 0.00023226574691571182, "mean_conf": 0.9885271548408543, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.011472845159145706, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9884340010551941 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.986894550375065 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.01155051575443739, "acc": 1.0, "ece": 0.011472845159145706, "brier": 0.00023226574691571182, "mean_conf": 0.9885271548408543, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.011472845159145706, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9884340010551941 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.986894550375065 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.4947290334619557, "acc": 0.7, "ece": 0.287796955424686, "brier": 0.33637680372940754, "mean_conf": 0.833418121452141, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.09567460317460316, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13341812145214105, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9110106147072752 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.728829882408079 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.4947290334619557, "acc": 0.7, "ece": 0.287796955424686, "brier": 0.33637680372940754, "mean_conf": 0.833418121452141, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.09567460317460316, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13341812145214105, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9110106147072752 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.728829882408079 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.24435457580658276, "acc": 0.8, "ece": 0.13648296301066162, "brier": 0.1573003046081522, "mean_conf": 0.885998173697877, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.08599817369787699, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.971238818889149 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9514172797276097 } }, "score_mae": 0.17490788768701337, "ranked_probability_score": 0.045512577209455875 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.24435457580658276, "acc": 0.8, "ece": 0.13648296301066162, "brier": 0.1573003046081522, "mean_conf": 0.885998173697877, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.08599817369787699, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.971238818889149 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9514172797276097 } }, "score_mae": 0.17490788768701337, "ranked_probability_score": 0.045512577209455875 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.007562171775958854, "acc": 1.0, "ece": 0.007529923370390379, "brier": 9.954740131517311e-05, "mean_conf": 0.9924700766296096, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007529923370390379, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.993558611784159 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9926831172438484 } }, "score_mae": 0.006370404758321721, "ranked_probability_score": 2.859678314457015e-05 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.007562171775958854, "acc": 1.0, "ece": 0.007529923370390379, "brier": 9.954740131517311e-05, "mean_conf": 0.9924700766296096, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007529923370390379, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.993558611784159 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9926831172438484 } }, "score_mae": 0.006370404758321721, "ranked_probability_score": 2.859678314457015e-05 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0069284605777098825, "acc": 1.0, "ece": 0.006904326857730014, "brier": 7.387437558645722e-05, "mean_conf": 0.99309567314227, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.006904326857730014, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9933371372737577 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.992695039586488 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0069284605777098825, "acc": 1.0, "ece": 0.006904326857730014, "brier": 7.387437558645722e-05, "mean_conf": 0.99309567314227, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.006904326857730014, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9933371372737577 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.992695039586488 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.005094294484511262, "acc": 1.0, "ece": 0.00507882994090425, "brier": 4.68341519556578e-05, "mean_conf": 0.9949211700590957, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00507882994090425, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9943638826996374 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9925169369514688 } }, "score_mae": 0.005737341993970986, "ranked_probability_score": 1.5152585643280026e-05 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.005094294484511262, "acc": 1.0, "ece": 0.00507882994090425, "brier": 4.68341519556578e-05, "mean_conf": 0.9949211700590957, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00507882994090425, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9943638826996374 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9925169369514688 } }, "score_mae": 0.005737341993970986, "ranked_probability_score": 1.5152585643280026e-05 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.24058729442418345, "acc": 1.0, "ece": 0.20305319415144826, "brier": 0.0994008757643052, "mean_conf": 0.7969468058485518, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.20305319415144818, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8693554226369429 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7339428253131272 } }, "score_mae": 0.17399208283920226, "ranked_probability_score": 0.02401195544403172 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.24058729442418345, "acc": 1.0, "ece": 0.20305319415144826, "brier": 0.0994008757643052, "mean_conf": 0.7969468058485518, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.20305319415144818, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8693554226369429 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7339428253131272 } }, "score_mae": 0.17399208283920226, "ranked_probability_score": 0.02401195544403172 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.840462962962963, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.7777777777777778, "flip_rate": 0.05555555555555555, "covered_only_flip_rate": 0.05555555555555555, "half_l1": 0.06670269098092366 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.4147128442841158, "share_at_0_9": 0.0, "normalized_entropy": 0.9917649331704367 }, "covered_only_confidence_metrics": { "mean_max_p": 0.4147128442841158, "share_at_0_9": 0.0, "normalized_entropy": 0.9917649331704367 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 101, "accuracy": 0.9181818181818182, "full_probability_metrics": { "n": 110, "nll": 0.15565941999568436, "acc": 0.9181818181818182, "ece": 0.03456906814248485, "brier": 0.09511747286804145, "mean_conf": 0.9304547632817187, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7909090909090909, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9454545454545454, "coverage_at_1pct_error": 0.8, "aurc": 0.006376713225851774, "error_rate_at_0_9": 0.0, "confidence_bias": 0.012272945099900512, "top_bins": { "0.9": { "n": 87, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 84, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9898392872977086 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8906043139855315 } }, "score_mae": 0.1274866006189334, "ranked_probability_score": 0.03269001396821308 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.15565941999568436, "acc": 0.9181818181818182, "ece": 0.03456906814248485, "brier": 0.09511747286804145, "mean_conf": 0.9304547632817187, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7909090909090909, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9454545454545454, "coverage_at_1pct_error": 0.8, "aurc": 0.006376713225851774, "error_rate_at_0_9": 0.0, "confidence_bias": 0.012272945099900512, "top_bins": { "0.9": { "n": 87, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 84, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9898392872977086 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8906043139855315 } }, "score_mae": 0.1274866006189334, "ranked_probability_score": 0.03269001396821308 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.5157419189976028 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.38513580130539204, "paired_flip": { "pairs": 64, "flip_rate": 0.84375, "both_correct_rate": 0.765625, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.875 }, "unknowable": { "n": 110, "mean_max_p": 0.4147128442841158, "share_at_0_9": 0.0, "control_mean_max_p": 0.9304547632817187, "control_share_at_0_9": 0.7909090909090909, "control_acc": 0.9181818181818182, "paired_confidence_drop": 0.5157419189976028, "share_less_confident_than_control": 1.0 }, "clean": { "n": 1046, "nll": 0.64340205208523, "acc": 0.7609942638623327, "ece": 0.02607917633618737, "brier": 0.3145325747542575, "mean_conf": 0.776584127219792, "confident_error_rate": 0.015296367112810707, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9694072657743786, "coverage_at_5pct_error": 0.5678776290630975, "coverage_at_1pct_error": 0.2418738049713193, "aurc": 0.0659154401199933, "error_rate_at_0_9": 0.030592734225621414, "confidence_bias": 0.015589863357459266, "top_bins": { "0.9": { "n": 523, "errors": 16, "error_rate": 0.030592734225621414 }, "0.95": { "n": 428, "errors": 10, "error_rate": 0.02336448598130841 }, "0.99": { "n": 136, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9694072657743786, "confidence_cutoff": 0.900898774733367 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8685782556750299, "confidence_cutoff": 0.5414887070655823 } }, "score_mae": 0.18542259338155553, "ranked_probability_score": 0.05309659577015893 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.2001095097427736, "acc": 0.55, "ece": 0.15707177801291555, "brier": 0.5423712656116211, "mean_conf": 0.5839834204481658, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.22846596579375214, "error_rate_at_0_9": 0.0, "confidence_bias": 0.03398342044816571, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.5908694179226545 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.41954513748428623 } } }, "buried_paws": { "n": 20, "nll": 0.7583959032791606, "acc": 0.75, "ece": 0.1962823095026524, "brier": 0.44253604470947644, "mean_conf": 0.894629996826019, "confident_error_rate": 0.15, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.161849374825381, "error_rate_at_0_9": 0.2, "confidence_bias": 0.14462999682601896, "top_bins": { "0.9": { "n": 15, "errors": 3, "error_rate": 0.2 }, "0.95": { "n": 13, "errors": 2, "error_rate": 0.15384615384615385 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9630329120498103 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8593514305546454 } } }, "buried_qnli": { "n": 20, "nll": 0.2951763106727876, "acc": 0.9, "ece": 0.10106320366043044, "brier": 0.18245509323688672, "mean_conf": 0.8800315839835061, "confident_error_rate": 0.05, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9230769230769231, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.032635805671796386, "error_rate_at_0_9": 0.07692307692307693, "confidence_bias": -0.019968416016493906, "top_bins": { "0.9": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9382989932723418 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.7824955712261455 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.5691632942022599, "acc": 0.65, "ece": 0.09037131558307589, "brier": 0.39247233447016044, "mean_conf": 0.6960255894462387, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.1926055119630971, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04602558944623869, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7124768495559692 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5802226066589355 } } }, "composition_final_combination": { "n": 32, "nll": 0.5780628525520849, "acc": 0.78125, "ece": 0.18448002890132353, "brier": 0.35513312120830176, "mean_conf": 0.9436765872113124, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.78125, "accuracy_at_0_9": 0.88, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.59375, "aurc": 0.05964396990758747, "error_rate_at_0_9": 0.12, "confidence_bias": 0.16242658721131242, "top_bins": { "0.9": { "n": 25, "errors": 3, "error_rate": 0.12 }, "0.95": { "n": 25, "errors": 3, "error_rate": 0.12 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9751177862732483 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8846153846153846, "confidence_cutoff": 0.8956495523452759 } } }, "composition_final_exception": { "n": 32, "nll": 0.1291452177610849, "acc": 0.96875, "ece": 0.03135938784471088, "brier": 0.06433444963615564, "mean_conf": 0.9620390965967792, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 0.9655172413793104, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "aurc": 0.004072377873563219, "error_rate_at_0_9": 0.034482758620689655, "confidence_bias": -0.00671090340322078, "top_bins": { "0.9": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9812241902339182 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.9633261726799828 } } }, "composition_final_negation": { "n": 32, "nll": 0.17987896070468248, "acc": 0.875, "ece": 0.10201081131864481, "brier": 0.12239407244814024, "mean_conf": 0.9252330939193277, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.90625, "coverage_at_1pct_error": 0.84375, "aurc": 0.012148294732242174, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05023309391932773, "top_bins": { "0.9": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9799535676072882 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9359505702856885 } } }, "contrastive_authorization": { "n": 40, "nll": 0.009886734443521155, "acc": 1.0, "ece": 0.00983711614968219, "brier": 0.0001971224341842575, "mean_conf": 0.9901628838503178, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00983711614968219, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9901536448259697 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9892038988557839 } } }, "contrastive_deadline": { "n": 40, "nll": 0.47269915336494506, "acc": 0.75, "ece": 0.11366225941886313, "brier": 0.30185000988919575, "mean_conf": 0.8610888519756067, "confident_error_rate": 0.025, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.96, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.575, "aurc": 0.060192217951763195, "error_rate_at_0_9": 0.04, "confidence_bias": 0.1110888519756067, "top_bins": { "0.9": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.95": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9618168777802716 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7242188075964775 } }, "score_mae": 0.2723265825254887, "ranked_probability_score": 0.08370646847307771 }, "emotion": { "n": 80, "nll": 0.9985265977595695, "acc": 0.65, "ece": 0.13719284197874734, "brier": 0.4738362030380367, "mean_conf": 0.5921262888964118, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18732507638516857, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05787371110358819, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.775, "confidence_cutoff": 0.5825281619880166 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.3873917359421322 } } }, "mmlu": { "n": 80, "nll": 0.7777836570002284, "acc": 0.6875, "ece": 0.13366762585626352, "brier": 0.40587483543235975, "mean_conf": 0.6636962556940373, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.225, "aurc": 0.1236239585461083, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02380374430596266, "top_bins": { "0.9": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.6687477777246551 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.45085220564241135 } } }, "mmlu_pro": { "n": 200, "nll": 1.601810692776627, "acc": 0.455, "ece": 0.056005765405377654, "brier": 0.6769671023476735, "mean_conf": 0.4724915706205288, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.105, "aurc": 0.3099459714151796, "error_rate_at_0_9": 0.0, "confidence_bias": 0.017491570620528807, "top_bins": { "0.9": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.65, "confidence_cutoff": 0.4038586624038567 }, "0.8": { "coverage": 0.8, "accuracy": 0.51875, "confidence_cutoff": 0.21069876298990906 } } }, "paws": { "n": 80, "nll": 0.5108312670021606, "acc": 0.8125, "ece": 0.10584698482403496, "brier": 0.29074335575111687, "mean_conf": 0.9094995046506804, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 0.8852459016393442, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1875, "aurc": 0.09380572263354736, "error_rate_at_0_9": 0.11475409836065574, "confidence_bias": 0.09699950465068041, "top_bins": { "0.9": { "n": 61, "errors": 7, "error_rate": 0.11475409836065574 }, "0.95": { "n": 51, "errors": 5, "error_rate": 0.09803921568627451 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.9752455175493786 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.863691483029977 } } }, "qnli": { "n": 80, "nll": 0.20791893083223725, "acc": 0.9125, "ece": 0.05459867399837923, "brier": 0.12776983331632913, "mean_conf": 0.9090606154386556, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8875, "coverage_at_1pct_error": 0.7125, "aurc": 0.014063367113830239, "error_rate_at_0_9": 0.0, "confidence_bias": -0.003439384561344405, "top_bins": { "0.9": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 46, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9612046730832484 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8546129210408652 } } }, "sciq": { "n": 80, "nll": 0.04200688131108375, "acc": 1.0, "ece": 0.0377444109725753, "brier": 0.010801956650448623, "mean_conf": 0.9622555890274246, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.037744410972575415, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 36, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9885163832978595 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9714295452328383 } } }, "tweet_offensive": { "n": 80, "nll": 0.3550170518878505, "acc": 0.85, "ece": 0.06284948647596125, "brier": 0.22255604167511941, "mean_conf": 0.8349219569479723, "confident_error_rate": 0.0, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.525, "aurc": 0.04598201886121128, "error_rate_at_0_9": 0.0, "confidence_bias": -0.015078043052027645, "top_bins": { "0.9": { "n": 34, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8773225676614429 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.7503990679741461 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.6998515898547917, "acc": 0.5, "ece": 0.05194225091583138, "brier": 0.5066392016640704, "mean_conf": 0.5519422509158314, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.05194225091583138, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.5558717416486663 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5445031685979118 } } }, "unknowable_authorization": { "n": 10, "nll": 0.7448168246612408, "acc": 0.5, "ece": 0.14552007136159667, "brier": 0.5484542254943233, "mean_conf": 0.6455200713615966, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.638968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.14552007136159661, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6613824766462303 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6261212825775146 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.0072341360015546075, "acc": 1.0, "ece": 0.007206565143181853, "brier": 0.00010969391392247062, "mean_conf": 0.9927934348568181, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007206565143181853, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9934249644496728 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9910678212098712 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.010829823238942816, "acc": 1.0, "ece": 0.01077059059766794, "brier": 0.00023517846224729413, "mean_conf": 0.9892294094023321, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01077059059766794, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9898583938275348 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9880541353470401 } } }, "unknowable_control_deadline": { "n": 10, "nll": 0.6220650317484174, "acc": 0.6, "ece": 0.2598616210654444, "brier": 0.4259277938499115, "mean_conf": 0.8165019808339021, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.11261904761904762, "error_rate_at_0_9": 0.0, "confidence_bias": 0.21650198083390215, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8638689517974854 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7093246526321422 } }, "score_mae": 0.39276232386391496, "ranked_probability_score": 0.12211607208292019 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.06131828267827373, "acc": 1.0, "ece": 0.048899604664545876, "brier": 0.026489029544736947, "mean_conf": 0.9511003953354541, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.048899604664545904, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9905512932556068 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9884751059386553 } }, "score_mae": 0.011149562571177114, "ranked_probability_score": 0.004455729704082858 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.01155051575443739, "acc": 1.0, "ece": 0.011472845159145706, "brier": 0.00023226574691571182, "mean_conf": 0.9885271548408543, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.011472845159145706, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9884340010551941 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.986894550375065 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.4947290334619557, "acc": 0.7, "ece": 0.287796955424686, "brier": 0.33637680372940754, "mean_conf": 0.833418121452141, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.09567460317460316, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13341812145214105, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9110106147072752 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.728829882408079 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.24435457580658276, "acc": 0.8, "ece": 0.13648296301066162, "brier": 0.1573003046081522, "mean_conf": 0.885998173697877, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.08599817369787699, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.971238818889149 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9514172797276097 } }, "score_mae": 0.17490788768701337, "ranked_probability_score": 0.045512577209455875 }, "unknowable_control_sla_response": { "n": 10, "nll": 0.007562171775958854, "acc": 1.0, "ece": 0.007529923370390379, "brier": 9.954740131517311e-05, "mean_conf": 0.9924700766296096, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007529923370390379, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.993558611784159 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9926831172438484 } }, "score_mae": 0.006370404758321721, "ranked_probability_score": 2.859678314457015e-05 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.0069284605777098825, "acc": 1.0, "ece": 0.006904326857730014, "brier": 7.387437558645722e-05, "mean_conf": 0.99309567314227, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.006904326857730014, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9933371372737577 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.992695039586488 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.005094294484511262, "acc": 1.0, "ece": 0.00507882994090425, "brier": 4.68341519556578e-05, "mean_conf": 0.9949211700590957, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.00507882994090425, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9943638826996374 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9925169369514688 } }, "score_mae": 0.005737341993970986, "ranked_probability_score": 1.5152585643280026e-05 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.24058729442418345, "acc": 1.0, "ece": 0.20305319415144826, "brier": 0.0994008757643052, "mean_conf": 0.7969468058485518, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.20305319415144818, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8693554226369429 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7339428253131272 } }, "score_mae": 0.17399208283920226, "ranked_probability_score": 0.02401195544403172 }, "unknowable_deadline": { "n": 10, "nll": 1.1198824344886444, "acc": 0.3, "ece": 0.08148312874012331, "brier": 0.6805657841146944, "mean_conf": 0.3814831287401233, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7779761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.08148312874012331, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3809420650916676 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3783619701862335 } }, "score_mae": 0.7100897222799378, "ranked_probability_score": 0.23206061049299467 }, "unknowable_late_fee": { "n": 10, "nll": 1.0972574537066067, "acc": 0.4, "ece": 0.058302205952400266, "brier": 0.6657394864910027, "mean_conf": 0.34169779404759976, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.545, "error_rate_at_0_9": null, "confidence_bias": -0.058302205952400266, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.34045907855033875 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.338800172173982 } }, "score_mae": 0.6052484333701532, "ranked_probability_score": 0.2115788870881336 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.112862234967989, "acc": 0.3, "ece": 0.07610132628439964, "brier": 0.6760672369470936, "mean_conf": 0.37610132628439963, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7066666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.07610132628439964, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.374070245800309 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3692213782900161 } } }, "unknowable_return_window": { "n": 10, "nll": 0.6957587334679805, "acc": 0.5, "ece": 0.018668265422355335, "brier": 0.5025710977010973, "mean_conf": 0.5186682654223553, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.536031746031746, "error_rate_at_0_9": null, "confidence_bias": 0.018668265422355335, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5166360139846802 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5068895069691686 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.1269738860601979, "acc": 0.1, "ece": 0.250380193136967, "brier": 0.6856910932887642, "mean_conf": 0.35038019313696706, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.99, "error_rate_at_0_9": null, "confidence_bias": 0.250380193136967, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.355785071849823 }, "0.8": { "coverage": 0.8, "accuracy": 0.0, "confidence_cutoff": 0.344328253478266 } }, "score_mae": 0.7282357632156017, "ranked_probability_score": 0.2367974137690141 }, "unknowable_sla_response": { "n": 10, "nll": 1.11174044690751, "acc": 0.2, "ece": 0.14516815754689638, "brier": 0.6755108858048389, "mean_conf": 0.3451681575468964, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8266666666666668, "error_rate_at_0_9": null, "confidence_bias": 0.14516815754689638, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.34279613749957905 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.34119775891304016 } }, "score_mae": 0.5145410599397902, "ranked_probability_score": 0.19786878800175628 }, "unknowable_spend_threshold": { "n": 10, "nll": 1.0856148730564725, "acc": 0.5, "ece": 0.15645529386679047, "brier": 0.6580448118620636, "mean_conf": 0.34354470613320953, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": -0.15645529386679047, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.34481036765668105 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3438909258777455 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.0992590022623214, "acc": 0.4, "ece": 0.06097743723051691, "brier": 0.6670869491795234, "mean_conf": 0.3390225627694831, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6283333333333333, "error_rate_at_0_9": null, "confidence_bias": -0.06097743723051691, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.3382512826248621 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3379666406522202 } }, "score_mae": 0.8017853379249573, "ranked_probability_score": 0.2445020184482385 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.0470916173198428, "acc": 0.5, "ece": 0.1316871692331874, "brier": 0.632766517964943, "mean_conf": 0.3683128307668126, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.44174603174603166, "error_rate_at_0_9": null, "confidence_bias": -0.1316871692331874, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.36913369205992497 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3592945337295532 } }, "score_mae": 0.5019434514556199, "ranked_probability_score": 0.18261599991953084 } }, "variants": { "clean": { "n": 1156, "nll": 0.6768249458898673, "acc": 0.7249134948096886, "ece": 0.024477828330115248, "brier": 0.3442841229222124, "mean_conf": 0.7421500086013453, "confident_error_rate": 0.01384083044982699, "coverage_at_0_9": 0.4524221453287197, "accuracy_at_0_9": 0.9694072657743786, "coverage_at_5pct_error": 0.513840830449827, "coverage_at_1pct_error": 0.21885813148788927, "aurc": 0.08264902631608068, "error_rate_at_0_9": 0.030592734225621414, "confidence_bias": 0.017236513791656694, "top_bins": { "0.9": { "n": 523, "errors": 16, "error_rate": 0.030592734225621414 }, "0.95": { "n": 428, "errors": 10, "error_rate": 0.02336448598130841 }, "0.99": { "n": 136, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9550173010380623, "confidence_cutoff": 0.8548957919375989 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.8335135135135135, "confidence_cutoff": 0.4427716451618151 } }, "score_mae": 0.357254356375101, "ranked_probability_score": 0.11477435471382855 }, "none_absent": { "n": 36, "nll": 0.9014463285206258, "acc": 0.6944444444444444, "ece": 0.15858101433999128, "brier": 0.4775363816368024, "mean_conf": 0.6826536262698608, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.08333333333333333, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.37021521934474755, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": -0.011790818174583606, "top_bins": { "0.9": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7222222222222222, "confidence_cutoff": 0.716648211671901 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7586206896551724, "confidence_cutoff": 0.5481367543895019 } } }, "none_present": { "n": 36, "nll": 1.0027024779338058, "acc": 0.6666666666666666, "ece": 0.13152879719346894, "brier": 0.46500145494174905, "mean_conf": 0.6890104192358644, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2777777777777778, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3611111111111111, "coverage_at_1pct_error": 0.3611111111111111, "aurc": 0.14684740218954656, "error_rate_at_0_9": 0.0, "confidence_bias": 0.02234375256919774, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8888888888888888, "confidence_cutoff": 0.6669138998935952 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7241379310344828, "confidence_cutoff": 0.48648502753184775 } } }, "permuted": { "n": 36, "nll": 0.7423354063927979, "acc": 0.7777777777777778, "ece": 0.16010649217253398, "brier": 0.3405369374493546, "mean_conf": 0.7259648234598495, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3333333333333333, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3611111111111111, "coverage_at_1pct_error": 0.3611111111111111, "aurc": 0.08746855570131847, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05181295431792832, "top_bins": { "0.9": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8888888888888888, "confidence_cutoff": 0.772501218728703 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8275862068965517, "confidence_cutoff": 0.4970026629010039 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.06495690740412934, "flip_rate": 0.05555555555555555 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.64340205208523, "acc": 0.7609942638623327, "ece": 0.02607917633618737, "brier": 0.3145325747542575, "mean_conf": 0.776584127219792, "confident_error_rate": 0.015296367112810707, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9694072657743786, "coverage_at_5pct_error": 0.5678776290630975, "coverage_at_1pct_error": 0.2418738049713193, "aurc": 0.0659154401199933, "error_rate_at_0_9": 0.030592734225621414, "confidence_bias": 0.015589863357459266, "top_bins": { "0.9": { "n": 523, "errors": 16, "error_rate": 0.030592734225621414 }, "0.95": { "n": 428, "errors": 10, "error_rate": 0.02336448598130841 }, "0.99": { "n": 136, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9694072657743786, "confidence_cutoff": 0.900898774733367 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.8685782556750299, "confidence_cutoff": 0.5414887070655823 } }, "score_mae": 0.18542259338155553, "ranked_probability_score": 0.05309659577015893 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Qwen3.5-9B": { "panels": { "old_core": { "point": 0.7391369047619047, "exact_fraction": "4967/6720", "ci95": [ 0.7010044642857143, 0.7761170014880953 ] }, "v3_core": { "point": 0.44625, "exact_fraction": "357/800", "ci95": [ 0.42458333333333337, 0.46791666666666665 ] }, "v4": { "point": 0.8984375, "exact_fraction": "115/128", "ci95": [ 0.8666927104329598, 0.9279272151898734 ] }, "v5": { "point": 0.7958333333333333, "exact_fraction": "191/240", "ci95": [ 0.7629046639472732, 0.8283149055437853 ] }, "transfer_v9_test": { "point": 0.7323135755258127, "exact_fraction": "383/523", "ci95": [ 0.7022895352025281, 0.7618586870466264 ] }, "original_four_panel_mean": { "point": 0.7199144345238095, "exact_fraction": "193513/268800", "ci95": [ 0.7047555031264947, 0.7351278336512089 ] }, "weighted_mean": { "point": 0.6972912327574433, "exact_fraction": "1307025/1874432", "ci95": [ 0.6827347147629699, 0.7119511849971354 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 110, "requested": 128 }, "boolean_constraints": { "correct": 63, "requested": 64 }, "dbpedia_14": { "correct": 108, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 76, "requested": 96 }, "ordinal_rubric": { "correct": 54, "requested": 64 }, "relational_composition": { "correct": 35, "requested": 96 }, "scoped_evidence": { "correct": 53, "requested": 96 }, "state_tracking": { "correct": 30, "requested": 96 }, "unknown_rejection": { "correct": 46, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 46, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 20, "requested": 80 }, "massive_en": { "correct": 106, "requested": 120 }, "massive_zh": { "correct": 104, "requested": 120 }, "multiset_reconciliation": { "correct": 16, "requested": 80 }, "ordinal_service_loss": { "correct": 16, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 22, "requested": 80 }, "temporal_exclusion": { "correct": 29, "requested": 80 }, "transaction_recovery": { "correct": 28, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 135, "requested": 160 }, "belebele_en": { "correct": 158, "requested": 160 }, "belebele_zh": { "correct": 147, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 77, "requested": 120 }, "squad2_answerability": { "correct": 89, "requested": 120 }, "snli": { "correct": 98, "requested": 120 }, "qasc": { "correct": 118, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 766, "accuracy": 0.7323135755258127, "full_probability_metrics": { "n": 1046, "nll": 0.759729403716022, "acc": 0.7323135755258127, "ece": 0.09750832444229235, "brier": 0.3673011918808915, "mean_conf": 0.8291984136018093, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.5688336520076482, "accuracy_at_0_9": 0.9159663865546218, "coverage_at_5pct_error": 0.47418738049713194, "coverage_at_1pct_error": 0.2084130019120459, "aurc": 0.0872044723520789, "error_rate_at_0_9": 0.08403361344537816, "confidence_bias": 0.09688483807599668, "top_bins": { "0.9": { "n": 595, "errors": 50, "error_rate": 0.08403361344537816 }, "0.95": { "n": 523, "errors": 30, "error_rate": 0.05736137667304015 }, "0.99": { "n": 327, "errors": 7, "error_rate": 0.021406727828746176 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9426386233269598, "confidence_cutoff": 0.951518493732782 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.828978622327791, "confidence_cutoff": 0.6224593334657152 } }, "score_mae": 0.28987980749634135, "ranked_probability_score": 0.08782854063750568 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.759729403716022, "acc": 0.7323135755258127, "ece": 0.09750832444229235, "brier": 0.3673011918808915, "mean_conf": 0.8291984136018093, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.5688336520076482, "accuracy_at_0_9": 0.9159663865546218, "coverage_at_5pct_error": 0.47418738049713194, "coverage_at_1pct_error": 0.2084130019120459, "aurc": 0.0872044723520789, "error_rate_at_0_9": 0.08403361344537816, "confidence_bias": 0.09688483807599668, "top_bins": { "0.9": { "n": 595, "errors": 50, "error_rate": 0.08403361344537816 }, "0.95": { "n": 523, "errors": 30, "error_rate": 0.05736137667304015 }, "0.99": { "n": 327, "errors": 7, "error_rate": 0.021406727828746176 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9426386233269598, "confidence_cutoff": 0.951518493732782 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.828978622327791, "confidence_cutoff": 0.6224593334657152 } }, "score_mae": 0.28987980749634135, "ranked_probability_score": 0.08782854063750568 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 8, "accuracy": 0.4, "full_probability_metrics": { "n": 20, "nll": 1.6789716954193772, "acc": 0.4, "ece": 0.36573305686207336, "brier": 0.8254645361260712, "mean_conf": 0.6646489117250975, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5415913869841888, "error_rate_at_0_9": 0.5, "confidence_bias": 0.2646489117250975, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6553088766308754 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.539919199319293 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.6789716954193772, "acc": 0.4, "ece": 0.36573305686207336, "brier": 0.8254645361260712, "mean_conf": 0.6646489117250975, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5415913869841888, "error_rate_at_0_9": 0.5, "confidence_bias": 0.2646489117250975, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6553088766308754 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.539919199319293 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 16, "accuracy": 0.8, "full_probability_metrics": { "n": 20, "nll": 0.41364062147400277, "acc": 0.8, "ece": 0.17365084393507335, "brier": 0.25633054757351986, "mean_conf": 0.9136674185543736, "confident_error_rate": 0.1, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.45, "coverage_at_1pct_error": 0.45, "aurc": 0.06957397263898812, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.11366741855437357, "top_bins": { "0.9": { "n": 13, "errors": 2, "error_rate": 0.15384615384615385 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9579122733914276 }, "0.8": { "coverage": 0.85, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.8807970744787121 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.41364062147400277, "acc": 0.8, "ece": 0.17365084393507335, "brier": 0.25633054757351986, "mean_conf": 0.9136674185543736, "confident_error_rate": 0.1, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.45, "coverage_at_1pct_error": 0.45, "aurc": 0.06957397263898812, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.11366741855437357, "top_bins": { "0.9": { "n": 13, "errors": 2, "error_rate": 0.15384615384615385 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9579122733914276 }, "0.8": { "coverage": 0.85, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.8807970744787121 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 19, "accuracy": 0.95, "full_probability_metrics": { "n": 20, "nll": 0.2921434213238844, "acc": 0.95, "ece": 0.08453801695369767, "brier": 0.13339955861821834, "mean_conf": 0.9265174852468341, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.35, "aurc": 0.04986533783553907, "error_rate_at_0_9": 0.0625, "confidence_bias": -0.023482514753165873, "top_bins": { "0.9": { "n": 16, "errors": 1, "error_rate": 0.0625 }, "0.95": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9770226296592014 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9046505363181393 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.2921434213238844, "acc": 0.95, "ece": 0.08453801695369767, "brier": 0.13339955861821834, "mean_conf": 0.9265174852468341, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.35, "aurc": 0.04986533783553907, "error_rate_at_0_9": 0.0625, "confidence_bias": -0.023482514753165873, "top_bins": { "0.9": { "n": 16, "errors": 1, "error_rate": 0.0625 }, "0.95": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9770226296592014 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9046505363181393 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 9, "accuracy": 0.45, "full_probability_metrics": { "n": 20, "nll": 1.2617601328295882, "acc": 0.45, "ece": 0.34399971383452643, "brier": 0.834135763504744, "mean_conf": 0.7640741993133061, "confident_error_rate": 0.2, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.617740895585323, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.3140741993133061, "top_bins": { "0.9": { "n": 6, "errors": 4, "error_rate": 0.6666666666666666 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.2727272727272727, "confidence_cutoff": 0.7772998578161043 }, "0.8": { "coverage": 0.85, "accuracy": 0.35294117647058826, "confidence_cutoff": 0.5926665906508809 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.2617601328295882, "acc": 0.45, "ece": 0.34399971383452643, "brier": 0.834135763504744, "mean_conf": 0.7640741993133061, "confident_error_rate": 0.2, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.617740895585323, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.3140741993133061, "top_bins": { "0.9": { "n": 6, "errors": 4, "error_rate": 0.6666666666666666 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.2727272727272727, "confidence_cutoff": 0.7772998578161043 }, "0.8": { "coverage": 0.85, "accuracy": 0.35294117647058826, "confidence_cutoff": 0.5926665906508809 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 17, "accuracy": 0.53125, "full_probability_metrics": { "n": 32, "nll": 0.6027086688012528, "acc": 0.53125, "ece": 0.2245713627825465, "brier": 0.41413801251586274, "mean_conf": 0.7009620316433398, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.22099202452059225, "error_rate_at_0_9": 0.25, "confidence_bias": 0.1697120316433398, "top_bins": { "0.9": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8125, "confidence_cutoff": 0.7057850360870361 }, "0.8": { "coverage": 0.90625, "accuracy": 0.5862068965517241, "confidence_cutoff": 0.5312093775743617 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.6027086688012528, "acc": 0.53125, "ece": 0.2245713627825465, "brier": 0.41413801251586274, "mean_conf": 0.7009620316433398, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.22099202452059225, "error_rate_at_0_9": 0.25, "confidence_bias": 0.1697120316433398, "top_bins": { "0.9": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8125, "confidence_cutoff": 0.7057850360870361 }, "0.8": { "coverage": 0.90625, "accuracy": 0.5862068965517241, "confidence_cutoff": 0.5312093775743617 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 19, "accuracy": 0.59375, "full_probability_metrics": { "n": 32, "nll": 0.7895514499373273, "acc": 0.59375, "ece": 0.31475717153147065, "brier": 0.574466147198007, "mean_conf": 0.7698756301405588, "confident_error_rate": 0.0, "coverage_at_0_9": 0.21875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.21875, "coverage_at_1pct_error": 0.21875, "aurc": 0.3271882885353148, "error_rate_at_0_9": 0.0, "confidence_bias": 0.17612563014055882, "top_bins": { "0.9": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 0.6111111111111112, "confidence_cutoff": 0.7549149877590688 }, "0.8": { "coverage": 0.84375, "accuracy": 0.5185185185185185, "confidence_cutoff": 0.6791787147521973 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.7895514499373273, "acc": 0.59375, "ece": 0.31475717153147065, "brier": 0.574466147198007, "mean_conf": 0.7698756301405588, "confident_error_rate": 0.0, "coverage_at_0_9": 0.21875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.21875, "coverage_at_1pct_error": 0.21875, "aurc": 0.3271882885353148, "error_rate_at_0_9": 0.0, "confidence_bias": 0.17612563014055882, "top_bins": { "0.9": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 0.6111111111111112, "confidence_cutoff": 0.7549149877590688 }, "0.8": { "coverage": 0.84375, "accuracy": 0.5185185185185185, "confidence_cutoff": 0.6791787147521973 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 20, "accuracy": 0.625, "full_probability_metrics": { "n": 32, "nll": 0.5997900737237437, "acc": 0.625, "ece": 0.10364475153777164, "brier": 0.4149719922019578, "mean_conf": 0.6595295351290815, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.21459078183057648, "error_rate_at_0_9": null, "confidence_bias": 0.03452953512908152, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.875, "accuracy": 0.6785714285714286, "confidence_cutoff": 0.5621765088386383 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.5997900737237437, "acc": 0.625, "ece": 0.10364475153777164, "brier": 0.4149719922019578, "mean_conf": 0.6595295351290815, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.21459078183057648, "error_rate_at_0_9": null, "confidence_bias": 0.03452953512908152, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.875, "accuracy": 0.6785714285714286, "confidence_cutoff": 0.5621765088386383 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 40, "accuracy": 1.0, "full_probability_metrics": { "n": 40, "nll": 0.012996411860518692, "acc": 1.0, "ece": 0.0128793991091124, "brier": 0.0004613228907562773, "mean_conf": 0.9871206008908876, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0128793991091124, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9914225150110113 }, "0.8": { "coverage": 0.85, "accuracy": 1.0, "confidence_cutoff": 0.97966764715126 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.012996411860518692, "acc": 1.0, "ece": 0.0128793991091124, "brier": 0.0004613228907562773, "mean_conf": 0.9871206008908876, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0128793991091124, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9914225150110113 }, "0.8": { "coverage": 0.85, "accuracy": 1.0, "confidence_cutoff": 0.97966764715126 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 31, "accuracy": 0.775, "full_probability_metrics": { "n": 40, "nll": 0.49745983599501137, "acc": 0.775, "ece": 0.13089066802312613, "brier": 0.3050484395003504, "mean_conf": 0.8347478671121624, "confident_error_rate": 0.025, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.375, "aurc": 0.08069818256712594, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.0597478671121624, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.8607219650012694 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7295618923204291 } }, "score_mae": 0.28644843270897535, "ranked_probability_score": 0.08285742255016963 }, "covered_only_probability_metrics": { "n": 40, "nll": 0.49745983599501137, "acc": 0.775, "ece": 0.13089066802312613, "brier": 0.3050484395003504, "mean_conf": 0.8347478671121624, "confident_error_rate": 0.025, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.375, "aurc": 0.08069818256712594, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.0597478671121624, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.8607219650012694 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7295618923204291 } }, "score_mae": 0.28644843270897535, "ranked_probability_score": 0.08285742255016963 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 42, "accuracy": 0.525, "full_probability_metrics": { "n": 80, "nll": 1.3187849053528407, "acc": 0.525, "ece": 0.20447016219613473, "brier": 0.5881928917298136, "mean_conf": 0.7294701621961348, "confident_error_rate": 0.075, "coverage_at_0_9": 0.3625, "accuracy_at_0_9": 0.7931034482758621, "coverage_at_5pct_error": 0.0375, "coverage_at_1pct_error": 0.0375, "aurc": 0.25124869903151126, "error_rate_at_0_9": 0.20689655172413793, "confidence_bias": 0.20447016219613479, "top_bins": { "0.9": { "n": 29, "errors": 6, "error_rate": 0.20689655172413793 }, "0.95": { "n": 21, "errors": 3, "error_rate": 0.14285714285714285 }, "0.99": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.7771893073791486 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.5038760210596767 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.3187849053528407, "acc": 0.525, "ece": 0.20447016219613473, "brier": 0.5881928917298136, "mean_conf": 0.7294701621961348, "confident_error_rate": 0.075, "coverage_at_0_9": 0.3625, "accuracy_at_0_9": 0.7931034482758621, "coverage_at_5pct_error": 0.0375, "coverage_at_1pct_error": 0.0375, "aurc": 0.25124869903151126, "error_rate_at_0_9": 0.20689655172413793, "confidence_bias": 0.20447016219613479, "top_bins": { "0.9": { "n": 29, "errors": 6, "error_rate": 0.20689655172413793 }, "0.95": { "n": 21, "errors": 3, "error_rate": 0.14285714285714285 }, "0.99": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.7771893073791486 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.5038760210596767 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 59, "accuracy": 0.7375, "full_probability_metrics": { "n": 80, "nll": 0.7246171543085598, "acc": 0.7375, "ece": 0.12236678796568477, "brier": 0.359554531615567, "mean_conf": 0.8571929488972623, "confident_error_rate": 0.025, "coverage_at_0_9": 0.5875, "accuracy_at_0_9": 0.9574468085106383, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.525, "aurc": 0.06346347577428549, "error_rate_at_0_9": 0.0425531914893617, "confidence_bias": 0.11969294889726223, "top_bins": { "0.9": { "n": 47, "errors": 2, "error_rate": 0.0425531914893617 }, "0.95": { "n": 44, "errors": 1, "error_rate": 0.022727272727272728 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9615272075352188 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.6888570836846313 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.7246171543085598, "acc": 0.7375, "ece": 0.12236678796568477, "brier": 0.359554531615567, "mean_conf": 0.8571929488972623, "confident_error_rate": 0.025, "coverage_at_0_9": 0.5875, "accuracy_at_0_9": 0.9574468085106383, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.525, "aurc": 0.06346347577428549, "error_rate_at_0_9": 0.0425531914893617, "confidence_bias": 0.11969294889726223, "top_bins": { "0.9": { "n": 47, "errors": 2, "error_rate": 0.0425531914893617 }, "0.95": { "n": 44, "errors": 1, "error_rate": 0.022727272727272728 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9615272075352188 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.6888570836846313 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 107, "accuracy": 0.535, "full_probability_metrics": { "n": 200, "nll": 1.6502405207825164, "acc": 0.535, "ece": 0.14518812226380381, "brier": 0.6523599717495105, "mean_conf": 0.6742868197918527, "confident_error_rate": 0.05, "coverage_at_0_9": 0.315, "accuracy_at_0_9": 0.8412698412698413, "coverage_at_5pct_error": 0.21, "coverage_at_1pct_error": 0.085, "aurc": 0.24665741690802748, "error_rate_at_0_9": 0.15873015873015872, "confidence_bias": 0.13928681979185265, "top_bins": { "0.9": { "n": 63, "errors": 10, "error_rate": 0.15873015873015872 }, "0.95": { "n": 52, "errors": 4, "error_rate": 0.07692307692307693 }, "0.99": { "n": 37, "errors": 2, "error_rate": 0.05405405405405406 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.72, "confidence_cutoff": 0.7254660404027048 }, "0.8": { "coverage": 0.8, "accuracy": 0.6, "confidence_cutoff": 0.3802954477895661 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 1.6502405207825164, "acc": 0.535, "ece": 0.14518812226380381, "brier": 0.6523599717495105, "mean_conf": 0.6742868197918527, "confident_error_rate": 0.05, "coverage_at_0_9": 0.315, "accuracy_at_0_9": 0.8412698412698413, "coverage_at_5pct_error": 0.21, "coverage_at_1pct_error": 0.085, "aurc": 0.24665741690802748, "error_rate_at_0_9": 0.15873015873015872, "confidence_bias": 0.13928681979185265, "top_bins": { "0.9": { "n": 63, "errors": 10, "error_rate": 0.15873015873015872 }, "0.95": { "n": 52, "errors": 4, "error_rate": 0.07692307692307693 }, "0.99": { "n": 37, "errors": 2, "error_rate": 0.05405405405405406 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.72, "confidence_cutoff": 0.7254660404027048 }, "0.8": { "coverage": 0.8, "accuracy": 0.6, "confidence_cutoff": 0.3802954477895661 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 71, "accuracy": 0.8875, "full_probability_metrics": { "n": 80, "nll": 0.3899038705435832, "acc": 0.8875, "ece": 0.10104432041975961, "brier": 0.20088637738716225, "mean_conf": 0.9519886566878337, "confident_error_rate": 0.075, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.9142857142857143, "coverage_at_5pct_error": 0.6125, "coverage_at_1pct_error": 0.2625, "aurc": 0.04244121984541574, "error_rate_at_0_9": 0.08571428571428572, "confidence_bias": 0.06448865668783377, "top_bins": { "0.9": { "n": 70, "errors": 6, "error_rate": 0.08571428571428572 }, "0.95": { "n": 65, "errors": 5, "error_rate": 0.07692307692307693 }, "0.99": { "n": 43, "errors": 1, "error_rate": 0.023255813953488372 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.975609756097561, "confidence_cutoff": 0.9924227590707956 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9230769230769231, "confidence_cutoff": 0.9525741267757555 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.3899038705435832, "acc": 0.8875, "ece": 0.10104432041975961, "brier": 0.20088637738716225, "mean_conf": 0.9519886566878337, "confident_error_rate": 0.075, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.9142857142857143, "coverage_at_5pct_error": 0.6125, "coverage_at_1pct_error": 0.2625, "aurc": 0.04244121984541574, "error_rate_at_0_9": 0.08571428571428572, "confidence_bias": 0.06448865668783377, "top_bins": { "0.9": { "n": 70, "errors": 6, "error_rate": 0.08571428571428572 }, "0.95": { "n": 65, "errors": 5, "error_rate": 0.07692307692307693 }, "0.99": { "n": 43, "errors": 1, "error_rate": 0.023255813953488372 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.975609756097561, "confidence_cutoff": 0.9924227590707956 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9230769230769231, "confidence_cutoff": 0.9525741267757555 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 72, "accuracy": 0.9, "full_probability_metrics": { "n": 80, "nll": 0.23988130350222256, "acc": 0.9, "ece": 0.04717481310206811, "brier": 0.13097838729486525, "mean_conf": 0.9392900690062236, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.9565217391304348, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.3, "aurc": 0.02645715938381179, "error_rate_at_0_9": 0.043478260869565216, "confidence_bias": 0.03929006900622356, "top_bins": { "0.9": { "n": 69, "errors": 3, "error_rate": 0.043478260869565216 }, "0.95": { "n": 56, "errors": 2, "error_rate": 0.03571428571428571 }, "0.99": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.9761904761904762, "confidence_cutoff": 0.9859363729731176 }, "0.8": { "coverage": 0.8375, "accuracy": 0.9552238805970149, "confidence_cutoff": 0.9241418242454529 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.23988130350222256, "acc": 0.9, "ece": 0.04717481310206811, "brier": 0.13097838729486525, "mean_conf": 0.9392900690062236, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.9565217391304348, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.3, "aurc": 0.02645715938381179, "error_rate_at_0_9": 0.043478260869565216, "confidence_bias": 0.03929006900622356, "top_bins": { "0.9": { "n": 69, "errors": 3, "error_rate": 0.043478260869565216 }, "0.95": { "n": 56, "errors": 2, "error_rate": 0.03571428571428571 }, "0.99": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.9761904761904762, "confidence_cutoff": 0.9859363729731176 }, "0.8": { "coverage": 0.8375, "accuracy": 0.9552238805970149, "confidence_cutoff": 0.9241418242454529 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 80, "accuracy": 1.0, "full_probability_metrics": { "n": 80, "nll": 0.0010838270645854098, "acc": 1.0, "ece": 0.001078861529081765, "brier": 1.7230418535925406e-05, "mean_conf": 0.9989211384709182, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.001078861529081765, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 79, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9996314230598986 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9989767935742468 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.0010838270645854098, "acc": 1.0, "ece": 0.001078861529081765, "brier": 1.7230418535925406e-05, "mean_conf": 0.9989211384709182, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.001078861529081765, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 79, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9996314230598986 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9989767935742468 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 67, "accuracy": 0.8375, "full_probability_metrics": { "n": 80, "nll": 0.5337836382608976, "acc": 0.8375, "ece": 0.12558235423232683, "brier": 0.2913576205828143, "mean_conf": 0.9065503684769107, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.725, "accuracy_at_0_9": 0.8793103448275862, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.35, "aurc": 0.07993693690207211, "error_rate_at_0_9": 0.1206896551724138, "confidence_bias": 0.0690503684769107, "top_bins": { "0.9": { "n": 58, "errors": 7, "error_rate": 0.1206896551724138 }, "0.95": { "n": 48, "errors": 6, "error_rate": 0.125 }, "0.99": { "n": 33, "errors": 2, "error_rate": 0.06060606060606061 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.8809523809523809, "confidence_cutoff": 0.9706877686982103 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8461538461538461, "confidence_cutoff": 0.8354835385757158 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.5337836382608976, "acc": 0.8375, "ece": 0.12558235423232683, "brier": 0.2913576205828143, "mean_conf": 0.9065503684769107, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.725, "accuracy_at_0_9": 0.8793103448275862, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.35, "aurc": 0.07993693690207211, "error_rate_at_0_9": 0.1206896551724138, "confidence_bias": 0.0690503684769107, "top_bins": { "0.9": { "n": 58, "errors": 7, "error_rate": 0.1206896551724138 }, "0.95": { "n": 48, "errors": 6, "error_rate": 0.125 }, "0.99": { "n": 33, "errors": 2, "error_rate": 0.06060606060606061 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.8809523809523809, "confidence_cutoff": 0.9706877686982103 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8461538461538461, "confidence_cutoff": 0.8354835385757158 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.006361165570785745, "acc": 1.0, "ece": 0.006340649093316886, "brier": 8.170670611759999e-05, "mean_conf": 0.9936593509066831, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.006340649093316886, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9940889314605925 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.993307149199782 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.006361165570785745, "acc": 1.0, "ece": 0.006340649093316886, "brier": 8.170670611759999e-05, "mean_conf": 0.9936593509066831, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.006340649093316886, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9940889314605925 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.993307149199782 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.014000720687219628, "acc": 1.0, "ece": 0.013874889411573221, "brier": 0.0004964926336892869, "mean_conf": 0.9861251105884268, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013874889411573221, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9902915229424077 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 0.97966764715126 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.014000720687219628, "acc": 1.0, "ece": 0.013874889411573221, "brier": 0.0004964926336892869, "mean_conf": 0.9861251105884268, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013874889411573221, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9902915229424077 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 0.97966764715126 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.5499913595249935, "acc": 0.8, "ece": 0.07949275944121263, "brier": 0.3225632146632617, "mean_conf": 0.7850458875613239, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.11817460317460318, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01495411243867617, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8172865784003556 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6945120692253113 } }, "score_mae": 0.3535677912101932, "ranked_probability_score": 0.09142082295275453 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5499913595249935, "acc": 0.8, "ece": 0.07949275944121263, "brier": 0.3225632146632617, "mean_conf": 0.7850458875613239, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.11817460317460318, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01495411243867617, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8172865784003556 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6945120692253113 } }, "score_mae": 0.3535677912101932, "ranked_probability_score": 0.09142082295275453 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.13180478256317213, "acc": 0.9, "ece": 0.07572427928415638, "brier": 0.09043185253212663, "mean_conf": 0.9520401416923654, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05204014169236537, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9895335686323151 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9844943390356534 } }, "score_mae": 0.06939199553636667, "ranked_probability_score": 0.02070671718386953 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.13180478256317213, "acc": 0.9, "ece": 0.07572427928415638, "brier": 0.09043185253212663, "mean_conf": 0.9520401416923654, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05204014169236537, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9895335686323151 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9844943390356534 } }, "score_mae": 0.06939199553636667, "ranked_probability_score": 0.02070671718386953 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.3082408664009614, "acc": 0.7, "ece": 0.21418062632596396, "brier": 0.2032299792668082, "mean_conf": 0.8158124352222366, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.06472222222222221, "error_rate_at_0_9": 0.0, "confidence_bias": 0.11581243522223661, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.951518493732782 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5822461594051465 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.3082408664009614, "acc": 0.7, "ece": 0.21418062632596396, "brier": 0.2032299792668082, "mean_conf": 0.8158124352222366, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.06472222222222221, "error_rate_at_0_9": 0.0, "confidence_bias": 0.11581243522223661, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.951518493732782 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5822461594051465 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 1.3293358839023506, "acc": 0.6, "ece": 0.37709240120231285, "brier": 0.7444595528107806, "mean_conf": 0.9367297570067562, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5015873015873016, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.3367297570067562, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 5, "errors": 3, "error_rate": 0.6 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9525741267757555 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9324533025161416 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3293358839023506, "acc": 0.6, "ece": 0.37709240120231285, "brier": 0.7444595528107806, "mean_conf": 0.9367297570067562, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5015873015873016, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.3367297570067562, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 5, "errors": 3, "error_rate": 0.6 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9525741267757555 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9324533025161416 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.5626398170261162, "acc": 0.8, "ece": 0.2943158930523678, "brier": 0.3338338690272229, "mean_conf": 0.7274849300044994, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09456349206349206, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07251506999550061, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8517094524187235 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5682692954459997 } }, "score_mae": 0.4016641578843407, "ranked_probability_score": 0.09978695634355816 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5626398170261162, "acc": 0.8, "ece": 0.2943158930523678, "brier": 0.3338338690272229, "mean_conf": 0.7274849300044994, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09456349206349206, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07251506999550061, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8517094524187235 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5682692954459997 } }, "score_mae": 0.4016641578843407, "ranked_probability_score": 0.09978695634355816 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.229409142592422, "acc": 0.5, "ece": 0.3799647011309829, "brier": 0.7242260695149491, "mean_conf": 0.879964701130983, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.22813492063492066, "error_rate_at_0_9": 0.2, "confidence_bias": 0.379964701130983, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9722628303687921 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7978758542859345 } }, "score_mae": 0.4242083162339035, "ranked_probability_score": 0.1745291876512243 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.229409142592422, "acc": 0.5, "ece": 0.3799647011309829, "brier": 0.7242260695149491, "mean_conf": 0.879964701130983, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.22813492063492066, "error_rate_at_0_9": 0.2, "confidence_bias": 0.379964701130983, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9722628303687921 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7978758542859345 } }, "score_mae": 0.4242083162339035, "ranked_probability_score": 0.1745291876512243 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0016131556787685366, "acc": 1.0, "ece": 0.001611314425340682, "brier": 5.5704722134473615e-06, "mean_conf": 0.9983886855746593, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.001611314425340682, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9989947923792518 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 0.9971777678976348 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0016131556787685366, "acc": 1.0, "ece": 0.001611314425340682, "brier": 5.5704722134473615e-06, "mean_conf": 0.9983886855746593, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.001611314425340682, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9989947923792518 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 0.9971777678976348 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.00860057924402766, "acc": 1.0, "ece": 0.008555711891411688, "brier": 0.00013797902504154202, "mean_conf": 0.9914442881085883, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.008555711891411688, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9927829127238098 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9905250847259756 } }, "score_mae": 0.01029645649473793, "ranked_probability_score": 4.656414038594634e-05 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.00860057924402766, "acc": 1.0, "ece": 0.008555711891411688, "brier": 0.00013797902504154202, "mean_conf": 0.9914442881085883, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.008555711891411688, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9927829127238098 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9905250847259756 } }, "score_mae": 0.01029645649473793, "ranked_probability_score": 4.656414038594634e-05 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 0.947032798197972, "acc": 0.6, "ece": 0.2450743781787131, "brier": 0.5729990895285746, "mean_conf": 0.4719506974204644, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.35599206349206347, "error_rate_at_0_9": null, "confidence_bias": -0.12804930257953556, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.42112735383943595 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.41360626139603485 } }, "score_mae": 0.49387562676796987, "ranked_probability_score": 0.1603654679025857 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.947032798197972, "acc": 0.6, "ece": 0.2450743781787131, "brier": 0.5729990895285746, "mean_conf": 0.4719506974204644, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.35599206349206347, "error_rate_at_0_9": null, "confidence_bias": -0.12804930257953556, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.42112735383943595 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.41360626139603485 } }, "score_mae": 0.49387562676796987, "ranked_probability_score": 0.1603654679025857 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.7573148148148149, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.75, "flip_rate": 0.1111111111111111, "covered_only_flip_rate": 0.1111111111111111, "half_l1": 0.11571143278369668 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.6762044228653428, "share_at_0_9": 0.07272727272727272, "normalized_entropy": 0.7584741541721761 }, "covered_only_confidence_metrics": { "mean_max_p": 0.6762044228653428, "share_at_0_9": 0.07272727272727272, "normalized_entropy": 0.7584741541721761 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 89, "accuracy": 0.8090909090909091, "full_probability_metrics": { "n": 110, "nll": 0.46263911558079907, "acc": 0.8090909090909091, "ece": 0.10823636302008073, "brier": 0.27204230692552595, "mean_conf": 0.8671496350197261, "confident_error_rate": 0.045454545454545456, "coverage_at_0_9": 0.6545454545454545, "accuracy_at_0_9": 0.9305555555555556, "coverage_at_5pct_error": 0.5727272727272728, "coverage_at_1pct_error": 0.5, "aurc": 0.054046347056566836, "error_rate_at_0_9": 0.06944444444444445, "confidence_bias": 0.058058725928817, "top_bins": { "0.9": { "n": 72, "errors": 5, "error_rate": 0.06944444444444445 }, "0.95": { "n": 65, "errors": 4, "error_rate": 0.06153846153846154 }, "0.99": { "n": 38, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9754419940997697 }, "0.8": { "coverage": 0.8, "accuracy": 0.8863636363636364, "confidence_cutoff": 0.7362475450057353 } }, "score_mae": 0.29216739068791864, "ranked_probability_score": 0.09114261936239637 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.46263911558079907, "acc": 0.8090909090909091, "ece": 0.10823636302008073, "brier": 0.27204230692552595, "mean_conf": 0.8671496350197261, "confident_error_rate": 0.045454545454545456, "coverage_at_0_9": 0.6545454545454545, "accuracy_at_0_9": 0.9305555555555556, "coverage_at_5pct_error": 0.5727272727272728, "coverage_at_1pct_error": 0.5, "aurc": 0.054046347056566836, "error_rate_at_0_9": 0.06944444444444445, "confidence_bias": 0.058058725928817, "top_bins": { "0.9": { "n": 72, "errors": 5, "error_rate": 0.06944444444444445 }, "0.95": { "n": 65, "errors": 4, "error_rate": 0.06153846153846154 }, "0.99": { "n": 38, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9754419940997697 }, "0.8": { "coverage": 0.8, "accuracy": 0.8863636363636364, "confidence_cutoff": 0.7362475450057353 } }, "score_mae": 0.29216739068791864, "ranked_probability_score": 0.09114261936239637 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.190945212154383 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.5961610297247668, "paired_flip": { "pairs": 64, "flip_rate": 0.734375, "both_correct_rate": 0.578125, "invariant_pairs": 24, "invariance_rate": 0.9583333333333334, "invariant_both_correct_rate": 0.5416666666666666 }, "unknowable": { "n": 110, "mean_max_p": 0.6762044228653428, "share_at_0_9": 0.07272727272727272, "control_mean_max_p": 0.8671496350197261, "control_share_at_0_9": 0.6545454545454545, "control_acc": 0.8090909090909091, "paired_confidence_drop": 0.190945212154383, "share_less_confident_than_control": 0.8818181818181818 }, "clean": { "n": 1046, "nll": 0.759729403716022, "acc": 0.7323135755258127, "ece": 0.09750832444229235, "brier": 0.3673011918808915, "mean_conf": 0.8291984136018093, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.5688336520076482, "accuracy_at_0_9": 0.9159663865546218, "coverage_at_5pct_error": 0.47418738049713194, "coverage_at_1pct_error": 0.2084130019120459, "aurc": 0.0872044723520789, "error_rate_at_0_9": 0.08403361344537816, "confidence_bias": 0.09688483807599668, "top_bins": { "0.9": { "n": 595, "errors": 50, "error_rate": 0.08403361344537816 }, "0.95": { "n": 523, "errors": 30, "error_rate": 0.05736137667304015 }, "0.99": { "n": 327, "errors": 7, "error_rate": 0.021406727828746176 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9426386233269598, "confidence_cutoff": 0.951518493732782 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.828978622327791, "confidence_cutoff": 0.6224593334657152 } }, "score_mae": 0.28987980749634135, "ranked_probability_score": 0.08782854063750568 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.6789716954193772, "acc": 0.4, "ece": 0.36573305686207336, "brier": 0.8254645361260712, "mean_conf": 0.6646489117250975, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5415913869841888, "error_rate_at_0_9": 0.5, "confidence_bias": 0.2646489117250975, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6553088766308754 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.539919199319293 } } }, "buried_paws": { "n": 20, "nll": 0.41364062147400277, "acc": 0.8, "ece": 0.17365084393507335, "brier": 0.25633054757351986, "mean_conf": 0.9136674185543736, "confident_error_rate": 0.1, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.8461538461538461, "coverage_at_5pct_error": 0.45, "coverage_at_1pct_error": 0.45, "aurc": 0.06957397263898812, "error_rate_at_0_9": 0.15384615384615385, "confidence_bias": 0.11366741855437357, "top_bins": { "0.9": { "n": 13, "errors": 2, "error_rate": 0.15384615384615385 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9579122733914276 }, "0.8": { "coverage": 0.85, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.8807970744787121 } } }, "buried_qnli": { "n": 20, "nll": 0.2921434213238844, "acc": 0.95, "ece": 0.08453801695369767, "brier": 0.13339955861821834, "mean_conf": 0.9265174852468341, "confident_error_rate": 0.05, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.35, "aurc": 0.04986533783553907, "error_rate_at_0_9": 0.0625, "confidence_bias": -0.023482514753165873, "top_bins": { "0.9": { "n": 16, "errors": 1, "error_rate": 0.0625 }, "0.95": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9770226296592014 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.9046505363181393 } } }, "buried_tweet_offensive": { "n": 20, "nll": 1.2617601328295882, "acc": 0.45, "ece": 0.34399971383452643, "brier": 0.834135763504744, "mean_conf": 0.7640741993133061, "confident_error_rate": 0.2, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.617740895585323, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.3140741993133061, "top_bins": { "0.9": { "n": 6, "errors": 4, "error_rate": 0.6666666666666666 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.2727272727272727, "confidence_cutoff": 0.7772998578161043 }, "0.8": { "coverage": 0.85, "accuracy": 0.35294117647058826, "confidence_cutoff": 0.5926665906508809 } } }, "composition_final_combination": { "n": 32, "nll": 0.6027086688012528, "acc": 0.53125, "ece": 0.2245713627825465, "brier": 0.41413801251586274, "mean_conf": 0.7009620316433398, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.22099202452059225, "error_rate_at_0_9": 0.25, "confidence_bias": 0.1697120316433398, "top_bins": { "0.9": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8125, "confidence_cutoff": 0.7057850360870361 }, "0.8": { "coverage": 0.90625, "accuracy": 0.5862068965517241, "confidence_cutoff": 0.5312093775743617 } } }, "composition_final_exception": { "n": 32, "nll": 0.7895514499373273, "acc": 0.59375, "ece": 0.31475717153147065, "brier": 0.574466147198007, "mean_conf": 0.7698756301405588, "confident_error_rate": 0.0, "coverage_at_0_9": 0.21875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.21875, "coverage_at_1pct_error": 0.21875, "aurc": 0.3271882885353148, "error_rate_at_0_9": 0.0, "confidence_bias": 0.17612563014055882, "top_bins": { "0.9": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5625, "accuracy": 0.6111111111111112, "confidence_cutoff": 0.7549149877590688 }, "0.8": { "coverage": 0.84375, "accuracy": 0.5185185185185185, "confidence_cutoff": 0.6791787147521973 } } }, "composition_final_negation": { "n": 32, "nll": 0.5997900737237437, "acc": 0.625, "ece": 0.10364475153777164, "brier": 0.4149719922019578, "mean_conf": 0.6595295351290815, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.21459078183057648, "error_rate_at_0_9": null, "confidence_bias": 0.03452953512908152, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.875, "accuracy": 0.6785714285714286, "confidence_cutoff": 0.5621765088386383 } } }, "contrastive_authorization": { "n": 40, "nll": 0.012996411860518692, "acc": 1.0, "ece": 0.0128793991091124, "brier": 0.0004613228907562773, "mean_conf": 0.9871206008908876, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0128793991091124, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9914225150110113 }, "0.8": { "coverage": 0.85, "accuracy": 1.0, "confidence_cutoff": 0.97966764715126 } } }, "contrastive_deadline": { "n": 40, "nll": 0.49745983599501137, "acc": 0.775, "ece": 0.13089066802312613, "brier": 0.3050484395003504, "mean_conf": 0.8347478671121624, "confident_error_rate": 0.025, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.375, "aurc": 0.08069818256712594, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.0597478671121624, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.8607219650012694 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7295618923204291 } }, "score_mae": 0.28644843270897535, "ranked_probability_score": 0.08285742255016963 }, "emotion": { "n": 80, "nll": 1.3187849053528407, "acc": 0.525, "ece": 0.20447016219613473, "brier": 0.5881928917298136, "mean_conf": 0.7294701621961348, "confident_error_rate": 0.075, "coverage_at_0_9": 0.3625, "accuracy_at_0_9": 0.7931034482758621, "coverage_at_5pct_error": 0.0375, "coverage_at_1pct_error": 0.0375, "aurc": 0.25124869903151126, "error_rate_at_0_9": 0.20689655172413793, "confidence_bias": 0.20447016219613479, "top_bins": { "0.9": { "n": 29, "errors": 6, "error_rate": 0.20689655172413793 }, "0.95": { "n": 21, "errors": 3, "error_rate": 0.14285714285714285 }, "0.99": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.7771893073791486 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.5038760210596767 } } }, "mmlu": { "n": 80, "nll": 0.7246171543085598, "acc": 0.7375, "ece": 0.12236678796568477, "brier": 0.359554531615567, "mean_conf": 0.8571929488972623, "confident_error_rate": 0.025, "coverage_at_0_9": 0.5875, "accuracy_at_0_9": 0.9574468085106383, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.525, "aurc": 0.06346347577428549, "error_rate_at_0_9": 0.0425531914893617, "confidence_bias": 0.11969294889726223, "top_bins": { "0.9": { "n": 47, "errors": 2, "error_rate": 0.0425531914893617 }, "0.95": { "n": 44, "errors": 1, "error_rate": 0.022727272727272728 }, "0.99": { "n": 27, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9615272075352188 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.6888570836846313 } } }, "mmlu_pro": { "n": 200, "nll": 1.6502405207825164, "acc": 0.535, "ece": 0.14518812226380381, "brier": 0.6523599717495105, "mean_conf": 0.6742868197918527, "confident_error_rate": 0.05, "coverage_at_0_9": 0.315, "accuracy_at_0_9": 0.8412698412698413, "coverage_at_5pct_error": 0.21, "coverage_at_1pct_error": 0.085, "aurc": 0.24665741690802748, "error_rate_at_0_9": 0.15873015873015872, "confidence_bias": 0.13928681979185265, "top_bins": { "0.9": { "n": 63, "errors": 10, "error_rate": 0.15873015873015872 }, "0.95": { "n": 52, "errors": 4, "error_rate": 0.07692307692307693 }, "0.99": { "n": 37, "errors": 2, "error_rate": 0.05405405405405406 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.72, "confidence_cutoff": 0.7254660404027048 }, "0.8": { "coverage": 0.8, "accuracy": 0.6, "confidence_cutoff": 0.3802954477895661 } } }, "paws": { "n": 80, "nll": 0.3899038705435832, "acc": 0.8875, "ece": 0.10104432041975961, "brier": 0.20088637738716225, "mean_conf": 0.9519886566878337, "confident_error_rate": 0.075, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.9142857142857143, "coverage_at_5pct_error": 0.6125, "coverage_at_1pct_error": 0.2625, "aurc": 0.04244121984541574, "error_rate_at_0_9": 0.08571428571428572, "confidence_bias": 0.06448865668783377, "top_bins": { "0.9": { "n": 70, "errors": 6, "error_rate": 0.08571428571428572 }, "0.95": { "n": 65, "errors": 5, "error_rate": 0.07692307692307693 }, "0.99": { "n": 43, "errors": 1, "error_rate": 0.023255813953488372 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.975609756097561, "confidence_cutoff": 0.9924227590707956 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9230769230769231, "confidence_cutoff": 0.9525741267757555 } } }, "qnli": { "n": 80, "nll": 0.23988130350222256, "acc": 0.9, "ece": 0.04717481310206811, "brier": 0.13097838729486525, "mean_conf": 0.9392900690062236, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.9565217391304348, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.3, "aurc": 0.02645715938381179, "error_rate_at_0_9": 0.043478260869565216, "confidence_bias": 0.03929006900622356, "top_bins": { "0.9": { "n": 69, "errors": 3, "error_rate": 0.043478260869565216 }, "0.95": { "n": 56, "errors": 2, "error_rate": 0.03571428571428571 }, "0.99": { "n": 29, "errors": 1, "error_rate": 0.034482758620689655 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.9761904761904762, "confidence_cutoff": 0.9859363729731176 }, "0.8": { "coverage": 0.8375, "accuracy": 0.9552238805970149, "confidence_cutoff": 0.9241418242454529 } } }, "sciq": { "n": 80, "nll": 0.0010838270645854098, "acc": 1.0, "ece": 0.001078861529081765, "brier": 1.7230418535925406e-05, "mean_conf": 0.9989211384709182, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.001078861529081765, "top_bins": { "0.9": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 80, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 79, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9996314230598986 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9989767935742468 } } }, "tweet_offensive": { "n": 80, "nll": 0.5337836382608976, "acc": 0.8375, "ece": 0.12558235423232683, "brier": 0.2913576205828143, "mean_conf": 0.9065503684769107, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.725, "accuracy_at_0_9": 0.8793103448275862, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.35, "aurc": 0.07993693690207211, "error_rate_at_0_9": 0.1206896551724138, "confidence_bias": 0.0690503684769107, "top_bins": { "0.9": { "n": 58, "errors": 7, "error_rate": 0.1206896551724138 }, "0.95": { "n": 48, "errors": 6, "error_rate": 0.125 }, "0.99": { "n": 33, "errors": 2, "error_rate": 0.06060606060606061 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.8809523809523809, "confidence_cutoff": 0.9706877686982103 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8461538461538461, "confidence_cutoff": 0.8354835385757158 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 1.1617664184543566, "acc": 0.5, "ece": 0.3746208526956175, "brier": 0.7887626483114705, "mean_conf": 0.8746208526956176, "confident_error_rate": 0.2, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3746208526956176, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8807970744787121 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8807970744787121 } } }, "unknowable_authorization": { "n": 10, "nll": 0.6934488230472001, "acc": 0.5, "ece": 0.1867687267452515, "brier": 0.5018159944322667, "mean_conf": 0.6122351493109085, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4, "error_rate_at_0_9": null, "confidence_bias": 0.1122351493109085, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5926665906508809 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5926665906508809 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.006361165570785745, "acc": 1.0, "ece": 0.006340649093316886, "brier": 8.170670611759999e-05, "mean_conf": 0.9936593509066831, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.006340649093316886, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9940889314605925 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.993307149199782 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.014000720687219628, "acc": 1.0, "ece": 0.013874889411573221, "brier": 0.0004964926336892869, "mean_conf": 0.9861251105884268, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.013874889411573221, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9902915229424077 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 0.97966764715126 } } }, "unknowable_control_deadline": { "n": 10, "nll": 0.5499913595249935, "acc": 0.8, "ece": 0.07949275944121263, "brier": 0.3225632146632617, "mean_conf": 0.7850458875613239, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.11817460317460318, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01495411243867617, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8172865784003556 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6945120692253113 } }, "score_mae": 0.3535677912101932, "ranked_probability_score": 0.09142082295275453 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.13180478256317213, "acc": 0.9, "ece": 0.07572427928415638, "brier": 0.09043185253212663, "mean_conf": 0.9520401416923654, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05204014169236537, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9895335686323151 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9844943390356534 } }, "score_mae": 0.06939199553636667, "ranked_probability_score": 0.02070671718386953 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.3082408664009614, "acc": 0.7, "ece": 0.21418062632596396, "brier": 0.2032299792668082, "mean_conf": 0.8158124352222366, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.06472222222222221, "error_rate_at_0_9": 0.0, "confidence_bias": 0.11581243522223661, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.951518493732782 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5822461594051465 } } }, "unknowable_control_return_window": { "n": 10, "nll": 1.3293358839023506, "acc": 0.6, "ece": 0.37709240120231285, "brier": 0.7444595528107806, "mean_conf": 0.9367297570067562, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5015873015873016, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.3367297570067562, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 5, "errors": 3, "error_rate": 0.6 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9525741267757555 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9324533025161416 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.5626398170261162, "acc": 0.8, "ece": 0.2943158930523678, "brier": 0.3338338690272229, "mean_conf": 0.7274849300044994, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09456349206349206, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07251506999550061, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8517094524187235 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5682692954459997 } }, "score_mae": 0.4016641578843407, "ranked_probability_score": 0.09978695634355816 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.229409142592422, "acc": 0.5, "ece": 0.3799647011309829, "brier": 0.7242260695149491, "mean_conf": 0.879964701130983, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.22813492063492066, "error_rate_at_0_9": 0.2, "confidence_bias": 0.379964701130983, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9722628303687921 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7978758542859345 } }, "score_mae": 0.4242083162339035, "ranked_probability_score": 0.1745291876512243 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.0016131556787685366, "acc": 1.0, "ece": 0.001611314425340682, "brier": 5.5704722134473615e-06, "mean_conf": 0.9983886855746593, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.001611314425340682, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9989947923792518 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 0.9971777678976348 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.00860057924402766, "acc": 1.0, "ece": 0.008555711891411688, "brier": 0.00013797902504154202, "mean_conf": 0.9914442881085883, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.008555711891411688, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9927829127238098 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9905250847259756 } }, "score_mae": 0.01029645649473793, "ranked_probability_score": 4.656414038594634e-05 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.947032798197972, "acc": 0.6, "ece": 0.2450743781787131, "brier": 0.5729990895285746, "mean_conf": 0.4719506974204644, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.35599206349206347, "error_rate_at_0_9": null, "confidence_bias": -0.12804930257953556, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.42112735383943595 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.41360626139603485 } }, "score_mae": 0.49387562676796987, "ranked_probability_score": 0.1603654679025857 }, "unknowable_deadline": { "n": 10, "nll": 1.111479690675203, "acc": 0.5, "ece": 0.2701362003390988, "brier": 0.6777891672314708, "mean_conf": 0.689918084300601, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.406031746031746, "error_rate_at_0_9": null, "confidence_bias": 0.18991808430060098, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6945120665403659 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6572184366816748 } }, "score_mae": 0.6964875653284339, "ranked_probability_score": 0.2251298978393616 }, "unknowable_late_fee": { "n": 10, "nll": 1.3351808700811403, "acc": 0.3, "ece": 0.30125312968887985, "brier": 0.8211778133661978, "mean_conf": 0.6012531296888797, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6733333333333332, "error_rate_at_0_9": null, "confidence_bias": 0.3012531296888797, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.6044544813362073 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5897976668209185 } }, "score_mae": 0.7402443442604068, "ranked_probability_score": 0.26067790489946385 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.5266278803101554, "acc": 0.5, "ece": 0.3606998514391655, "brier": 0.8351449236748334, "mean_conf": 0.8606998514391655, "confident_error_rate": 0.2, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3606998514391655, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8338453546395311 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8215017685965494 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7534862247535351, "acc": 0.5, "ece": 0.23221910442101817, "brier": 0.5570931513854337, "mean_conf": 0.6446544061887459, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5166666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.14465440618874592, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.9, "accuracy": 0.4444444444444444, "confidence_cutoff": 0.6224593334657152 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.3554790630562707, "acc": 0.2, "ece": 0.3814446017153069, "brier": 0.8500621511547465, "mean_conf": 0.5814446017153069, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6195238095238095, "error_rate_at_0_9": null, "confidence_bias": 0.38144460171530686, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5761168851782748 }, "0.8": { "coverage": 1.0, "accuracy": 0.2, "confidence_cutoff": 0.560305229420882 } }, "score_mae": 0.9006705867057496, "ranked_probability_score": 0.3130876507987639 }, "unknowable_sla_response": { "n": 10, "nll": 1.560345332204161, "acc": 0.2, "ece": 0.4243019792654225, "brier": 0.9595777833467798, "mean_conf": 0.6243019792654225, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9016666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.4243019792654225, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.6097590145540531 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.598909419807511 } }, "score_mae": 0.748417139379927, "ranked_probability_score": 0.28739526770036344 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.9765844397389658, "acc": 0.5, "ece": 0.2599291902287565, "brier": 0.6671851095363397, "mean_conf": 0.7599291902287566, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.2599291902287566, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.7240769836154692 }, "0.8": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.7240769836154692 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.4363601630761345, "acc": 0.4, "ece": 0.3570773245067516, "brier": 0.8666986425616308, "mean_conf": 0.7570773245067517, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.545, "error_rate_at_0_9": null, "confidence_bias": 0.35707732450675167, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.7856768311137197 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7506115772103479 } }, "score_mae": 0.920561476970977, "ranked_probability_score": 0.34486101722647494 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.1155900042356717, "acc": 0.3, "ece": 0.2431851270555882, "brier": 0.6854298471207794, "mean_conf": 0.4321140821786173, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7289285714285715, "error_rate_at_0_9": null, "confidence_bias": 0.1321140821786173, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.4359540045261383 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3616644524535863 } }, "score_mae": 0.572711808576579, "ranked_probability_score": 0.20032440415910474 } }, "variants": { "clean": { "n": 1156, "nll": 0.800121492546096, "acc": 0.7006920415224913, "ece": 0.1145122784444857, "brier": 0.40337752511127334, "mean_conf": 0.8146401618881318, "confident_error_rate": 0.04671280276816609, "coverage_at_0_9": 0.5216262975778547, "accuracy_at_0_9": 0.9104477611940298, "coverage_at_5pct_error": 0.4290657439446367, "coverage_at_1pct_error": 0.18858131487889274, "aurc": 0.1054878936328668, "error_rate_at_0_9": 0.08955223880597014, "confidence_bias": 0.1139481203656405, "top_bins": { "0.9": { "n": 603, "errors": 54, "error_rate": 0.08955223880597014 }, "0.95": { "n": 523, "errors": 30, "error_rate": 0.05736137667304015 }, "0.99": { "n": 327, "errors": 7, "error_rate": 0.021406727828746176 } }, "selective": { "0.5": { "coverage": 0.5008650519031141, "accuracy": 0.9222797927461139, "confidence_cutoff": 0.9233054163423848 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.7956756756756757, "confidence_cutoff": 0.6020724717334613 } }, "score_mae": 0.4673681872615928, "ranked_probability_score": 0.1568600968124118 }, "none_absent": { "n": 36, "nll": 1.1677094943790083, "acc": 0.6388888888888888, "ece": 0.2898932195884103, "brier": 0.577996742618921, "mean_conf": 0.7958521096356145, "confident_error_rate": 0.1111111111111111, "coverage_at_0_9": 0.3888888888888889, "accuracy_at_0_9": 0.7142857142857143, "coverage_at_5pct_error": 0.1388888888888889, "coverage_at_1pct_error": 0.1388888888888889, "aurc": 0.28332455202798684, "error_rate_at_0_9": 0.2857142857142857, "confidence_bias": 0.15696322074672564, "top_bins": { "0.9": { "n": 14, "errors": 4, "error_rate": 0.2857142857142857 }, "0.95": { "n": 12, "errors": 4, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5555555555555556, "confidence_cutoff": 0.8108400453800727 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.6896551724137931, "confidence_cutoff": 0.6582762051990345 } } }, "none_present": { "n": 36, "nll": 0.8029942473820655, "acc": 0.7222222222222222, "ece": 0.1100631787339853, "brier": 0.35388665296507843, "mean_conf": 0.8142509832674087, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4722222222222222, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6388888888888888, "coverage_at_1pct_error": 0.5833333333333334, "aurc": 0.06660800274951266, "error_rate_at_0_9": 0.0, "confidence_bias": 0.09202876104518654, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8999175369690381 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8620689655172413, "confidence_cutoff": 0.6852506727161554 } } }, "permuted": { "n": 36, "nll": 0.6903983270726369, "acc": 0.7777777777777778, "ece": 0.09236168637528684, "brier": 0.2941021515352798, "mean_conf": 0.8050676740520326, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5555555555555556, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7222222222222222, "coverage_at_1pct_error": 0.5833333333333334, "aurc": 0.04779573836129742, "error_rate_at_0_9": 0.0, "confidence_bias": 0.02728989627425482, "top_bins": { "0.9": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9338906494331607 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.896551724137931, "confidence_cutoff": 0.5437619150850391 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.10558767445885621, "flip_rate": 0.1111111111111111 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.759729403716022, "acc": 0.7323135755258127, "ece": 0.09750832444229235, "brier": 0.3673011918808915, "mean_conf": 0.8291984136018093, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.5688336520076482, "accuracy_at_0_9": 0.9159663865546218, "coverage_at_5pct_error": 0.47418738049713194, "coverage_at_1pct_error": 0.2084130019120459, "aurc": 0.0872044723520789, "error_rate_at_0_9": 0.08403361344537816, "confidence_bias": 0.09688483807599668, "top_bins": { "0.9": { "n": 595, "errors": 50, "error_rate": 0.08403361344537816 }, "0.95": { "n": 523, "errors": 30, "error_rate": 0.05736137667304015 }, "0.99": { "n": 327, "errors": 7, "error_rate": 0.021406727828746176 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9426386233269598, "confidence_cutoff": 0.951518493732782 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.828978622327791, "confidence_cutoff": 0.6224593334657152 } }, "score_mae": 0.28987980749634135, "ranked_probability_score": 0.08782854063750568 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Decider": { "panels": { "old_core": { "point": 0.6401413690476191, "exact_fraction": "17207/26880", "ci95": [ 0.5992559523809524, 0.6806184895833334 ] }, "v3_core": { "point": 0.4658333333333333, "exact_fraction": "559/1200", "ci95": [ 0.4362500000000001, 0.49499999999999994 ] }, "v4": { "point": 0.9203125, "exact_fraction": "589/640", "ci95": [ 0.8903095518867924, 0.9479299363057325 ] }, "v5": { "point": 0.84375, "exact_fraction": "27/32", "ci95": [ 0.8130660699474849, 0.8734622752789641 ] }, "transfer_v9_test": { "point": 0.6931166347992351, "exact_fraction": "725/1046", "ci95": [ 0.6616537810123801, 0.724007561436673 ] }, "original_four_panel_mean": { "point": 0.7175093005952381, "exact_fraction": "385733/537600", "ci95": [ 0.7012097728950374, 0.7336543123969403 ] }, "weighted_mean": { "point": 0.6770776142675043, "exact_fraction": "23796299/35145600", "ci95": [ 0.6609690392469958, 0.6933630662445373 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 111, "requested": 128 }, "boolean_constraints": { "correct": 46, "requested": 64 }, "dbpedia_14": { "correct": 110, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 8, "requested": 96 }, "ordinal_rubric": { "correct": 54, "requested": 64 }, "relational_composition": { "correct": 52, "requested": 96 }, "scoped_evidence": { "correct": 46, "requested": 96 }, "state_tracking": { "correct": 28, "requested": 96 }, "unknown_rejection": { "correct": 38, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 45, "requested": 80 }, "capacitated_assignment": { "correct": 41, "requested": 80 }, "constraint_assignment": { "correct": 19, "requested": 80 }, "massive_en": { "correct": 108, "requested": 120 }, "massive_zh": { "correct": 106, "requested": 120 }, "multiset_reconciliation": { "correct": 19, "requested": 80 }, "ordinal_service_loss": { "correct": 18, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 21, "requested": 80 }, "temporal_exclusion": { "correct": 34, "requested": 80 }, "transaction_recovery": { "correct": 33, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 147, "requested": 160 }, "belebele_en": { "correct": 149, "requested": 160 }, "belebele_zh": { "correct": 146, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 80, "requested": 120 }, "squad2_answerability": { "correct": 101, "requested": 120 }, "snli": { "correct": 109, "requested": 120 }, "qasc": { "correct": 115, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 725, "accuracy": 0.6931166347992351, "full_probability_metrics": { "n": 1046, "nll": 0.7969332191608542, "acc": 0.6931166347992351, "ece": 0.08090597402201222, "brier": 0.41215054236049414, "mean_conf": 0.7666068088925038, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.4483747609942639, "accuracy_at_0_9": 0.908315565031983, "coverage_at_5pct_error": 0.35181644359464626, "coverage_at_1pct_error": 0.19120458891013384, "aurc": 0.11720241935804643, "error_rate_at_0_9": 0.09168443496801706, "confidence_bias": 0.07349017409326863, "top_bins": { "0.9": { "n": 469, "errors": 43, "error_rate": 0.09168443496801706 }, "0.95": { "n": 344, "errors": 16, "error_rate": 0.046511627906976744 }, "0.99": { "n": 113, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5009560229445507, "accuracy": 0.8874045801526718, "confidence_cutoff": 0.8828 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7945041816009558, "confidence_cutoff": 0.5623999999999999 } }, "score_mae": 0.5195736273610764, "ranked_probability_score": 0.184713309982324 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.7969332191608542, "acc": 0.6931166347992351, "ece": 0.08090597402201222, "brier": 0.41215054236049414, "mean_conf": 0.7666068088925038, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.4483747609942639, "accuracy_at_0_9": 0.908315565031983, "coverage_at_5pct_error": 0.35181644359464626, "coverage_at_1pct_error": 0.19120458891013384, "aurc": 0.11720241935804643, "error_rate_at_0_9": 0.09168443496801706, "confidence_bias": 0.07349017409326863, "top_bins": { "0.9": { "n": 469, "errors": 43, "error_rate": 0.09168443496801706 }, "0.95": { "n": 344, "errors": 16, "error_rate": 0.046511627906976744 }, "0.99": { "n": 113, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5009560229445507, "accuracy": 0.8874045801526718, "confidence_cutoff": 0.8828 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7945041816009558, "confidence_cutoff": 0.5623999999999999 } }, "score_mae": 0.5195736273610764, "ranked_probability_score": 0.184713309982324 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 17, "accuracy": 0.85, "full_probability_metrics": { "n": 20, "nll": 0.4639612304098854, "acc": 0.85, "ece": 0.14167657774951575, "brier": 0.22331793650847115, "mean_conf": 0.7444834222504841, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.027718223254213963, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10551657774951584, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7637999999999998 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.6017 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.4639612304098854, "acc": 0.85, "ece": 0.14167657774951575, "brier": 0.22331793650847115, "mean_conf": 0.7444834222504841, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.027718223254213963, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10551657774951584, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7637999999999998 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.6017 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 15, "accuracy": 0.75, "full_probability_metrics": { "n": 20, "nll": 0.6012843951869397, "acc": 0.75, "ece": 0.12283500000000008, "brier": 0.38466947700000004, "mean_conf": 0.8728349999999999, "confident_error_rate": 0.1, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.1365742384550434, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.12283499999999992, "top_bins": { "0.9": { "n": 12, "errors": 2, "error_rate": 0.16666666666666666 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9241 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.7773 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.6012843951869397, "acc": 0.75, "ece": 0.12283500000000008, "brier": 0.38466947700000004, "mean_conf": 0.8728349999999999, "confident_error_rate": 0.1, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.1365742384550434, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.12283499999999992, "top_bins": { "0.9": { "n": 12, "errors": 2, "error_rate": 0.16666666666666666 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9241 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.7773 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 18, "accuracy": 0.9, "full_probability_metrics": { "n": 20, "nll": 0.34719070735248697, "acc": 0.9, "ece": 0.11635000000000006, "brier": 0.155148236, "mean_conf": 0.9124700000000001, "confident_error_rate": 0.05, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.127052733772935, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.012470000000000092, "top_bins": { "0.9": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.95": { "n": 11, "errors": 1, "error_rate": 0.09090909090909091 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9616 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.8495 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.34719070735248697, "acc": 0.9, "ece": 0.11635000000000006, "brier": 0.155148236, "mean_conf": 0.9124700000000001, "confident_error_rate": 0.05, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.127052733772935, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.012470000000000092, "top_bins": { "0.9": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.95": { "n": 11, "errors": 1, "error_rate": 0.09090909090909091 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9616 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.8495 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 16, "accuracy": 0.8, "full_probability_metrics": { "n": 20, "nll": 0.4407950321556191, "acc": 0.8, "ece": 0.07471999999999997, "brier": 0.28890536000000006, "mean_conf": 0.7631, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09617093261056109, "error_rate_at_0_9": 0.0, "confidence_bias": -0.036900000000000044, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.7422 }, "0.8": { "coverage": 0.85, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6622 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.4407950321556191, "acc": 0.8, "ece": 0.07471999999999997, "brier": 0.28890536000000006, "mean_conf": 0.7631, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09617093261056109, "error_rate_at_0_9": 0.0, "confidence_bias": -0.036900000000000044, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.7422 }, "0.8": { "coverage": 0.85, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6622 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 17, "accuracy": 0.53125, "full_probability_metrics": { "n": 32, "nll": 0.7839904683434762, "acc": 0.53125, "ece": 0.28895, "brier": 0.5575436062499999, "mean_conf": 0.8202, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.28125, "coverage_at_1pct_error": 0.28125, "aurc": 0.22343766144399238, "error_rate_at_0_9": 0.25, "confidence_bias": 0.28895000000000004, "top_bins": { "0.9": { "n": 12, "errors": 3, "error_rate": 0.25 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.8556 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6153846153846154, "confidence_cutoff": 0.7422 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.7839904683434762, "acc": 0.53125, "ece": 0.28895, "brier": 0.5575436062499999, "mean_conf": 0.8202, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.28125, "coverage_at_1pct_error": 0.28125, "aurc": 0.22343766144399238, "error_rate_at_0_9": 0.25, "confidence_bias": 0.28895000000000004, "top_bins": { "0.9": { "n": 12, "errors": 3, "error_rate": 0.25 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.8556 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6153846153846154, "confidence_cutoff": 0.7422 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 18, "accuracy": 0.5625, "full_probability_metrics": { "n": 32, "nll": 1.0710423786305578, "acc": 0.5625, "ece": 0.374315625, "brier": 0.713839373125, "mean_conf": 0.8714531249999999, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.40625, "accuracy_at_0_9": 0.5384615384615384, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.42621328369957967, "error_rate_at_0_9": 0.46153846153846156, "confidence_bias": 0.30895312499999994, "top_bins": { "0.9": { "n": 13, "errors": 6, "error_rate": 0.46153846153846156 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8969 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.8368 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 1.0710423786305578, "acc": 0.5625, "ece": 0.374315625, "brier": 0.713839373125, "mean_conf": 0.8714531249999999, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.40625, "accuracy_at_0_9": 0.5384615384615384, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.42621328369957967, "error_rate_at_0_9": 0.46153846153846156, "confidence_bias": 0.30895312499999994, "top_bins": { "0.9": { "n": 13, "errors": 6, "error_rate": 0.46153846153846156 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8969 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.8368 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 20, "accuracy": 0.625, "full_probability_metrics": { "n": 32, "nll": 0.9801228405234432, "acc": 0.625, "ece": 0.3267812499999999, "brier": 0.60622501625, "mean_conf": 0.8237375, "confident_error_rate": 0.25, "coverage_at_0_9": 0.40625, "accuracy_at_0_9": 0.38461538461538464, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5465774778050861, "error_rate_at_0_9": 0.6153846153846154, "confidence_bias": 0.1987375, "top_bins": { "0.9": { "n": 13, "errors": 8, "error_rate": 0.6153846153846154 }, "0.95": { "n": 4, "errors": 4, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.47058823529411764, "confidence_cutoff": 0.8924 }, "0.8": { "coverage": 0.84375, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.7038 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.9801228405234432, "acc": 0.625, "ece": 0.3267812499999999, "brier": 0.60622501625, "mean_conf": 0.8237375, "confident_error_rate": 0.25, "coverage_at_0_9": 0.40625, "accuracy_at_0_9": 0.38461538461538464, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5465774778050861, "error_rate_at_0_9": 0.6153846153846154, "confidence_bias": 0.1987375, "top_bins": { "0.9": { "n": 13, "errors": 8, "error_rate": 0.6153846153846154 }, "0.95": { "n": 4, "errors": 4, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.47058823529411764, "confidence_cutoff": 0.8924 }, "0.8": { "coverage": 0.84375, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.7038 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 40, "accuracy": 1.0, "full_probability_metrics": { "n": 40, "nll": 0.041954223438680764, "acc": 1.0, "ece": 0.03766000000000019, "brier": 0.012356652999999999, "mean_conf": 0.96234, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03766000000000003, "top_bins": { "0.9": { "n": 38, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 35, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9759 }, "0.8": { "coverage": 0.825, "accuracy": 1.0, "confidence_cutoff": 0.9579 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.041954223438680764, "acc": 1.0, "ece": 0.03766000000000019, "brier": 0.012356652999999999, "mean_conf": 0.96234, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03766000000000003, "top_bins": { "0.9": { "n": 38, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 35, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9759 }, "0.8": { "coverage": 0.825, "accuracy": 1.0, "confidence_cutoff": 0.9579 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 12, "accuracy": 0.3, "full_probability_metrics": { "n": 40, "nll": 1.3016893264396523, "acc": 0.3, "ece": 0.5102823286871234, "brier": 0.8693657229477225, "mean_conf": 0.7852273286871233, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.4876214030377747, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.4852273286871233, "top_bins": { "0.9": { "n": 14, "errors": 6, "error_rate": 0.42857142857142855 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8842 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5804580458045805 } }, "score_mae": 0.5791950837818259, "ranked_probability_score": 0.22157079717179223 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.3016893264396523, "acc": 0.3, "ece": 0.5102823286871234, "brier": 0.8693657229477225, "mean_conf": 0.7852273286871233, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.4876214030377747, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.4852273286871233, "top_bins": { "0.9": { "n": 14, "errors": 6, "error_rate": 0.42857142857142855 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8842 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5804580458045805 } }, "score_mae": 0.5791950837818259, "ranked_probability_score": 0.22157079717179223 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 69, "accuracy": 0.8625, "full_probability_metrics": { "n": 80, "nll": 0.3793053097373299, "acc": 0.8625, "ece": 0.07891555126283949, "brier": 0.20119036017095054, "mean_conf": 0.8422220740392549, "confident_error_rate": 0.0, "coverage_at_0_9": 0.525, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.75, "coverage_at_1pct_error": 0.575, "aurc": 0.030592982852822803, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02027792596074518, "top_bins": { "0.9": { "n": 42, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 33, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9213078692130787 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.6332366763323668 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.3793053097373299, "acc": 0.8625, "ece": 0.07891555126283949, "brier": 0.20119036017095054, "mean_conf": 0.8422220740392549, "confident_error_rate": 0.0, "coverage_at_0_9": 0.525, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.75, "coverage_at_1pct_error": 0.575, "aurc": 0.030592982852822803, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02027792596074518, "top_bins": { "0.9": { "n": 42, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 33, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9213078692130787 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.6332366763323668 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 50, "accuracy": 0.625, "full_probability_metrics": { "n": 80, "nll": 0.9304628014027901, "acc": 0.625, "ece": 0.07987469496168448, "brier": 0.5019225299010661, "mean_conf": 0.7000872909712854, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.95, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.1, "aurc": 0.19001933807526764, "error_rate_at_0_9": 0.05, "confidence_bias": 0.07508729097128541, "top_bins": { "0.9": { "n": 20, "errors": 1, "error_rate": 0.05 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7339 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.461053894610539 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.9304628014027901, "acc": 0.625, "ece": 0.07987469496168448, "brier": 0.5019225299010661, "mean_conf": 0.7000872909712854, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.95, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.1, "aurc": 0.19001933807526764, "error_rate_at_0_9": 0.05, "confidence_bias": 0.07508729097128541, "top_bins": { "0.9": { "n": 20, "errors": 1, "error_rate": 0.05 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7339 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.461053894610539 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 75, "accuracy": 0.375, "full_probability_metrics": { "n": 200, "nll": 1.884863725460874, "acc": 0.375, "ece": 0.0944813542659704, "brier": 0.7664539706775763, "mean_conf": 0.42504039277854366, "confident_error_rate": 0.02, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.035, "coverage_at_1pct_error": 0.035, "aurc": 0.42668454717684356, "error_rate_at_0_9": 0.2, "confidence_bias": 0.05004039277854366, "top_bins": { "0.9": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.95": { "n": 12, "errors": 3, "error_rate": 0.25 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.52, "confidence_cutoff": 0.32556744325567444 }, "0.8": { "coverage": 0.8, "accuracy": 0.40625, "confidence_cutoff": 0.16 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 1.884863725460874, "acc": 0.375, "ece": 0.0944813542659704, "brier": 0.7664539706775763, "mean_conf": 0.42504039277854366, "confident_error_rate": 0.02, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.035, "coverage_at_1pct_error": 0.035, "aurc": 0.42668454717684356, "error_rate_at_0_9": 0.2, "confidence_bias": 0.05004039277854366, "top_bins": { "0.9": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.95": { "n": 12, "errors": 3, "error_rate": 0.25 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.52, "confidence_cutoff": 0.32556744325567444 }, "0.8": { "coverage": 0.8, "accuracy": 0.40625, "confidence_cutoff": 0.16 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 62, "accuracy": 0.775, "full_probability_metrics": { "n": 80, "nll": 0.5415239740597191, "acc": 0.775, "ece": 0.13320499999999996, "brier": 0.34993291449999997, "mean_conf": 0.8896075, "confident_error_rate": 0.05, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.325, "aurc": 0.09217740259526386, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.11460749999999997, "top_bins": { "0.9": { "n": 48, "errors": 4, "error_rate": 0.08333333333333333 }, "0.95": { "n": 40, "errors": 3, "error_rate": 0.075 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9539 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.8013 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.5415239740597191, "acc": 0.775, "ece": 0.13320499999999996, "brier": 0.34993291449999997, "mean_conf": 0.8896075, "confident_error_rate": 0.05, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.325, "aurc": 0.09217740259526386, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.11460749999999997, "top_bins": { "0.9": { "n": 48, "errors": 4, "error_rate": 0.08333333333333333 }, "0.95": { "n": 40, "errors": 3, "error_rate": 0.075 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9539 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.8013 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 70, "accuracy": 0.875, "full_probability_metrics": { "n": 80, "nll": 0.31191325413859894, "acc": 0.875, "ece": 0.04622250000000007, "brier": 0.1877246535, "mean_conf": 0.8643225000000001, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9791666666666666, "coverage_at_5pct_error": 0.7625, "coverage_at_1pct_error": 0.1, "aurc": 0.05099437962226065, "error_rate_at_0_9": 0.020833333333333332, "confidence_bias": -0.010677499999999895, "top_bins": { "0.9": { "n": 48, "errors": 1, "error_rate": 0.020833333333333332 }, "0.95": { "n": 36, "errors": 1, "error_rate": 0.027777777777777776 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9366 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9230769230769231, "confidence_cutoff": 0.7602 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.31191325413859894, "acc": 0.875, "ece": 0.04622250000000007, "brier": 0.1877246535, "mean_conf": 0.8643225000000001, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9791666666666666, "coverage_at_5pct_error": 0.7625, "coverage_at_1pct_error": 0.1, "aurc": 0.05099437962226065, "error_rate_at_0_9": 0.020833333333333332, "confidence_bias": -0.010677499999999895, "top_bins": { "0.9": { "n": 48, "errors": 1, "error_rate": 0.020833333333333332 }, "0.95": { "n": 36, "errors": 1, "error_rate": 0.027777777777777776 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9366 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9230769230769231, "confidence_cutoff": 0.7602 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 79, "accuracy": 0.9875, "full_probability_metrics": { "n": 80, "nll": 0.056289563896296746, "acc": 0.9875, "ece": 0.033623917010264306, "brier": 0.02329240818112382, "mean_conf": 0.966728582989736, "confident_error_rate": 0.0, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02077141701026408, "top_bins": { "0.9": { "n": 74, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 57, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9940994099409942 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9784 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.056289563896296746, "acc": 0.9875, "ece": 0.033623917010264306, "brier": 0.02329240818112382, "mean_conf": 0.966728582989736, "confident_error_rate": 0.0, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02077141701026408, "top_bins": { "0.9": { "n": 74, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 57, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9940994099409942 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9784 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 71, "accuracy": 0.8875, "full_probability_metrics": { "n": 80, "nll": 0.2879933573013497, "acc": 0.8875, "ece": 0.057040000000000014, "brier": 0.1762249465, "mean_conf": 0.8614874999999997, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.975, "coverage_at_5pct_error": 0.825, "coverage_at_1pct_error": 0.4625, "aurc": 0.03168589057301142, "error_rate_at_0_9": 0.025, "confidence_bias": -0.026012500000000216, "top_bins": { "0.9": { "n": 40, "errors": 1, "error_rate": 0.025 }, "0.95": { "n": 30, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9095 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.7422 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.2879933573013497, "acc": 0.8875, "ece": 0.057040000000000014, "brier": 0.1762249465, "mean_conf": 0.8614874999999997, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.975, "coverage_at_5pct_error": 0.825, "coverage_at_1pct_error": 0.4625, "aurc": 0.03168589057301142, "error_rate_at_0_9": 0.025, "confidence_bias": -0.026012500000000216, "top_bins": { "0.9": { "n": 40, "errors": 1, "error_rate": 0.025 }, "0.95": { "n": 30, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9095 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.7422 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.025411758317540733, "acc": 1.0, "ece": 0.02450000000000009, "brier": 0.0033954560000000003, "mean_conf": 0.9755, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024499999999999966, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9923 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.977 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.025411758317540733, "acc": 1.0, "ece": 0.02450000000000009, "brier": 0.0033954560000000003, "mean_conf": 0.9755, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024499999999999966, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9923 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.977 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.3953369788503307, "acc": 0.8, "ece": 0.1387900000000002, "brier": 0.266232006, "mean_conf": 0.9387900000000002, "confident_error_rate": 0.1, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.043611111111111114, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.1387900000000002, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9759 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9241 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.3953369788503307, "acc": 0.8, "ece": 0.1387900000000002, "brier": 0.266232006, "mean_conf": 0.9387900000000002, "confident_error_rate": 0.1, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.043611111111111114, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.1387900000000002, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9759 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9241 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.215897679804811, "acc": 0.3, "ece": 0.5537200636134008, "brier": 0.8924684054266614, "mean_conf": 0.8240800636134006, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.37130952380952376, "error_rate_at_0_9": 0.0, "confidence_bias": 0.5240800636134006, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8679 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7543 } }, "score_mae": 0.5824802536800024, "ranked_probability_score": 0.22724486673020516 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.215897679804811, "acc": 0.3, "ece": 0.5537200636134008, "brier": 0.8924684054266614, "mean_conf": 0.8240800636134006, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.37130952380952376, "error_rate_at_0_9": 0.0, "confidence_bias": 0.5240800636134006, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8679 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7543 } }, "score_mae": 0.5824802536800024, "ranked_probability_score": 0.22724486673020516 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.892783368571971, "acc": 0.7, "ece": 0.2995399999999999, "brier": 0.44739400399999996, "mean_conf": 0.87518, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.2041269841269841, "error_rate_at_0_9": 0.4, "confidence_bias": 0.17518, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9354 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8621 } }, "score_mae": 0.30667999999999995, "ranked_probability_score": 0.110259646 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.892783368571971, "acc": 0.7, "ece": 0.2995399999999999, "brier": 0.44739400399999996, "mean_conf": 0.87518, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.2041269841269841, "error_rate_at_0_9": 0.4, "confidence_bias": 0.17518, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9354 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8621 } }, "score_mae": 0.30667999999999995, "ranked_probability_score": 0.110259646 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.65295247489328, "acc": 0.8, "ece": 0.16346000000000005, "brier": 0.36393330200000007, "mean_conf": 0.9634600000000001, "confident_error_rate": 0.2, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.08567460317460318, "error_rate_at_0_9": 0.2, "confidence_bias": 0.16346000000000005, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9788 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9465 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.65295247489328, "acc": 0.8, "ece": 0.16346000000000005, "brier": 0.36393330200000007, "mean_conf": 0.9634600000000001, "confident_error_rate": 0.2, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.08567460317460318, "error_rate_at_0_9": 0.2, "confidence_bias": 0.16346000000000005, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9788 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9465 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 0.7212710272228258, "acc": 0.6, "ece": 0.3413500000000001, "brier": 0.5223997619999999, "mean_conf": 0.7297100000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3576190476190476, "error_rate_at_0_9": null, "confidence_bias": 0.1297100000000001, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7234 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7038 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7212710272228258, "acc": 0.6, "ece": 0.3413500000000001, "brier": 0.5223997619999999, "mean_conf": 0.7297100000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3576190476190476, "error_rate_at_0_9": null, "confidence_bias": 0.1297100000000001, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7234 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7038 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.7962381952076705, "acc": 0.8, "ece": 0.29414298429842983, "brier": 0.4306105054901508, "mean_conf": 0.5834847784778477, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09456349206349206, "error_rate_at_0_9": null, "confidence_bias": -0.2165152215221523, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.5682 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5187518751875188 } }, "score_mae": 0.5197684778477848, "ranked_probability_score": 0.1442794479406923 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7962381952076705, "acc": 0.8, "ece": 0.29414298429842983, "brier": 0.4306105054901508, "mean_conf": 0.5834847784778477, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09456349206349206, "error_rate_at_0_9": null, "confidence_bias": -0.2165152215221523, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.5682 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5187518751875188 } }, "score_mae": 0.5197684778477848, "ranked_probability_score": 0.1442794479406923 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.1756865764448203, "acc": 0.3, "ece": 0.48920753014947527, "brier": 0.8012694828555553, "mean_conf": 0.7610475301494752, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4046428571428571, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.46104753014947525, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7803 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6982301769823018 } }, "score_mae": 0.6108795862836959, "ranked_probability_score": 0.22142678360251872 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.1756865764448203, "acc": 0.3, "ece": 0.48920753014947527, "brier": 0.8012694828555553, "mean_conf": 0.7610475301494752, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4046428571428571, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.46104753014947525, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7803 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6982301769823018 } }, "score_mae": 0.6108795862836959, "ranked_probability_score": 0.22142678360251872 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.005551851143716301, "acc": 1.0, "ece": 0.005530165037101353, "brier": 6.653382630215632e-05, "mean_conf": 0.9944698349628986, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.005530165037101353, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9970997099709971 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9925007499250076 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.005551851143716301, "acc": 1.0, "ece": 0.005530165037101353, "brier": 6.653382630215632e-05, "mean_conf": 0.9944698349628986, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.005530165037101353, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9970997099709971 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9925007499250076 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.2628593748977697, "acc": 0.9, "ece": 0.10521167486251676, "brier": 0.1554169585848721, "mean_conf": 0.7947883251374833, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.1052116748625167, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9236 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5880411958804119 } }, "score_mae": 0.20696154189041538, "ranked_probability_score": 0.03927155506980815 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.2628593748977697, "acc": 0.9, "ece": 0.10521167486251676, "brier": 0.1554169585848721, "mean_conf": 0.7947883251374833, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.1052116748625167, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9236 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5880411958804119 } }, "score_mae": 0.20696154189041538, "ranked_probability_score": 0.03927155506980815 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.3045097156493346, "acc": 0.4, "ece": 0.23849407899224073, "brier": 0.7370612903396646, "mean_conf": 0.6248140789922408, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6368650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.22481407899224082, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6362 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5422457754224578 } }, "score_mae": 0.6521860787815608, "ranked_probability_score": 0.21836761179284664 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3045097156493346, "acc": 0.4, "ece": 0.23849407899224073, "brier": 0.7370612903396646, "mean_conf": 0.6248140789922408, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6368650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.22481407899224082, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6362 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5422457754224578 } }, "score_mae": 0.6521860787815608, "ranked_probability_score": 0.21836761179284664 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.7150462962962962, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.8333333333333334, "flip_rate": 0.1111111111111111, "covered_only_flip_rate": 0.1111111111111111, "half_l1": 0.07013769457710495 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.6881870324763705, "share_at_0_9": 0.16363636363636364, "normalized_entropy": 0.6772185760728027 }, "covered_only_confidence_metrics": { "mean_max_p": 0.6881870324763705, "share_at_0_9": 0.16363636363636364, "normalized_entropy": 0.6772185760728027 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 76, "accuracy": 0.6909090909090909, "full_probability_metrics": { "n": 110, "nll": 0.6771362728185518, "acc": 0.6909090909090909, "ece": 0.1425895101212133, "brier": 0.4200225187748368, "mean_conf": 0.8241204192121223, "confident_error_rate": 0.05454545454545454, "coverage_at_0_9": 0.4818181818181818, "accuracy_at_0_9": 0.8867924528301887, "coverage_at_5pct_error": 0.4090909090909091, "coverage_at_1pct_error": 0.3090909090909091, "aurc": 0.12834585107983904, "error_rate_at_0_9": 0.11320754716981132, "confidence_bias": 0.1332113283030314, "top_bins": { "0.9": { "n": 53, "errors": 6, "error_rate": 0.11320754716981132 }, "0.95": { "n": 42, "errors": 2, "error_rate": 0.047619047619047616 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8727272727272727, "confidence_cutoff": 0.8905000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.7272727272727273, "confidence_cutoff": 0.6622 } }, "score_mae": 0.47982598974724316, "ranked_probability_score": 0.16014165185601179 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.6771362728185518, "acc": 0.6909090909090909, "ece": 0.1425895101212133, "brier": 0.4200225187748368, "mean_conf": 0.8241204192121223, "confident_error_rate": 0.05454545454545454, "coverage_at_0_9": 0.4818181818181818, "accuracy_at_0_9": 0.8867924528301887, "coverage_at_5pct_error": 0.4090909090909091, "coverage_at_1pct_error": 0.3090909090909091, "aurc": 0.12834585107983904, "error_rate_at_0_9": 0.11320754716981132, "confidence_bias": 0.1332113283030314, "top_bins": { "0.9": { "n": 53, "errors": 6, "error_rate": 0.11320754716981132 }, "0.95": { "n": 42, "errors": 2, "error_rate": 0.047619047619047616 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8727272727272727, "confidence_cutoff": 0.8905000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.7272727272727273, "confidence_cutoff": 0.6622 } }, "score_mae": 0.47982598974724316, "ranked_probability_score": 0.16014165185601179 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.13593338673575203 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.6619585773882137, "paired_flip": { "pairs": 64, "flip_rate": 0.453125, "both_correct_rate": 0.40625, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.5833333333333334 }, "unknowable": { "n": 110, "mean_max_p": 0.6881870324763705, "share_at_0_9": 0.16363636363636364, "control_mean_max_p": 0.8241204192121223, "control_share_at_0_9": 0.4818181818181818, "control_acc": 0.6909090909090909, "paired_confidence_drop": 0.13593338673575203, "share_less_confident_than_control": 0.7545454545454545 }, "clean": { "n": 1046, "nll": 0.7969332191608542, "acc": 0.6931166347992351, "ece": 0.08090597402201222, "brier": 0.41215054236049414, "mean_conf": 0.7666068088925038, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.4483747609942639, "accuracy_at_0_9": 0.908315565031983, "coverage_at_5pct_error": 0.35181644359464626, "coverage_at_1pct_error": 0.19120458891013384, "aurc": 0.11720241935804643, "error_rate_at_0_9": 0.09168443496801706, "confidence_bias": 0.07349017409326863, "top_bins": { "0.9": { "n": 469, "errors": 43, "error_rate": 0.09168443496801706 }, "0.95": { "n": 344, "errors": 16, "error_rate": 0.046511627906976744 }, "0.99": { "n": 113, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5009560229445507, "accuracy": 0.8874045801526718, "confidence_cutoff": 0.8828 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7945041816009558, "confidence_cutoff": 0.5623999999999999 } }, "score_mae": 0.5195736273610764, "ranked_probability_score": 0.184713309982324 }, "tasks": { "buried_emotion": { "n": 20, "nll": 0.4639612304098854, "acc": 0.85, "ece": 0.14167657774951575, "brier": 0.22331793650847115, "mean_conf": 0.7444834222504841, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.027718223254213963, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10551657774951584, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7637999999999998 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.6017 } } }, "buried_paws": { "n": 20, "nll": 0.6012843951869397, "acc": 0.75, "ece": 0.12283500000000008, "brier": 0.38466947700000004, "mean_conf": 0.8728349999999999, "confident_error_rate": 0.1, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.1365742384550434, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.12283499999999992, "top_bins": { "0.9": { "n": 12, "errors": 2, "error_rate": 0.16666666666666666 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9241 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.7773 } } }, "buried_qnli": { "n": 20, "nll": 0.34719070735248697, "acc": 0.9, "ece": 0.11635000000000006, "brier": 0.155148236, "mean_conf": 0.9124700000000001, "confident_error_rate": 0.05, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.127052733772935, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.012470000000000092, "top_bins": { "0.9": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.95": { "n": 11, "errors": 1, "error_rate": 0.09090909090909091 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9616 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.8495 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.4407950321556191, "acc": 0.8, "ece": 0.07471999999999997, "brier": 0.28890536000000006, "mean_conf": 0.7631, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09617093261056109, "error_rate_at_0_9": 0.0, "confidence_bias": -0.036900000000000044, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.7422 }, "0.8": { "coverage": 0.85, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6622 } } }, "composition_final_combination": { "n": 32, "nll": 0.7839904683434762, "acc": 0.53125, "ece": 0.28895, "brier": 0.5575436062499999, "mean_conf": 0.8202, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.28125, "coverage_at_1pct_error": 0.28125, "aurc": 0.22343766144399238, "error_rate_at_0_9": 0.25, "confidence_bias": 0.28895000000000004, "top_bins": { "0.9": { "n": 12, "errors": 3, "error_rate": 0.25 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.8556 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6153846153846154, "confidence_cutoff": 0.7422 } } }, "composition_final_exception": { "n": 32, "nll": 1.0710423786305578, "acc": 0.5625, "ece": 0.374315625, "brier": 0.713839373125, "mean_conf": 0.8714531249999999, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.40625, "accuracy_at_0_9": 0.5384615384615384, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.42621328369957967, "error_rate_at_0_9": 0.46153846153846156, "confidence_bias": 0.30895312499999994, "top_bins": { "0.9": { "n": 13, "errors": 6, "error_rate": 0.46153846153846156 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8969 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.8368 } } }, "composition_final_negation": { "n": 32, "nll": 0.9801228405234432, "acc": 0.625, "ece": 0.3267812499999999, "brier": 0.60622501625, "mean_conf": 0.8237375, "confident_error_rate": 0.25, "coverage_at_0_9": 0.40625, "accuracy_at_0_9": 0.38461538461538464, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5465774778050861, "error_rate_at_0_9": 0.6153846153846154, "confidence_bias": 0.1987375, "top_bins": { "0.9": { "n": 13, "errors": 8, "error_rate": 0.6153846153846154 }, "0.95": { "n": 4, "errors": 4, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.47058823529411764, "confidence_cutoff": 0.8924 }, "0.8": { "coverage": 0.84375, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.7038 } } }, "contrastive_authorization": { "n": 40, "nll": 0.041954223438680764, "acc": 1.0, "ece": 0.03766000000000019, "brier": 0.012356652999999999, "mean_conf": 0.96234, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03766000000000003, "top_bins": { "0.9": { "n": 38, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 35, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9759 }, "0.8": { "coverage": 0.825, "accuracy": 1.0, "confidence_cutoff": 0.9579 } } }, "contrastive_deadline": { "n": 40, "nll": 1.3016893264396523, "acc": 0.3, "ece": 0.5102823286871234, "brier": 0.8693657229477225, "mean_conf": 0.7852273286871233, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.4876214030377747, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.4852273286871233, "top_bins": { "0.9": { "n": 14, "errors": 6, "error_rate": 0.42857142857142855 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8842 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5804580458045805 } }, "score_mae": 0.5791950837818259, "ranked_probability_score": 0.22157079717179223 }, "emotion": { "n": 80, "nll": 0.3793053097373299, "acc": 0.8625, "ece": 0.07891555126283949, "brier": 0.20119036017095054, "mean_conf": 0.8422220740392549, "confident_error_rate": 0.0, "coverage_at_0_9": 0.525, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.75, "coverage_at_1pct_error": 0.575, "aurc": 0.030592982852822803, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02027792596074518, "top_bins": { "0.9": { "n": 42, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 33, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9213078692130787 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.6332366763323668 } } }, "mmlu": { "n": 80, "nll": 0.9304628014027901, "acc": 0.625, "ece": 0.07987469496168448, "brier": 0.5019225299010661, "mean_conf": 0.7000872909712854, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.95, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.1, "aurc": 0.19001933807526764, "error_rate_at_0_9": 0.05, "confidence_bias": 0.07508729097128541, "top_bins": { "0.9": { "n": 20, "errors": 1, "error_rate": 0.05 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7339 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.461053894610539 } } }, "mmlu_pro": { "n": 200, "nll": 1.884863725460874, "acc": 0.375, "ece": 0.0944813542659704, "brier": 0.7664539706775763, "mean_conf": 0.42504039277854366, "confident_error_rate": 0.02, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.035, "coverage_at_1pct_error": 0.035, "aurc": 0.42668454717684356, "error_rate_at_0_9": 0.2, "confidence_bias": 0.05004039277854366, "top_bins": { "0.9": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.95": { "n": 12, "errors": 3, "error_rate": 0.25 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.52, "confidence_cutoff": 0.32556744325567444 }, "0.8": { "coverage": 0.8, "accuracy": 0.40625, "confidence_cutoff": 0.16 } } }, "paws": { "n": 80, "nll": 0.5415239740597191, "acc": 0.775, "ece": 0.13320499999999996, "brier": 0.34993291449999997, "mean_conf": 0.8896075, "confident_error_rate": 0.05, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.325, "aurc": 0.09217740259526386, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.11460749999999997, "top_bins": { "0.9": { "n": 48, "errors": 4, "error_rate": 0.08333333333333333 }, "0.95": { "n": 40, "errors": 3, "error_rate": 0.075 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9539 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.8013 } } }, "qnli": { "n": 80, "nll": 0.31191325413859894, "acc": 0.875, "ece": 0.04622250000000007, "brier": 0.1877246535, "mean_conf": 0.8643225000000001, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9791666666666666, "coverage_at_5pct_error": 0.7625, "coverage_at_1pct_error": 0.1, "aurc": 0.05099437962226065, "error_rate_at_0_9": 0.020833333333333332, "confidence_bias": -0.010677499999999895, "top_bins": { "0.9": { "n": 48, "errors": 1, "error_rate": 0.020833333333333332 }, "0.95": { "n": 36, "errors": 1, "error_rate": 0.027777777777777776 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9366 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9230769230769231, "confidence_cutoff": 0.7602 } } }, "sciq": { "n": 80, "nll": 0.056289563896296746, "acc": 0.9875, "ece": 0.033623917010264306, "brier": 0.02329240818112382, "mean_conf": 0.966728582989736, "confident_error_rate": 0.0, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02077141701026408, "top_bins": { "0.9": { "n": 74, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 57, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9940994099409942 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9784 } } }, "tweet_offensive": { "n": 80, "nll": 0.2879933573013497, "acc": 0.8875, "ece": 0.057040000000000014, "brier": 0.1762249465, "mean_conf": 0.8614874999999997, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.975, "coverage_at_5pct_error": 0.825, "coverage_at_1pct_error": 0.4625, "aurc": 0.03168589057301142, "error_rate_at_0_9": 0.025, "confidence_bias": -0.026012500000000216, "top_bins": { "0.9": { "n": 40, "errors": 1, "error_rate": 0.025 }, "0.95": { "n": 30, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9095 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.7422 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.9523653544887349, "acc": 0.5, "ece": 0.26436, "brier": 0.6804739919999998, "mean_conf": 0.7643599999999999, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.26435999999999993, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8432999999999999 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7038 } } }, "unknowable_authorization": { "n": 10, "nll": 1.8487554347422879, "acc": 0.5, "ece": 0.47390999999999983, "brier": 0.9483254780000001, "mean_conf": 0.9739099999999998, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4261904761904762, "error_rate_at_0_9": 0.5, "confidence_bias": 0.47390999999999983, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.9736 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9666 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.025411758317540733, "acc": 1.0, "ece": 0.02450000000000009, "brier": 0.0033954560000000003, "mean_conf": 0.9755, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024499999999999966, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9923 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.977 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.3953369788503307, "acc": 0.8, "ece": 0.1387900000000002, "brier": 0.266232006, "mean_conf": 0.9387900000000002, "confident_error_rate": 0.1, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.8888888888888888, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.043611111111111114, "error_rate_at_0_9": 0.1111111111111111, "confidence_bias": 0.1387900000000002, "top_bins": { "0.9": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9759 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9241 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.215897679804811, "acc": 0.3, "ece": 0.5537200636134008, "brier": 0.8924684054266614, "mean_conf": 0.8240800636134006, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.37130952380952376, "error_rate_at_0_9": 0.0, "confidence_bias": 0.5240800636134006, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8679 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7543 } }, "score_mae": 0.5824802536800024, "ranked_probability_score": 0.22724486673020516 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.892783368571971, "acc": 0.7, "ece": 0.2995399999999999, "brier": 0.44739400399999996, "mean_conf": 0.87518, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.2041269841269841, "error_rate_at_0_9": 0.4, "confidence_bias": 0.17518, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9354 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8621 } }, "score_mae": 0.30667999999999995, "ranked_probability_score": 0.110259646 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.65295247489328, "acc": 0.8, "ece": 0.16346000000000005, "brier": 0.36393330200000007, "mean_conf": 0.9634600000000001, "confident_error_rate": 0.2, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.08567460317460318, "error_rate_at_0_9": 0.2, "confidence_bias": 0.16346000000000005, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9788 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.9465 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.7212710272228258, "acc": 0.6, "ece": 0.3413500000000001, "brier": 0.5223997619999999, "mean_conf": 0.7297100000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3576190476190476, "error_rate_at_0_9": null, "confidence_bias": 0.1297100000000001, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7234 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7038 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.7962381952076705, "acc": 0.8, "ece": 0.29414298429842983, "brier": 0.4306105054901508, "mean_conf": 0.5834847784778477, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.09456349206349206, "error_rate_at_0_9": null, "confidence_bias": -0.2165152215221523, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.5682 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5187518751875188 } }, "score_mae": 0.5197684778477848, "ranked_probability_score": 0.1442794479406923 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.1756865764448203, "acc": 0.3, "ece": 0.48920753014947527, "brier": 0.8012694828555553, "mean_conf": 0.7610475301494752, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4046428571428571, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.46104753014947525, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7803 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6982301769823018 } }, "score_mae": 0.6108795862836959, "ranked_probability_score": 0.22142678360251872 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.005551851143716301, "acc": 1.0, "ece": 0.005530165037101353, "brier": 6.653382630215632e-05, "mean_conf": 0.9944698349628986, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.005530165037101353, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9970997099709971 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9925007499250076 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.2628593748977697, "acc": 0.9, "ece": 0.10521167486251676, "brier": 0.1554169585848721, "mean_conf": 0.7947883251374833, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.1052116748625167, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9236 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5880411958804119 } }, "score_mae": 0.20696154189041538, "ranked_probability_score": 0.03927155506980815 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 1.3045097156493346, "acc": 0.4, "ece": 0.23849407899224073, "brier": 0.7370612903396646, "mean_conf": 0.6248140789922408, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6368650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.22481407899224082, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6362 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5422457754224578 } }, "score_mae": 0.6521860787815608, "ranked_probability_score": 0.21836761179284664 }, "unknowable_deadline": { "n": 10, "nll": 1.5094586628679576, "acc": 0.3, "ece": 0.42434893419808933, "brier": 0.8687562014927769, "mean_conf": 0.7243489341980893, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6629761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.42434893419808933, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7725772577257726 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6523347665233477 } }, "score_mae": 0.6869158809971587, "ranked_probability_score": 0.2518342737208908 }, "unknowable_late_fee": { "n": 10, "nll": 1.1205794166173113, "acc": 0.4, "ece": 0.0975, "brier": 0.6865798999999999, "mean_conf": 0.49749999999999994, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6283333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.09749999999999992, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.4622 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.458 } }, "score_mae": 0.6590199999999999, "ranked_probability_score": 0.21491238299999998 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.3615788769844355, "acc": 0.5, "ece": 0.2865226617338267, "brier": 0.7674760700241765, "mean_conf": 0.7865226617338266, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.28652266173382657, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.7989 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6956 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7665786600235723, "acc": 0.5, "ece": 0.16543000000000002, "brier": 0.5658363660000001, "mean_conf": 0.66543, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5843650793650793, "error_rate_at_0_9": null, "confidence_bias": 0.16542999999999997, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6729 }, "0.8": { "coverage": 0.9, "accuracy": 0.5555555555555556, "confidence_cutoff": 0.6292 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.6919938682466733, "acc": 0.2, "ece": 0.4282194186705941, "brier": 0.9876840988189617, "mean_conf": 0.628219418670594, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.925436507936508, "error_rate_at_0_9": null, "confidence_bias": 0.42821941867059404, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.6302 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.5811 } }, "score_mae": 1.0403694523159945, "ranked_probability_score": 0.3899505087748525 }, "unknowable_sla_response": { "n": 10, "nll": 1.1987134714126293, "acc": 0.3, "ece": 0.1892100009999, "brier": 0.7225741696529429, "mean_conf": 0.44813000099989997, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8566666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.14813000099989998, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.4329 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.4306 } }, "score_mae": 0.6760324387561244, "ranked_probability_score": 0.23029776448038505 }, "unknowable_spend_threshold": { "n": 10, "nll": 1.281916911815412, "acc": 0.5, "ece": 0.3973441395860414, "brier": 0.8283090247043594, "mean_conf": 0.8973441395860412, "confident_error_rate": 0.3, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3973441395860412, "top_bins": { "0.9": { "n": 6, "errors": 3, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.91010898910109 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8829117088291171 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.202760544319816, "acc": 0.4, "ece": 0.13457084291570842, "brier": 0.7221005646009923, "mean_conf": 0.5345708429157083, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5783333333333334, "error_rate_at_0_9": null, "confidence_bias": 0.1345708429157083, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.45785421457854214 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4479 } }, "score_mae": 0.8513548645135487, "ranked_probability_score": 0.27382282027676097 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.5645982107080147, "acc": 0.3, "ece": 0.3497213591359137, "brier": 0.8429192851173013, "mean_conf": 0.6497213591359136, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7625396825396825, "error_rate_at_0_9": null, "confidence_bias": 0.34972135913591357, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6604 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5875587558755876 } }, "score_mae": 0.7694714181418142, "ranked_probability_score": 0.2752835442970781 } }, "variants": { "clean": { "n": 1156, "nll": 0.8465269388966452, "acc": 0.6652249134948097, "ece": 0.10062995017251339, "brier": 0.4475084937830381, "mean_conf": 0.7591447194411416, "confident_error_rate": 0.04498269896193772, "coverage_at_0_9": 0.421280276816609, "accuracy_at_0_9": 0.893223819301848, "coverage_at_5pct_error": 0.27162629757785467, "coverage_at_1pct_error": 0.10899653979238755, "aurc": 0.13975024873720063, "error_rate_at_0_9": 0.10677618069815195, "confidence_bias": 0.09391980594633187, "top_bins": { "0.9": { "n": 487, "errors": 52, "error_rate": 0.10677618069815195 }, "0.95": { "n": 354, "errors": 21, "error_rate": 0.059322033898305086 }, "0.99": { "n": 113, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.856401384083045, "confidence_cutoff": 0.8556 }, "0.8": { "coverage": 0.801038062283737, "accuracy": 0.7570194384449244, "confidence_cutoff": 0.5598 } }, "score_mae": 0.6174312705209626, "ranked_probability_score": 0.21770214964832543 }, "none_absent": { "n": 36, "nll": 2.1316095293241157, "acc": 0.3333333333333333, "ece": 0.38235156121657127, "brier": 0.98387788422635, "mean_conf": 0.7156848945499046, "confident_error_rate": 0.1388888888888889, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.4444444444444444, "coverage_at_5pct_error": 0.027777777777777776, "coverage_at_1pct_error": 0.027777777777777776, "aurc": 0.5848008268789461, "error_rate_at_0_9": 0.5555555555555556, "confidence_bias": 0.38235156121657127, "top_bins": { "0.9": { "n": 9, "errors": 5, "error_rate": 0.5555555555555556 }, "0.95": { "n": 6, "errors": 4, "error_rate": 0.6666666666666666 }, "0.99": { "n": 2, "errors": 1, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.7476252374762524 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.3793103448275862, "confidence_cutoff": 0.5341 } } }, "none_present": { "n": 36, "nll": 0.5430543920256307, "acc": 0.7777777777777778, "ece": 0.12144972380752493, "brier": 0.27957117405775156, "mean_conf": 0.75953021255633, "confident_error_rate": 0.027777777777777776, "coverage_at_0_9": 0.4166666666666667, "accuracy_at_0_9": 0.9333333333333333, "coverage_at_5pct_error": 0.6944444444444444, "coverage_at_1pct_error": 0.3333333333333333, "aurc": 0.06410958682886508, "error_rate_at_0_9": 0.06666666666666667, "confidence_bias": -0.01824756522144777, "top_bins": { "0.9": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9444444444444444, "confidence_cutoff": 0.8112 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8620689655172413, "confidence_cutoff": 0.5272 } } }, "permuted": { "n": 36, "nll": 0.4614275542394204, "acc": 0.8611111111111112, "ece": 0.11076690754142444, "brier": 0.23082780120241075, "mean_conf": 0.7785117317399927, "confident_error_rate": 0.027777777777777776, "coverage_at_0_9": 0.4444444444444444, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 0.6944444444444444, "coverage_at_1pct_error": 0.3333333333333333, "aurc": 0.04704579410073126, "error_rate_at_0_9": 0.0625, "confidence_bias": -0.0825993793711185, "top_bins": { "0.9": { "n": 16, "errors": 1, "error_rate": 0.0625 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9444444444444444, "confidence_cutoff": 0.8249175082491751 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.9310344827586207, "confidence_cutoff": 0.5627562756275627 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.06915215646569589, "flip_rate": 0.1111111111111111 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.7969332191608542, "acc": 0.6931166347992351, "ece": 0.08090597402201222, "brier": 0.41215054236049414, "mean_conf": 0.7666068088925038, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.4483747609942639, "accuracy_at_0_9": 0.908315565031983, "coverage_at_5pct_error": 0.35181644359464626, "coverage_at_1pct_error": 0.19120458891013384, "aurc": 0.11720241935804643, "error_rate_at_0_9": 0.09168443496801706, "confidence_bias": 0.07349017409326863, "top_bins": { "0.9": { "n": 469, "errors": 43, "error_rate": 0.09168443496801706 }, "0.95": { "n": 344, "errors": 16, "error_rate": 0.046511627906976744 }, "0.99": { "n": 113, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5009560229445507, "accuracy": 0.8874045801526718, "confidence_cutoff": 0.8828 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7945041816009558, "confidence_cutoff": 0.5623999999999999 } }, "score_mae": 0.5195736273610764, "ranked_probability_score": 0.184713309982324 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Qwen3.5-4B": { "panels": { "old_core": { "point": 0.6988839285714286, "exact_fraction": "3131/4480", "ci95": [ 0.6632803199404762, 0.7337797619047619 ] }, "v3_core": { "point": 0.43333333333333335, "exact_fraction": "13/30", "ci95": [ 0.4179166666666667, 0.4479166666666667 ] }, "v4": { "point": 0.8796875, "exact_fraction": "563/640", "ci95": [ 0.8461463032298809, 0.9125000000000001 ] }, "v5": { "point": 0.7979166666666667, "exact_fraction": "383/480", "ci95": [ 0.7641944237372369, 0.830579286387442 ] }, "transfer_v9_test": { "point": 0.6883365200764818, "exact_fraction": "360/523", "ci95": [ 0.6578699340245052, 0.719399070599905 ] }, "original_four_panel_mean": { "point": 0.7024553571428571, "exact_fraction": "3147/4480", "ci95": [ 0.6873480253417569, 0.7176086887612844 ] }, "weighted_mean": { "point": 0.6728896149162342, "exact_fraction": "94596437/140582400", "ci95": [ 0.6588660818454937, 0.6869238202835648 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 108, "requested": 128 }, "boolean_constraints": { "correct": 40, "requested": 64 }, "dbpedia_14": { "correct": 109, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 70, "requested": 96 }, "ordinal_rubric": { "correct": 58, "requested": 64 }, "relational_composition": { "correct": 49, "requested": 96 }, "scoped_evidence": { "correct": 49, "requested": 96 }, "state_tracking": { "correct": 27, "requested": 96 }, "unknown_rejection": { "correct": 39, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 40, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 19, "requested": 80 }, "massive_en": { "correct": 110, "requested": 120 }, "massive_zh": { "correct": 104, "requested": 120 }, "multiset_reconciliation": { "correct": 22, "requested": 80 }, "ordinal_service_loss": { "correct": 17, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 20, "requested": 80 }, "temporal_exclusion": { "correct": 25, "requested": 80 }, "transaction_recovery": { "correct": 21, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 134, "requested": 160 }, "belebele_en": { "correct": 149, "requested": 160 }, "belebele_zh": { "correct": 146, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 73, "requested": 120 }, "squad2_answerability": { "correct": 98, "requested": 120 }, "snli": { "correct": 96, "requested": 120 }, "qasc": { "correct": 116, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 720, "accuracy": 0.6883365200764818, "full_probability_metrics": { "n": 1046, "nll": 0.8185509865100163, "acc": 0.6883365200764818, "ece": 0.08978015832969408, "brier": 0.4198595508629067, "mean_conf": 0.77574690988783, "confident_error_rate": 0.032504780114722756, "coverage_at_0_9": 0.42638623326959846, "accuracy_at_0_9": 0.9237668161434978, "coverage_at_5pct_error": 0.34703632887189295, "coverage_at_1pct_error": 0.1959847036328872, "aurc": 0.12334711495626294, "error_rate_at_0_9": 0.07623318385650224, "confidence_bias": 0.08741038981134819, "top_bins": { "0.9": { "n": 446, "errors": 34, "error_rate": 0.07623318385650224 }, "0.95": { "n": 331, "errors": 12, "error_rate": 0.03625377643504532 }, "0.99": { "n": 172, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5038240917782026, "accuracy": 0.8956356736242884, "confidence_cutoff": 0.8354835361242294 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.7589073634204275, "confidence_cutoff": 0.5621765088386383 } }, "score_mae": 0.40818287864267966, "ranked_probability_score": 0.15157939130185247 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.8185509865100163, "acc": 0.6883365200764818, "ece": 0.08978015832969408, "brier": 0.4198595508629067, "mean_conf": 0.77574690988783, "confident_error_rate": 0.032504780114722756, "coverage_at_0_9": 0.42638623326959846, "accuracy_at_0_9": 0.9237668161434978, "coverage_at_5pct_error": 0.34703632887189295, "coverage_at_1pct_error": 0.1959847036328872, "aurc": 0.12334711495626294, "error_rate_at_0_9": 0.07623318385650224, "confidence_bias": 0.08741038981134819, "top_bins": { "0.9": { "n": 446, "errors": 34, "error_rate": 0.07623318385650224 }, "0.95": { "n": 331, "errors": 12, "error_rate": 0.03625377643504532 }, "0.99": { "n": 172, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5038240917782026, "accuracy": 0.8956356736242884, "confidence_cutoff": 0.8354835361242294 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.7589073634204275, "confidence_cutoff": 0.5621765088386383 } }, "score_mae": 0.40818287864267966, "ranked_probability_score": 0.15157939130185247 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 9, "accuracy": 0.45, "full_probability_metrics": { "n": 20, "nll": 1.7616024963066572, "acc": 0.45, "ece": 0.23555467490844156, "brier": 0.8076512996162115, "mean_conf": 0.6326583796971109, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5139849014075021, "error_rate_at_0_9": 0.5, "confidence_bias": 0.1826583796971109, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.620902601373769 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.4701217430462008 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.7616024963066572, "acc": 0.45, "ece": 0.23555467490844156, "brier": 0.8076512996162115, "mean_conf": 0.6326583796971109, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5139849014075021, "error_rate_at_0_9": 0.5, "confidence_bias": 0.1826583796971109, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.620902601373769 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.4701217430462008 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 15, "accuracy": 0.75, "full_probability_metrics": { "n": 20, "nll": 0.5347892841599211, "acc": 0.75, "ece": 0.12973636761307714, "brier": 0.361960551470936, "mean_conf": 0.7220932058990002, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.17242260061919504, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027906794100999832, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7549149841070175 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6224593222141266 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5347892841599211, "acc": 0.75, "ece": 0.12973636761307714, "brier": 0.361960551470936, "mean_conf": 0.7220932058990002, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.17242260061919504, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027906794100999832, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7549149841070175 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6224593222141266 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 15, "accuracy": 0.75, "full_probability_metrics": { "n": 20, "nll": 0.4139708021155313, "acc": 0.75, "ece": 0.22482597026973963, "brier": 0.2769511861963836, "mean_conf": 0.7978020852431655, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.07702878390958887, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04780208524316554, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8519528061151505 }, "0.8": { "coverage": 0.85, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.6513548493385315 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.4139708021155313, "acc": 0.75, "ece": 0.22482597026973963, "brier": 0.2769511861963836, "mean_conf": 0.7978020852431655, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.07702878390958887, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04780208524316554, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8519528061151505 }, "0.8": { "coverage": 0.85, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.6513548493385315 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 8, "accuracy": 0.4, "full_probability_metrics": { "n": 20, "nll": 1.108168990139449, "acc": 0.4, "ece": 0.3968666262924671, "brier": 0.8098649357203225, "mean_conf": 0.7310038901865482, "confident_error_rate": 0.1, "coverage_at_0_9": 0.15, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.6227259363443574, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.3310038901865482, "top_bins": { "0.9": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.7549149990081787 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.562176525592804 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.108168990139449, "acc": 0.4, "ece": 0.3968666262924671, "brier": 0.8098649357203225, "mean_conf": 0.7310038901865482, "confident_error_rate": 0.1, "coverage_at_0_9": 0.15, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.6227259363443574, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.3310038901865482, "top_bins": { "0.9": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.7549149990081787 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.562176525592804 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 19, "accuracy": 0.59375, "full_probability_metrics": { "n": 32, "nll": 0.6852191112597177, "acc": 0.59375, "ece": 0.13784091233828547, "brier": 0.49152420789994505, "mean_conf": 0.6535524222119982, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.36392465430988447, "error_rate_at_0_9": null, "confidence_bias": 0.05980242221199816, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.6470588235294118, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 1.0, "accuracy": 0.59375, "confidence_cutoff": 0.5312093775743617 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.6852191112597177, "acc": 0.59375, "ece": 0.13784091233828547, "brier": 0.49152420789994505, "mean_conf": 0.6535524222119982, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.36392465430988447, "error_rate_at_0_9": null, "confidence_bias": 0.05980242221199816, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.6470588235294118, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 1.0, "accuracy": 0.59375, "confidence_cutoff": 0.5312093775743617 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 24, "accuracy": 0.75, "full_probability_metrics": { "n": 32, "nll": 0.599781568152657, "acc": 0.75, "ece": 0.17743844039204257, "brier": 0.4160551599811939, "mean_conf": 0.6860587595991833, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.15625, "coverage_at_1pct_error": 0.15625, "aurc": 0.208582880242363, "error_rate_at_0_9": null, "confidence_bias": -0.06394124040081672, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.625, "accuracy": 0.75, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.875, "accuracy": 0.7142857142857143, "confidence_cutoff": 0.5621765088386383 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.599781568152657, "acc": 0.75, "ece": 0.17743844039204257, "brier": 0.4160551599811939, "mean_conf": 0.6860587595991833, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.15625, "coverage_at_1pct_error": 0.15625, "aurc": 0.208582880242363, "error_rate_at_0_9": null, "confidence_bias": -0.06394124040081672, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.625, "accuracy": 0.75, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.875, "accuracy": 0.7142857142857143, "confidence_cutoff": 0.5621765088386383 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 17, "accuracy": 0.53125, "full_probability_metrics": { "n": 32, "nll": 0.6924044194686412, "acc": 0.53125, "ece": 0.08037292918423755, "brier": 0.49945749844962634, "mean_conf": 0.5748460586951171, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.41768973214285715, "error_rate_at_0_9": null, "confidence_bias": 0.04359605869511707, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.9375, "accuracy": 0.5333333333333333, "confidence_cutoff": 0.5312093775743617 }, "0.8": { "coverage": 0.9375, "accuracy": 0.5333333333333333, "confidence_cutoff": 0.5312093775743617 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.6924044194686412, "acc": 0.53125, "ece": 0.08037292918423755, "brier": 0.49945749844962634, "mean_conf": 0.5748460586951171, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.41768973214285715, "error_rate_at_0_9": null, "confidence_bias": 0.04359605869511707, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.9375, "accuracy": 0.5333333333333333, "confidence_cutoff": 0.5312093775743617 }, "0.8": { "coverage": 0.9375, "accuracy": 0.5333333333333333, "confidence_cutoff": 0.5312093775743617 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 39, "accuracy": 0.975, "full_probability_metrics": { "n": 40, "nll": 0.15304291279547544, "acc": 0.975, "ece": 0.11150452634319664, "brier": 0.059213383073962914, "mean_conf": 0.8681647689081728, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10683523109182713, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 1.0, "confidence_cutoff": 0.9046505093574524 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8519528061151505 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.15304291279547544, "acc": 0.975, "ece": 0.11150452634319664, "brier": 0.059213383073962914, "mean_conf": 0.8681647689081728, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10683523109182713, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 1.0, "confidence_cutoff": 0.9046505093574524 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8519528061151505 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 21, "accuracy": 0.525, "full_probability_metrics": { "n": 40, "nll": 1.2319041756035396, "acc": 0.525, "ece": 0.3699352992669369, "brier": 0.7539063357122022, "mean_conf": 0.8949352992669368, "confident_error_rate": 0.15, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 0.7391304347826086, "coverage_at_5pct_error": 0.325, "coverage_at_1pct_error": 0.325, "aurc": 0.22172487042854044, "error_rate_at_0_9": 0.2608695652173913, "confidence_bias": 0.3699352992669368, "top_bins": { "0.9": { "n": 23, "errors": 6, "error_rate": 0.2608695652173913 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.7142857142857143, "confidence_cutoff": 0.9130832842821273 }, "0.8": { "coverage": 0.8, "accuracy": 0.59375, "confidence_cutoff": 0.8517093277703526 } }, "score_mae": 0.48048659564651397, "ranked_probability_score": 0.19770359364836126 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.2319041756035396, "acc": 0.525, "ece": 0.3699352992669369, "brier": 0.7539063357122022, "mean_conf": 0.8949352992669368, "confident_error_rate": 0.15, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 0.7391304347826086, "coverage_at_5pct_error": 0.325, "coverage_at_1pct_error": 0.325, "aurc": 0.22172487042854044, "error_rate_at_0_9": 0.2608695652173913, "confidence_bias": 0.3699352992669368, "top_bins": { "0.9": { "n": 23, "errors": 6, "error_rate": 0.2608695652173913 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.7142857142857143, "confidence_cutoff": 0.9130832842821273 }, "0.8": { "coverage": 0.8, "accuracy": 0.59375, "confidence_cutoff": 0.8517093277703526 } }, "score_mae": 0.48048659564651397, "ranked_probability_score": 0.19770359364836126 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 52, "accuracy": 0.65, "full_probability_metrics": { "n": 80, "nll": 1.2991443436251493, "acc": 0.65, "ece": 0.14391917572945798, "brier": 0.5280414718903804, "mean_conf": 0.6959401739892926, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 0.8636363636363636, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.20075605624478937, "error_rate_at_0_9": 0.13636363636363635, "confidence_bias": 0.04594017398929262, "top_bins": { "0.9": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.95": { "n": 16, "errors": 2, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.775, "confidence_cutoff": 0.7103974809259211 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.4672345468972216 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.2991443436251493, "acc": 0.65, "ece": 0.14391917572945798, "brier": 0.5280414718903804, "mean_conf": 0.6959401739892926, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 0.8636363636363636, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.20075605624478937, "error_rate_at_0_9": 0.13636363636363635, "confidence_bias": 0.04594017398929262, "top_bins": { "0.9": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.95": { "n": 16, "errors": 2, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.775, "confidence_cutoff": 0.7103974809259211 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.4672345468972216 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 53, "accuracy": 0.6625, "full_probability_metrics": { "n": 80, "nll": 0.8458015712221141, "acc": 0.6625, "ece": 0.11878555485760586, "brier": 0.420282028150366, "mean_conf": 0.7743550267098469, "confident_error_rate": 0.025, "coverage_at_0_9": 0.475, "accuracy_at_0_9": 0.9473684210526315, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.1625, "aurc": 0.11798891104228289, "error_rate_at_0_9": 0.05263157894736842, "confidence_bias": 0.11185502670984693, "top_bins": { "0.9": { "n": 38, "errors": 2, "error_rate": 0.05263157894736842 }, "0.95": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.8767757758727135 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.5494763851165771 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.8458015712221141, "acc": 0.6625, "ece": 0.11878555485760586, "brier": 0.420282028150366, "mean_conf": 0.7743550267098469, "confident_error_rate": 0.025, "coverage_at_0_9": 0.475, "accuracy_at_0_9": 0.9473684210526315, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.1625, "aurc": 0.11798891104228289, "error_rate_at_0_9": 0.05263157894736842, "confidence_bias": 0.11185502670984693, "top_bins": { "0.9": { "n": 38, "errors": 2, "error_rate": 0.05263157894736842 }, "0.95": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.8767757758727135 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.5494763851165771 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 90, "accuracy": 0.45, "full_probability_metrics": { "n": 200, "nll": 1.7107358624049716, "acc": 0.45, "ece": 0.17683018871263062, "brier": 0.7316651997135892, "mean_conf": 0.622309792866119, "confident_error_rate": 0.055, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.78, "coverage_at_5pct_error": 0.09, "coverage_at_1pct_error": 0.09, "aurc": 0.3360987748557564, "error_rate_at_0_9": 0.22, "confidence_bias": 0.17230979286611897, "top_bins": { "0.9": { "n": 50, "errors": 11, "error_rate": 0.22 }, "0.95": { "n": 36, "errors": 4, "error_rate": 0.1111111111111111 }, "0.99": { "n": 14, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.61, "confidence_cutoff": 0.59842231230227 }, "0.8": { "coverage": 0.8, "accuracy": 0.49375, "confidence_cutoff": 0.34648628893585864 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 1.7107358624049716, "acc": 0.45, "ece": 0.17683018871263062, "brier": 0.7316651997135892, "mean_conf": 0.622309792866119, "confident_error_rate": 0.055, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.78, "coverage_at_5pct_error": 0.09, "coverage_at_1pct_error": 0.09, "aurc": 0.3360987748557564, "error_rate_at_0_9": 0.22, "confidence_bias": 0.17230979286611897, "top_bins": { "0.9": { "n": 50, "errors": 11, "error_rate": 0.22 }, "0.95": { "n": 36, "errors": 4, "error_rate": 0.1111111111111111 }, "0.99": { "n": 14, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.61, "confidence_cutoff": 0.59842231230227 }, "0.8": { "coverage": 0.8, "accuracy": 0.49375, "confidence_cutoff": 0.34648628893585864 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 67, "accuracy": 0.8375, "full_probability_metrics": { "n": 80, "nll": 0.35533320626265397, "acc": 0.8375, "ece": 0.06349571991158881, "brier": 0.22298675541219115, "mean_conf": 0.8889640418012277, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9423076923076923, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.3625, "aurc": 0.04826210748102641, "error_rate_at_0_9": 0.057692307692307696, "confidence_bias": 0.051464041801227656, "top_bins": { "0.9": { "n": 52, "errors": 3, "error_rate": 0.057692307692307696 }, "0.95": { "n": 44, "errors": 2, "error_rate": 0.045454545454545456 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.957912266254425 }, "0.8": { "coverage": 0.8375, "accuracy": 0.8955223880597015, "confidence_cutoff": 0.7981867790222168 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.35533320626265397, "acc": 0.8375, "ece": 0.06349571991158881, "brier": 0.22298675541219115, "mean_conf": 0.8889640418012277, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9423076923076923, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.3625, "aurc": 0.04826210748102641, "error_rate_at_0_9": 0.057692307692307696, "confidence_bias": 0.051464041801227656, "top_bins": { "0.9": { "n": 52, "errors": 3, "error_rate": 0.057692307692307696 }, "0.95": { "n": 44, "errors": 2, "error_rate": 0.045454545454545456 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.957912266254425 }, "0.8": { "coverage": 0.8375, "accuracy": 0.8955223880597015, "confidence_cutoff": 0.7981867790222168 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 70, "accuracy": 0.875, "full_probability_metrics": { "n": 80, "nll": 0.27008576430809955, "acc": 0.875, "ece": 0.06270969004835936, "brier": 0.16838197731753382, "mean_conf": 0.8621369416126982, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7875, "coverage_at_1pct_error": 0.675, "aurc": 0.023823625191530473, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01286305838730184, "top_bins": { "0.9": { "n": 44, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 1.0, "confidence_cutoff": 0.9149009585380554 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9384615384615385, "confidence_cutoff": 0.7549149990081787 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.27008576430809955, "acc": 0.875, "ece": 0.06270969004835936, "brier": 0.16838197731753382, "mean_conf": 0.8621369416126982, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7875, "coverage_at_1pct_error": 0.675, "aurc": 0.023823625191530473, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01286305838730184, "top_bins": { "0.9": { "n": 44, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 1.0, "confidence_cutoff": 0.9149009585380554 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9384615384615385, "confidence_cutoff": 0.7549149990081787 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 79, "accuracy": 0.9875, "full_probability_metrics": { "n": 80, "nll": 0.024597743607985088, "acc": 0.9875, "ece": 0.017201267457787762, "brier": 0.01401481488332792, "mean_conf": 0.9876950562653001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.00019505626530003095, "top_bins": { "0.9": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 69, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9983088634911936 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9957912149074755 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.024597743607985088, "acc": 0.9875, "ece": 0.017201267457787762, "brier": 0.01401481488332792, "mean_conf": 0.9876950562653001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.00019505626530003095, "top_bins": { "0.9": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 69, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9983088634911936 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9957912149074755 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 62, "accuracy": 0.775, "full_probability_metrics": { "n": 80, "nll": 0.4788035138896979, "acc": 0.775, "ece": 0.13131222857955432, "brier": 0.31136528839064526, "mean_conf": 0.8788950858903263, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.6375, "accuracy_at_0_9": 0.9019607843137255, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.4125, "aurc": 0.07463234449967474, "error_rate_at_0_9": 0.09803921568627451, "confidence_bias": 0.10389508589032626, "top_bins": { "0.9": { "n": 51, "errors": 5, "error_rate": 0.09803921568627451 }, "0.95": { "n": 43, "errors": 3, "error_rate": 0.06976744186046512 }, "0.99": { "n": 29, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9626731164753437 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7310585677623749 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4788035138896979, "acc": 0.775, "ece": 0.13131222857955432, "brier": 0.31136528839064526, "mean_conf": 0.8788950858903263, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.6375, "accuracy_at_0_9": 0.9019607843137255, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.4125, "aurc": 0.07463234449967474, "error_rate_at_0_9": 0.09803921568627451, "confidence_bias": 0.10389508589032626, "top_bins": { "0.9": { "n": 51, "errors": 5, "error_rate": 0.09803921568627451 }, "0.95": { "n": 43, "errors": 3, "error_rate": 0.06976744186046512 }, "0.99": { "n": 29, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9626731164753437 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7310585677623749 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.025223510056745335, "acc": 1.0, "ece": 0.024870141968131088, "brier": 0.0013850173812376646, "mean_conf": 0.9751298580318689, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024870141968131088, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.977022647857666 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9669140577316284 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.025223510056745335, "acc": 1.0, "ece": 0.024870141968131088, "brier": 0.0013850173812376646, "mean_conf": 0.9751298580318689, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024870141968131088, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.977022647857666 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9669140577316284 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.1298511164229339, "acc": 1.0, "ece": 0.11604033336043357, "brier": 0.04391351760878065, "mean_conf": 0.8839596666395664, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.11604033336043362, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9241418242454529 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.867035761475563 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.1298511164229339, "acc": 1.0, "ece": 0.11604033336043357, "brier": 0.04391351760878065, "mean_conf": 0.8839596666395664, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.11604033336043362, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9241418242454529 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.867035761475563 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.28317685070455, "acc": 0.5, "ece": 0.3989198412928101, "brier": 0.7936511974742065, "mean_conf": 0.8989198412928101, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.22813492063492066, "error_rate_at_0_9": 0.2, "confidence_bias": 0.3989198412928101, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9168976629023815 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8552710031596867 } }, "score_mae": 0.5078395118506894, "ranked_probability_score": 0.20872340330835004 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.28317685070455, "acc": 0.5, "ece": 0.3989198412928101, "brier": 0.7936511974742065, "mean_conf": 0.8989198412928101, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.22813492063492066, "error_rate_at_0_9": 0.2, "confidence_bias": 0.3989198412928101, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9168976629023815 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8552710031596867 } }, "score_mae": 0.5078395118506894, "ranked_probability_score": 0.20872340330835004 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.315748672136443, "acc": 0.8, "ece": 0.16795399992779297, "brier": 0.22667555248570997, "mean_conf": 0.9204368209954715, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.12043682099547148, "top_bins": { "0.9": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9929405196515245 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8429129860466782 } }, "score_mae": 0.1564949542213734, "ranked_probability_score": 0.05347760706701671 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.315748672136443, "acc": 0.8, "ece": 0.16795399992779297, "brier": 0.22667555248570997, "mean_conf": 0.9204368209954715, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.12043682099547148, "top_bins": { "0.9": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9929405196515245 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8429129860466782 } }, "score_mae": 0.1564949542213734, "ranked_probability_score": 0.05347760706701671 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.5602286205594307, "acc": 0.7, "ece": 0.2873367316105812, "brier": 0.3823452670870403, "mean_conf": 0.7218438101527778, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.17023809523809524, "error_rate_at_0_9": 0.0, "confidence_bias": 0.02184381015277781, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7240769836154692 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5910443995245035 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5602286205594307, "acc": 0.7, "ece": 0.2873367316105812, "brier": 0.3823452670870403, "mean_conf": 0.7218438101527778, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.17023809523809524, "error_rate_at_0_9": 0.0, "confidence_bias": 0.02184381015277781, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7240769836154692 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5910443995245035 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 0.7818441070326679, "acc": 0.4, "ece": 0.22078231573104856, "brier": 0.5844299134575873, "mean_conf": 0.6207823157310486, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6223412698412698, "error_rate_at_0_9": null, "confidence_bias": 0.22078231573104856, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6513549089431763 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5926666259765625 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7818441070326679, "acc": 0.4, "ece": 0.22078231573104856, "brier": 0.5844299134575873, "mean_conf": 0.6207823157310486, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6223412698412698, "error_rate_at_0_9": null, "confidence_bias": 0.22078231573104856, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6513549089431763 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5926666259765625 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 0.804533226690723, "acc": 0.6, "ece": 0.2340373569076295, "brier": 0.475886021350622, "mean_conf": 0.6281279752114293, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.1927380952380952, "error_rate_at_0_9": null, "confidence_bias": 0.02812797521142929, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.5908308117215456 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5465493961140334 } }, "score_mae": 0.5310390617134967, "ranked_probability_score": 0.14893273680900337 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.804533226690723, "acc": 0.6, "ece": 0.2340373569076295, "brier": 0.475886021350622, "mean_conf": 0.6281279752114293, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.1927380952380952, "error_rate_at_0_9": null, "confidence_bias": 0.02812797521142929, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.5908308117215456 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5465493961140334 } }, "score_mae": 0.5310390617134967, "ranked_probability_score": 0.14893273680900337 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.8673243057110749, "acc": 0.5, "ece": 0.34069017812107877, "brier": 0.6151627684436366, "mean_conf": 0.8057588661097611, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.26563492063492067, "error_rate_at_0_9": 0.0, "confidence_bias": 0.3057588661097611, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8311570503491884 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6927553116034197 } }, "score_mae": 0.4462295179629626, "ranked_probability_score": 0.15432601193155754 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.8673243057110749, "acc": 0.5, "ece": 0.34069017812107877, "brier": 0.6151627684436366, "mean_conf": 0.8057588661097611, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.26563492063492067, "error_rate_at_0_9": 0.0, "confidence_bias": 0.3057588661097611, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8311570503491884 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6927553116034197 } }, "score_mae": 0.4462295179629626, "ranked_probability_score": 0.15432601193155754 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0009397492052290771, "acc": 1.0, "ece": 0.0009389555488823476, "brier": 2.5304794032352415e-06, "mean_conf": 0.9990610444511177, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0009389555488823476, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999372457176099 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9990820056879174 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0009397492052290771, "acc": 1.0, "ece": 0.0009389555488823476, "brier": 2.5304794032352415e-06, "mean_conf": 0.9990610444511177, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0009389555488823476, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999372457176099 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9990820056879174 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.032994434069061856, "acc": 1.0, "ece": 0.032021431691086066, "brier": 0.0032790083929257674, "mean_conf": 0.9679785683089139, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.032021431691086066, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9883039407241394 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9405974281103923 } }, "score_mae": 0.03637617069097921, "ranked_probability_score": 0.0009428956817269238 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.032994434069061856, "acc": 1.0, "ece": 0.032021431691086066, "brier": 0.0032790083929257674, "mean_conf": 0.9679785683089139, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.032021431691086066, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9883039407241394 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9405974281103923 } }, "score_mae": 0.03637617069097921, "ranked_probability_score": 0.0009428956817269238 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.909174389475378, "acc": 0.5, "ece": 0.18418607341118262, "brier": 0.5688427500607292, "mean_conf": 0.6252629754981021, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.348968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.12526297549810206, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6339176108265635 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5908308117215456 } }, "score_mae": 0.48190318740123983, "ranked_probability_score": 0.158576883627425 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.909174389475378, "acc": 0.5, "ece": 0.18418607341118262, "brier": 0.5688427500607292, "mean_conf": 0.6252629754981021, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.348968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.12526297549810206, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6339176108265635 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5908308117215456 } }, "score_mae": 0.48190318740123983, "ranked_probability_score": 0.158576883627425 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.7023148148148147, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.7777777777777778, "flip_rate": 0.1388888888888889, "covered_only_flip_rate": 0.1388888888888889, "half_l1": 0.13898033187034217 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.5989395145691971, "share_at_0_9": 0.00909090909090909, "normalized_entropy": 0.8119652720869368 }, "covered_only_confidence_metrics": { "mean_max_p": 0.5989395145691971, "share_at_0_9": 0.00909090909090909, "normalized_entropy": 0.8119652720869368 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 80, "accuracy": 0.7272727272727273, "full_probability_metrics": { "n": 110, "nll": 0.5191853620058399, "acc": 0.7272727272727273, "ece": 0.1061678090423689, "brier": 0.33596123129289807, "mean_conf": 0.8224783402202606, "confident_error_rate": 0.00909090909090909, "coverage_at_0_9": 0.4727272727272727, "accuracy_at_0_9": 0.9807692307692307, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.42727272727272725, "aurc": 0.0882085859234952, "error_rate_at_0_9": 0.019230769230769232, "confidence_bias": 0.09520561294753327, "top_bins": { "0.9": { "n": 52, "errors": 1, "error_rate": 0.019230769230769232 }, "0.95": { "n": 37, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9636363636363636, "confidence_cutoff": 0.8807970806956291 }, "0.8": { "coverage": 0.8, "accuracy": 0.8068181818181818, "confidence_cutoff": 0.6339176108265635 } }, "score_mae": 0.3599804006401236, "ranked_probability_score": 0.1208299230708466 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.5191853620058399, "acc": 0.7272727272727273, "ece": 0.1061678090423689, "brier": 0.33596123129289807, "mean_conf": 0.8224783402202606, "confident_error_rate": 0.00909090909090909, "coverage_at_0_9": 0.4727272727272727, "accuracy_at_0_9": 0.9807692307692307, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.42727272727272725, "aurc": 0.0882085859234952, "error_rate_at_0_9": 0.019230769230769232, "confidence_bias": 0.09520561294753327, "top_bins": { "0.9": { "n": 52, "errors": 1, "error_rate": 0.019230769230769232 }, "0.95": { "n": 37, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 20, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9636363636363636, "confidence_cutoff": 0.8807970806956291 }, "0.8": { "coverage": 0.8, "accuracy": 0.8068181818181818, "confidence_cutoff": 0.6339176108265635 } }, "score_mae": 0.3599804006401236, "ranked_probability_score": 0.1208299230708466 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.2235388256510637 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.6620898054587592, "paired_flip": { "pairs": 64, "flip_rate": 0.640625, "both_correct_rate": 0.53125, "invariant_pairs": 24, "invariance_rate": 0.9583333333333334, "invariant_both_correct_rate": 0.5833333333333334 }, "unknowable": { "n": 110, "mean_max_p": 0.5989395145691971, "share_at_0_9": 0.00909090909090909, "control_mean_max_p": 0.8224783402202606, "control_share_at_0_9": 0.4727272727272727, "control_acc": 0.7272727272727273, "paired_confidence_drop": 0.2235388256510637, "share_less_confident_than_control": 0.9181818181818182 }, "clean": { "n": 1046, "nll": 0.8185509865100163, "acc": 0.6883365200764818, "ece": 0.08978015832969408, "brier": 0.4198595508629067, "mean_conf": 0.77574690988783, "confident_error_rate": 0.032504780114722756, "coverage_at_0_9": 0.42638623326959846, "accuracy_at_0_9": 0.9237668161434978, "coverage_at_5pct_error": 0.34703632887189295, "coverage_at_1pct_error": 0.1959847036328872, "aurc": 0.12334711495626294, "error_rate_at_0_9": 0.07623318385650224, "confidence_bias": 0.08741038981134819, "top_bins": { "0.9": { "n": 446, "errors": 34, "error_rate": 0.07623318385650224 }, "0.95": { "n": 331, "errors": 12, "error_rate": 0.03625377643504532 }, "0.99": { "n": 172, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5038240917782026, "accuracy": 0.8956356736242884, "confidence_cutoff": 0.8354835361242294 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.7589073634204275, "confidence_cutoff": 0.5621765088386383 } }, "score_mae": 0.40818287864267966, "ranked_probability_score": 0.15157939130185247 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.7616024963066572, "acc": 0.45, "ece": 0.23555467490844156, "brier": 0.8076512996162115, "mean_conf": 0.6326583796971109, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.5139849014075021, "error_rate_at_0_9": 0.5, "confidence_bias": 0.1826583796971109, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.620902601373769 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.4701217430462008 } } }, "buried_paws": { "n": 20, "nll": 0.5347892841599211, "acc": 0.75, "ece": 0.12973636761307714, "brier": 0.361960551470936, "mean_conf": 0.7220932058990002, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.17242260061919504, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027906794100999832, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7549149841070175 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6224593222141266 } } }, "buried_qnli": { "n": 20, "nll": 0.4139708021155313, "acc": 0.75, "ece": 0.22482597026973963, "brier": 0.2769511861963836, "mean_conf": 0.7978020852431655, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.07702878390958887, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04780208524316554, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8519528061151505 }, "0.8": { "coverage": 0.85, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.6513548493385315 } } }, "buried_tweet_offensive": { "n": 20, "nll": 1.108168990139449, "acc": 0.4, "ece": 0.3968666262924671, "brier": 0.8098649357203225, "mean_conf": 0.7310038901865482, "confident_error_rate": 0.1, "coverage_at_0_9": 0.15, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.6227259363443574, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.3310038901865482, "top_bins": { "0.9": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.7549149990081787 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.562176525592804 } } }, "composition_final_combination": { "n": 32, "nll": 0.6852191112597177, "acc": 0.59375, "ece": 0.13784091233828547, "brier": 0.49152420789994505, "mean_conf": 0.6535524222119982, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.36392465430988447, "error_rate_at_0_9": null, "confidence_bias": 0.05980242221199816, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.6470588235294118, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 1.0, "accuracy": 0.59375, "confidence_cutoff": 0.5312093775743617 } } }, "composition_final_exception": { "n": 32, "nll": 0.599781568152657, "acc": 0.75, "ece": 0.17743844039204257, "brier": 0.4160551599811939, "mean_conf": 0.6860587595991833, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.15625, "coverage_at_1pct_error": 0.15625, "aurc": 0.208582880242363, "error_rate_at_0_9": null, "confidence_bias": -0.06394124040081672, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.625, "accuracy": 0.75, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.875, "accuracy": 0.7142857142857143, "confidence_cutoff": 0.5621765088386383 } } }, "composition_final_negation": { "n": 32, "nll": 0.6924044194686412, "acc": 0.53125, "ece": 0.08037292918423755, "brier": 0.49945749844962634, "mean_conf": 0.5748460586951171, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.41768973214285715, "error_rate_at_0_9": null, "confidence_bias": 0.04359605869511707, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.9375, "accuracy": 0.5333333333333333, "confidence_cutoff": 0.5312093775743617 }, "0.8": { "coverage": 0.9375, "accuracy": 0.5333333333333333, "confidence_cutoff": 0.5312093775743617 } } }, "contrastive_authorization": { "n": 40, "nll": 0.15304291279547544, "acc": 0.975, "ece": 0.11150452634319664, "brier": 0.059213383073962914, "mean_conf": 0.8681647689081728, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10683523109182713, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 1.0, "confidence_cutoff": 0.9046505093574524 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8519528061151505 } } }, "contrastive_deadline": { "n": 40, "nll": 1.2319041756035396, "acc": 0.525, "ece": 0.3699352992669369, "brier": 0.7539063357122022, "mean_conf": 0.8949352992669368, "confident_error_rate": 0.15, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 0.7391304347826086, "coverage_at_5pct_error": 0.325, "coverage_at_1pct_error": 0.325, "aurc": 0.22172487042854044, "error_rate_at_0_9": 0.2608695652173913, "confidence_bias": 0.3699352992669368, "top_bins": { "0.9": { "n": 23, "errors": 6, "error_rate": 0.2608695652173913 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.7142857142857143, "confidence_cutoff": 0.9130832842821273 }, "0.8": { "coverage": 0.8, "accuracy": 0.59375, "confidence_cutoff": 0.8517093277703526 } }, "score_mae": 0.48048659564651397, "ranked_probability_score": 0.19770359364836126 }, "emotion": { "n": 80, "nll": 1.2991443436251493, "acc": 0.65, "ece": 0.14391917572945798, "brier": 0.5280414718903804, "mean_conf": 0.6959401739892926, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 0.8636363636363636, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.20075605624478937, "error_rate_at_0_9": 0.13636363636363635, "confidence_bias": 0.04594017398929262, "top_bins": { "0.9": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.95": { "n": 16, "errors": 2, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.775, "confidence_cutoff": 0.7103974809259211 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.4672345468972216 } } }, "mmlu": { "n": 80, "nll": 0.8458015712221141, "acc": 0.6625, "ece": 0.11878555485760586, "brier": 0.420282028150366, "mean_conf": 0.7743550267098469, "confident_error_rate": 0.025, "coverage_at_0_9": 0.475, "accuracy_at_0_9": 0.9473684210526315, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.1625, "aurc": 0.11798891104228289, "error_rate_at_0_9": 0.05263157894736842, "confidence_bias": 0.11185502670984693, "top_bins": { "0.9": { "n": 38, "errors": 2, "error_rate": 0.05263157894736842 }, "0.95": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.8767757758727135 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.5494763851165771 } } }, "mmlu_pro": { "n": 200, "nll": 1.7107358624049716, "acc": 0.45, "ece": 0.17683018871263062, "brier": 0.7316651997135892, "mean_conf": 0.622309792866119, "confident_error_rate": 0.055, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.78, "coverage_at_5pct_error": 0.09, "coverage_at_1pct_error": 0.09, "aurc": 0.3360987748557564, "error_rate_at_0_9": 0.22, "confidence_bias": 0.17230979286611897, "top_bins": { "0.9": { "n": 50, "errors": 11, "error_rate": 0.22 }, "0.95": { "n": 36, "errors": 4, "error_rate": 0.1111111111111111 }, "0.99": { "n": 14, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.61, "confidence_cutoff": 0.59842231230227 }, "0.8": { "coverage": 0.8, "accuracy": 0.49375, "confidence_cutoff": 0.34648628893585864 } } }, "paws": { "n": 80, "nll": 0.35533320626265397, "acc": 0.8375, "ece": 0.06349571991158881, "brier": 0.22298675541219115, "mean_conf": 0.8889640418012277, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.9423076923076923, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.3625, "aurc": 0.04826210748102641, "error_rate_at_0_9": 0.057692307692307696, "confidence_bias": 0.051464041801227656, "top_bins": { "0.9": { "n": 52, "errors": 3, "error_rate": 0.057692307692307696 }, "0.95": { "n": 44, "errors": 2, "error_rate": 0.045454545454545456 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.957912266254425 }, "0.8": { "coverage": 0.8375, "accuracy": 0.8955223880597015, "confidence_cutoff": 0.7981867790222168 } } }, "qnli": { "n": 80, "nll": 0.27008576430809955, "acc": 0.875, "ece": 0.06270969004835936, "brier": 0.16838197731753382, "mean_conf": 0.8621369416126982, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7875, "coverage_at_1pct_error": 0.675, "aurc": 0.023823625191530473, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01286305838730184, "top_bins": { "0.9": { "n": 44, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 26, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 1.0, "confidence_cutoff": 0.9149009585380554 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9384615384615385, "confidence_cutoff": 0.7549149990081787 } } }, "sciq": { "n": 80, "nll": 0.024597743607985088, "acc": 0.9875, "ece": 0.017201267457787762, "brier": 0.01401481488332792, "mean_conf": 0.9876950562653001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.00019505626530003095, "top_bins": { "0.9": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 69, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9983088634911936 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9957912149074755 } } }, "tweet_offensive": { "n": 80, "nll": 0.4788035138896979, "acc": 0.775, "ece": 0.13131222857955432, "brier": 0.31136528839064526, "mean_conf": 0.8788950858903263, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.6375, "accuracy_at_0_9": 0.9019607843137255, "coverage_at_5pct_error": 0.5125, "coverage_at_1pct_error": 0.4125, "aurc": 0.07463234449967474, "error_rate_at_0_9": 0.09803921568627451, "confidence_bias": 0.10389508589032626, "top_bins": { "0.9": { "n": 51, "errors": 5, "error_rate": 0.09803921568627451 }, "0.95": { "n": 43, "errors": 3, "error_rate": 0.06976744186046512 }, "0.99": { "n": 29, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9626731164753437 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7310585677623749 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.7277422705180417, "acc": 0.5, "ece": 0.12156718969345096, "brier": 0.5329850387950508, "mean_conf": 0.6215671896934509, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.12156718969345093, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.6224593222141266 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5926665663719177 } } }, "unknowable_authorization": { "n": 10, "nll": 0.9724163382885477, "acc": 0.5, "ece": 0.39358392953872684, "brier": 0.7070895143783773, "mean_conf": 0.8047850131988525, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.44158730158730164, "error_rate_at_0_9": 0.0, "confidence_bias": 0.30478501319885254, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8354835510253906 }, "0.8": { "coverage": 0.9, "accuracy": 0.4444444444444444, "confidence_cutoff": 0.7310585975646973 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.025223510056745335, "acc": 1.0, "ece": 0.024870141968131088, "brier": 0.0013850173812376646, "mean_conf": 0.9751298580318689, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024870141968131088, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.977022647857666 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9669140577316284 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.1298511164229339, "acc": 1.0, "ece": 0.11604033336043357, "brier": 0.04391351760878065, "mean_conf": 0.8839596666395664, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.11604033336043362, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9241418242454529 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.867035761475563 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.28317685070455, "acc": 0.5, "ece": 0.3989198412928101, "brier": 0.7936511974742065, "mean_conf": 0.8989198412928101, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.22813492063492066, "error_rate_at_0_9": 0.2, "confidence_bias": 0.3989198412928101, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9168976629023815 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8552710031596867 } }, "score_mae": 0.5078395118506894, "ranked_probability_score": 0.20872340330835004 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.315748672136443, "acc": 0.8, "ece": 0.16795399992779297, "brier": 0.22667555248570997, "mean_conf": 0.9204368209954715, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "aurc": 0.03111111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": 0.12043682099547148, "top_bins": { "0.9": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9929405196515245 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8429129860466782 } }, "score_mae": 0.1564949542213734, "ranked_probability_score": 0.05347760706701671 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.5602286205594307, "acc": 0.7, "ece": 0.2873367316105812, "brier": 0.3823452670870403, "mean_conf": 0.7218438101527778, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.17023809523809524, "error_rate_at_0_9": 0.0, "confidence_bias": 0.02184381015277781, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7240769836154692 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5910443995245035 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.7818441070326679, "acc": 0.4, "ece": 0.22078231573104856, "brier": 0.5844299134575873, "mean_conf": 0.6207823157310486, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6223412698412698, "error_rate_at_0_9": null, "confidence_bias": 0.22078231573104856, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6513549089431763 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5926666259765625 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.804533226690723, "acc": 0.6, "ece": 0.2340373569076295, "brier": 0.475886021350622, "mean_conf": 0.6281279752114293, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.1927380952380952, "error_rate_at_0_9": null, "confidence_bias": 0.02812797521142929, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.5908308117215456 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5465493961140334 } }, "score_mae": 0.5310390617134967, "ranked_probability_score": 0.14893273680900337 }, "unknowable_control_sla_response": { "n": 10, "nll": 0.8673243057110749, "acc": 0.5, "ece": 0.34069017812107877, "brier": 0.6151627684436366, "mean_conf": 0.8057588661097611, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.26563492063492067, "error_rate_at_0_9": 0.0, "confidence_bias": 0.3057588661097611, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8311570503491884 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6927553116034197 } }, "score_mae": 0.4462295179629626, "ranked_probability_score": 0.15432601193155754 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.0009397492052290771, "acc": 1.0, "ece": 0.0009389555488823476, "brier": 2.5304794032352415e-06, "mean_conf": 0.9990610444511177, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0009389555488823476, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999372457176099 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9990820056879174 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.032994434069061856, "acc": 1.0, "ece": 0.032021431691086066, "brier": 0.0032790083929257674, "mean_conf": 0.9679785683089139, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.032021431691086066, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9883039407241394 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9405974281103923 } }, "score_mae": 0.03637617069097921, "ranked_probability_score": 0.0009428956817269238 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.909174389475378, "acc": 0.5, "ece": 0.18418607341118262, "brier": 0.5688427500607292, "mean_conf": 0.6252629754981021, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.348968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.12526297549810206, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6339176108265635 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5908308117215456 } }, "score_mae": 0.48190318740123983, "ranked_probability_score": 0.158576883627425 }, "unknowable_deadline": { "n": 10, "nll": 1.3566997743725462, "acc": 0.3, "ece": 0.37761513788312495, "brier": 0.8312853013512175, "mean_conf": 0.6776151378831249, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7258333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.3776151378831249, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6896720862602336 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6671349952583387 } }, "score_mae": 0.6967843327248195, "ranked_probability_score": 0.24193911382547525 }, "unknowable_late_fee": { "n": 10, "nll": 1.1466600060159637, "acc": 0.2, "ece": 0.2047736188314488, "brier": 0.6966814625390321, "mean_conf": 0.4047736188314488, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7933333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.2047736188314488, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.41360626857299415 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.375756539359299 } }, "score_mae": 0.6701006391412532, "ranked_probability_score": 0.22751329976118542 }, "unknowable_quantity_limit": { "n": 10, "nll": 2.0092527042811827, "acc": 0.2, "ece": 0.6216561403809924, "brier": 1.2297471605711467, "mean_conf": 0.8216561403809924, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9016666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.6216561403809924, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.8259012945264964 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.8141003675662173 } } }, "unknowable_return_window": { "n": 10, "nll": 0.6978135101302584, "acc": 0.5, "ece": 0.037258934974670366, "brier": 0.5046036667028062, "mean_conf": 0.5372589349746704, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.037258934974670366, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5312093496322632 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5312093496322632 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.3658304792808509, "acc": 0.3, "ece": 0.14251216160844127, "brier": 0.8052047065445059, "mean_conf": 0.44251216160844126, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7992857142857142, "error_rate_at_0_9": null, "confidence_bias": 0.14251216160844127, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.7, "accuracy": 0.2857142857142857, "confidence_cutoff": 0.4443701240795777 }, "0.8": { "coverage": 1.0, "accuracy": 0.3, "confidence_cutoff": 0.4223187973058761 } }, "score_mae": 0.8626284344799707, "ranked_probability_score": 0.2911051074013177 }, "unknowable_sla_response": { "n": 10, "nll": 1.104195904339522, "acc": 0.3, "ece": 0.09091891782818348, "brier": 0.668855838508042, "mean_conf": 0.39091891782818344, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7483333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.09091891782818345, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.4017633377980191 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.375756539359299 } }, "score_mae": 0.5448122345684135, "ranked_probability_score": 0.1942530831163975 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.9628105150373208, "acc": 0.5, "ece": 0.3090212723555498, "brier": 0.6979119505865701, "mean_conf": 0.8090212723555499, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.3090212723555499, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8130953168041901 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7737237148149109 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.543205924604783, "acc": 0.2, "ece": 0.36897412239170374, "brier": 0.8548948387543256, "mean_conf": 0.5689741223917038, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6433333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.36897412239170374, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.5883927323356541 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5121442828221405 } }, "score_mae": 0.8900047811123242, "ranked_probability_score": 0.304024600788393 }, "unknowable_warranty_claim": { "n": 10, "nll": 0.9894975228002533, "acc": 0.5, "ece": 0.06971356653986657, "brier": 0.5982254675230838, "mean_conf": 0.5092521511147485, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.48813492063492064, "error_rate_at_0_9": null, "confidence_bias": 0.00925215111474853, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5282521090432373 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.48102426528930664 } }, "score_mae": 0.5070625252703578, "ranked_probability_score": 0.1747939497649958 } }, "variants": { "clean": { "n": 1156, "nll": 0.8520463506800776, "acc": 0.657439446366782, "ece": 0.10172459005884273, "brier": 0.4502144806791885, "mean_conf": 0.758922676769275, "confident_error_rate": 0.029411764705882353, "coverage_at_0_9": 0.3866782006920415, "accuracy_at_0_9": 0.9239373601789709, "coverage_at_5pct_error": 0.3140138408304498, "coverage_at_1pct_error": 0.1773356401384083, "aurc": 0.14442497872079563, "error_rate_at_0_9": 0.07606263982102908, "confidence_bias": 0.10148323040249307, "top_bins": { "0.9": { "n": 447, "errors": 34, "error_rate": 0.07606263982102908 }, "0.95": { "n": 331, "errors": 12, "error_rate": 0.03625377643504532 }, "0.99": { "n": 172, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8633217993079585, "confidence_cutoff": 0.8023303210908943 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.7329729729729729, "confidence_cutoff": 0.5378948232173688 } }, "score_mae": 0.515826358357746, "ranked_probability_score": 0.18433894172976809 }, "none_absent": { "n": 36, "nll": 0.7384471001802487, "acc": 0.6944444444444444, "ece": 0.13877030548112615, "brier": 0.44447813786431567, "mean_conf": 0.6849277022185669, "confident_error_rate": 0.0, "coverage_at_0_9": 0.16666666666666666, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.1881415305686208, "error_rate_at_0_9": 0.0, "confidence_bias": -0.009516742225877506, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7777777777777778, "confidence_cutoff": 0.6895400077010968 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7241379310344828, "confidence_cutoff": 0.4998892308199406 } } }, "none_present": { "n": 36, "nll": 1.0205542416541602, "acc": 0.75, "ece": 0.1502595391205593, "brier": 0.41221107156815195, "mean_conf": 0.7468122907241654, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3611111111111111, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4166666666666667, "coverage_at_1pct_error": 0.4166666666666667, "aurc": 0.1067426579613761, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0031877092758345604, "top_bins": { "0.9": { "n": 13, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8888888888888888, "confidence_cutoff": 0.7937426889608805 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7931034482758621, "confidence_cutoff": 0.5292905514853112 } } }, "permuted": { "n": 36, "nll": 0.6806943337072539, "acc": 0.8333333333333334, "ece": 0.12021384007910868, "brier": 0.2631600927780251, "mean_conf": 0.8037173277760403, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4722222222222222, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7777777777777778, "coverage_at_1pct_error": 0.6944444444444444, "aurc": 0.03173078947464249, "error_rate_at_0_9": 0.0, "confidence_bias": -0.029616005557293046, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8951721441514457 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.9310344827586207, "confidence_cutoff": 0.6018614967969089 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.12393603363445362, "flip_rate": 0.1388888888888889 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.8185509865100163, "acc": 0.6883365200764818, "ece": 0.08978015832969408, "brier": 0.4198595508629067, "mean_conf": 0.77574690988783, "confident_error_rate": 0.032504780114722756, "coverage_at_0_9": 0.42638623326959846, "accuracy_at_0_9": 0.9237668161434978, "coverage_at_5pct_error": 0.34703632887189295, "coverage_at_1pct_error": 0.1959847036328872, "aurc": 0.12334711495626294, "error_rate_at_0_9": 0.07623318385650224, "confidence_bias": 0.08741038981134819, "top_bins": { "0.9": { "n": 446, "errors": 34, "error_rate": 0.07623318385650224 }, "0.95": { "n": 331, "errors": 12, "error_rate": 0.03625377643504532 }, "0.99": { "n": 172, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5038240917782026, "accuracy": 0.8956356736242884, "confidence_cutoff": 0.8354835361242294 }, "0.8": { "coverage": 0.8049713193116634, "accuracy": 0.7589073634204275, "confidence_cutoff": 0.5621765088386383 } }, "score_mae": 0.40818287864267966, "ranked_probability_score": 0.15157939130185247 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Sol": { "panels": { "old_core": { "point": 0.7374627976190476, "exact_fraction": "19823/26880", "ci95": [ 0.6992931547619048, 0.7742950148809524 ] }, "v3_core": { "point": 0.4608333333333333, "exact_fraction": "553/1200", "ci95": [ 0.43875000000000003, 0.48291666666666666 ] }, "v4": { "point": 0.765625, "exact_fraction": "49/64", "ci95": [ 0.7210721817484662, 0.8086081288343558 ] }, "v5": { "point": 0.8416666666666667, "exact_fraction": "101/120", "ci95": [ 0.8111792524213076, 0.8710143196177259 ] }, "transfer_v9_test": { "point": 0.5707456978967496, "exact_fraction": "597/1046", "ci95": [ 0.5390532509619704, 0.6026013588654585 ] }, "original_four_panel_mean": { "point": 0.7013969494047619, "exact_fraction": "377071/537600", "ci95": [ 0.6845734141379205, 0.7180071456587547 ] }, "weighted_mean": { "point": 0.66315277730356, "exact_fraction": "93227609/140582400", "ci95": [ 0.6477457765148583, 0.6784570084760436 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 107, "requested": 128 }, "boolean_constraints": { "correct": 32, "requested": 64 }, "dbpedia_14": { "correct": 107, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 95, "requested": 96 }, "ordinal_rubric": { "correct": 42, "requested": 64 }, "relational_composition": { "correct": 36, "requested": 96 }, "scoped_evidence": { "correct": 76, "requested": 96 }, "state_tracking": { "correct": 26, "requested": 96 }, "unknown_rejection": { "correct": 64, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 40, "requested": 80 }, "capacitated_assignment": { "correct": 38, "requested": 80 }, "constraint_assignment": { "correct": 21, "requested": 80 }, "massive_en": { "correct": 109, "requested": 120 }, "massive_zh": { "correct": 105, "requested": 120 }, "multiset_reconciliation": { "correct": 23, "requested": 80 }, "ordinal_service_loss": { "correct": 20, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 20, "requested": 80 }, "temporal_exclusion": { "correct": 33, "requested": 80 }, "transaction_recovery": { "correct": 31, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 133, "requested": 160 }, "belebele_en": { "correct": 112, "requested": 160 }, "belebele_zh": { "correct": 112, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 83, "requested": 120 }, "squad2_answerability": { "correct": 100, "requested": 120 }, "snli": { "correct": 106, "requested": 120 }, "qasc": { "correct": 115, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 597, "accuracy": 0.5707456978967496, "full_probability_metrics": { "n": 1046, "nll": 1.1093572870411794, "acc": 0.5707456978967496, "ece": 0.11253234387750352, "brier": 0.5454244231030179, "mean_conf": 0.67847434299176, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.3508604206500956, "accuracy_at_0_9": 0.8637602179836512, "coverage_at_5pct_error": 0.1615678776290631, "coverage_at_1pct_error": 0.07839388145315487, "aurc": 0.2073272193865173, "error_rate_at_0_9": 0.1362397820163488, "confidence_bias": 0.10772864509501046, "top_bins": { "0.9": { "n": 367, "errors": 50, "error_rate": 0.1362397820163488 }, "0.95": { "n": 291, "errors": 31, "error_rate": 0.10652920962199312 }, "0.99": { "n": 182, "errors": 12, "error_rate": 0.06593406593406594 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7820267686424475, "confidence_cutoff": 0.7346880435943604 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6618876941457587, "confidence_cutoff": 0.34225571665548854 } }, "score_mae": 0.5620276509000492, "ranked_probability_score": 0.18985219074479165 }, "covered_only_probability_metrics": { "n": 1046, "nll": 1.1093572870411794, "acc": 0.5707456978967496, "ece": 0.11253234387750352, "brier": 0.5454244231030179, "mean_conf": 0.67847434299176, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.3508604206500956, "accuracy_at_0_9": 0.8637602179836512, "coverage_at_5pct_error": 0.1615678776290631, "coverage_at_1pct_error": 0.07839388145315487, "aurc": 0.2073272193865173, "error_rate_at_0_9": 0.1362397820163488, "confidence_bias": 0.10772864509501046, "top_bins": { "0.9": { "n": 367, "errors": 50, "error_rate": 0.1362397820163488 }, "0.95": { "n": 291, "errors": 31, "error_rate": 0.10652920962199312 }, "0.99": { "n": 182, "errors": 12, "error_rate": 0.06593406593406594 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7820267686424475, "confidence_cutoff": 0.7346880435943604 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6618876941457587, "confidence_cutoff": 0.34225571665548854 } }, "score_mae": 0.5620276509000492, "ranked_probability_score": 0.18985219074479165 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 4, "accuracy": 0.2, "full_probability_metrics": { "n": 20, "nll": 1.7908087714131653, "acc": 0.2, "ece": 0.028528881168032223, "brier": 0.8313516934182543, "mean_conf": 0.22852888116803224, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8366744776980845, "error_rate_at_0_9": null, "confidence_bias": 0.028528881168032233, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.23856641064834516 }, "0.8": { "coverage": 0.8, "accuracy": 0.1875, "confidence_cutoff": 0.20732088641399823 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.7908087714131653, "acc": 0.2, "ece": 0.028528881168032223, "brier": 0.8313516934182543, "mean_conf": 0.22852888116803224, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8366744776980845, "error_rate_at_0_9": null, "confidence_bias": 0.028528881168032233, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.23856641064834516 }, "0.8": { "coverage": 0.8, "accuracy": 0.1875, "confidence_cutoff": 0.20732088641399823 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 14, "accuracy": 0.7, "full_probability_metrics": { "n": 20, "nll": 0.8029379214431758, "acc": 0.7, "ece": 0.29940200432752617, "brier": 0.4942866453297709, "mean_conf": 0.7994499991746407, "confident_error_rate": 0.15, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.7, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.2710809305933145, "error_rate_at_0_9": 0.3, "confidence_bias": 0.0994499991746407, "top_bins": { "0.9": { "n": 10, "errors": 3, "error_rate": 0.3 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9074558886127438 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6426825736354248 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.8029379214431758, "acc": 0.7, "ece": 0.29940200432752617, "brier": 0.4942866453297709, "mean_conf": 0.7994499991746407, "confident_error_rate": 0.15, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.7, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.2710809305933145, "error_rate_at_0_9": 0.3, "confidence_bias": 0.0994499991746407, "top_bins": { "0.9": { "n": 10, "errors": 3, "error_rate": 0.3 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9074558886127438 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6426825736354248 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 15, "accuracy": 0.75, "full_probability_metrics": { "n": 20, "nll": 0.9929085140388691, "acc": 0.75, "ece": 0.22639238954441143, "brier": 0.46928063274900966, "mean_conf": 0.9208617071387074, "confident_error_rate": 0.2, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.7142857142857143, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.22706332753933373, "error_rate_at_0_9": 0.2857142857142857, "confidence_bias": 0.1708617071387074, "top_bins": { "0.9": { "n": 14, "errors": 4, "error_rate": 0.2857142857142857 }, "0.95": { "n": 11, "errors": 3, "error_rate": 0.2727272727272727 }, "0.99": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9770263437961588 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8613669954915646 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.9929085140388691, "acc": 0.75, "ece": 0.22639238954441143, "brier": 0.46928063274900966, "mean_conf": 0.9208617071387074, "confident_error_rate": 0.2, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.7142857142857143, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.22706332753933373, "error_rate_at_0_9": 0.2857142857142857, "confidence_bias": 0.1708617071387074, "top_bins": { "0.9": { "n": 14, "errors": 4, "error_rate": 0.2857142857142857 }, "0.95": { "n": 11, "errors": 3, "error_rate": 0.2727272727272727 }, "0.99": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9770263437961588 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8613669954915646 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 7, "accuracy": 0.35, "full_probability_metrics": { "n": 20, "nll": 1.0708164346340534, "acc": 0.35, "ece": 0.37229141976580093, "brier": 0.788094537410754, "mean_conf": 0.7222914197658008, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6951221076395224, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3722914197658008, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.7836121916770935 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5308689037224323 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.0708164346340534, "acc": 0.35, "ece": 0.37229141976580093, "brier": 0.788094537410754, "mean_conf": 0.7222914197658008, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6951221076395224, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3722914197658008, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.7836121916770935 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5308689037224323 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 25, "accuracy": 0.78125, "full_probability_metrics": { "n": 32, "nll": 0.4592510513559308, "acc": 0.78125, "ece": 0.11873426027196524, "brier": 0.31564664422642535, "mean_conf": 0.7480866799274459, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.40625, "coverage_at_1pct_error": 0.40625, "aurc": 0.11490055820497895, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03316332007255407, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8125, "confidence_cutoff": 0.6918026002704323 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7692307692307693, "confidence_cutoff": 0.6016458689675966 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.4592510513559308, "acc": 0.78125, "ece": 0.11873426027196524, "brier": 0.31564664422642535, "mean_conf": 0.7480866799274459, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.40625, "coverage_at_1pct_error": 0.40625, "aurc": 0.11490055820497895, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03316332007255407, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8125, "confidence_cutoff": 0.6918026002704323 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7692307692307693, "confidence_cutoff": 0.6016458689675966 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 23, "accuracy": 0.71875, "full_probability_metrics": { "n": 32, "nll": 0.4905876266422741, "acc": 0.71875, "ece": 0.14358990496568, "brier": 0.34488071613522836, "mean_conf": 0.7820237940462751, "confident_error_rate": 0.0, "coverage_at_0_9": 0.28125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3125, "coverage_at_1pct_error": 0.3125, "aurc": 0.11050093611777564, "error_rate_at_0_9": 0.0, "confidence_bias": 0.06327379404627509, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.7355679054389028 }, "0.8": { "coverage": 0.84375, "accuracy": 0.8148148148148148, "confidence_cutoff": 0.6881686654141148 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.4905876266422741, "acc": 0.71875, "ece": 0.14358990496568, "brier": 0.34488071613522836, "mean_conf": 0.7820237940462751, "confident_error_rate": 0.0, "coverage_at_0_9": 0.28125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3125, "coverage_at_1pct_error": 0.3125, "aurc": 0.11050093611777564, "error_rate_at_0_9": 0.0, "confidence_bias": 0.06327379404627509, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.7355679054389028 }, "0.8": { "coverage": 0.84375, "accuracy": 0.8148148148148148, "confidence_cutoff": 0.6881686654141148 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 23, "accuracy": 0.71875, "full_probability_metrics": { "n": 32, "nll": 0.5520341586586174, "acc": 0.71875, "ece": 0.03756021464975888, "brier": 0.3781696546081076, "mean_conf": 0.6811897853502411, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1875, "coverage_at_1pct_error": 0.1875, "aurc": 0.1752653126264595, "error_rate_at_0_9": 0.0, "confidence_bias": -0.037560214649758894, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6485482647582371 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8076923076923077, "confidence_cutoff": 0.5773103833198547 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.5520341586586174, "acc": 0.71875, "ece": 0.03756021464975888, "brier": 0.3781696546081076, "mean_conf": 0.6811897853502411, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1875, "coverage_at_1pct_error": 0.1875, "aurc": 0.1752653126264595, "error_rate_at_0_9": 0.0, "confidence_bias": -0.037560214649758894, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6485482647582371 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8076923076923077, "confidence_cutoff": 0.5773103833198547 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 20, "accuracy": 0.5, "full_probability_metrics": { "n": 40, "nll": 1.1639439615681977, "acc": 0.5, "ece": 0.42154747051796204, "brier": 0.7285618108946876, "mean_conf": 0.921547470517962, "confident_error_rate": 0.225, "coverage_at_0_9": 0.725, "accuracy_at_0_9": 0.6896551724137931, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.475, "aurc": 0.16084830910365294, "error_rate_at_0_9": 0.3103448275862069, "confidence_bias": 0.42154747051796204, "top_bins": { "0.9": { "n": 29, "errors": 9, "error_rate": 0.3103448275862069 }, "0.95": { "n": 23, "errors": 3, "error_rate": 0.13043478260869565 }, "0.99": { "n": 22, "errors": 2, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9948383029261881 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8330378532409668 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 1.1639439615681977, "acc": 0.5, "ece": 0.42154747051796204, "brier": 0.7285618108946876, "mean_conf": 0.921547470517962, "confident_error_rate": 0.225, "coverage_at_0_9": 0.725, "accuracy_at_0_9": 0.6896551724137931, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.475, "aurc": 0.16084830910365294, "error_rate_at_0_9": 0.3103448275862069, "confidence_bias": 0.42154747051796204, "top_bins": { "0.9": { "n": 29, "errors": 9, "error_rate": 0.3103448275862069 }, "0.95": { "n": 23, "errors": 3, "error_rate": 0.13043478260869565 }, "0.99": { "n": 22, "errors": 2, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9948383029261881 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8330378532409668 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 16, "accuracy": 0.4, "full_probability_metrics": { "n": 40, "nll": 1.2285759783018704, "acc": 0.4, "ece": 0.3437115542867167, "brier": 0.7503070890573962, "mean_conf": 0.6328909652012455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.4849932405076257, "error_rate_at_0_9": 0.0, "confidence_bias": 0.23289096520124553, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6242572293128235 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5133316669860979 } }, "score_mae": 0.7228090809003997, "ranked_probability_score": 0.2457146149499231 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.2285759783018704, "acc": 0.4, "ece": 0.3437115542867167, "brier": 0.7503070890573962, "mean_conf": 0.6328909652012455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.4849932405076257, "error_rate_at_0_9": 0.0, "confidence_bias": 0.23289096520124553, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6242572293128235 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5133316669860979 } }, "score_mae": 0.7228090809003997, "ranked_probability_score": 0.2457146149499231 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 18, "accuracy": 0.225, "full_probability_metrics": { "n": 80, "nll": 1.8077631069315436, "acc": 0.225, "ece": 0.019276751630152564, "brier": 0.8388902680619832, "mean_conf": 0.20938043257213207, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8358816544241062, "error_rate_at_0_9": null, "confidence_bias": -0.015619567427867936, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.20345203578472137 }, "0.8": { "coverage": 0.8, "accuracy": 0.21875, "confidence_cutoff": 0.18823534898897937 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.8077631069315436, "acc": 0.225, "ece": 0.019276751630152564, "brier": 0.8388902680619832, "mean_conf": 0.20938043257213207, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8358816544241062, "error_rate_at_0_9": null, "confidence_bias": -0.015619567427867936, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.20345203578472137 }, "0.8": { "coverage": 0.8, "accuracy": 0.21875, "confidence_cutoff": 0.18823534898897937 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 42, "accuracy": 0.525, "full_probability_metrics": { "n": 80, "nll": 1.23471739261249, "acc": 0.525, "ece": 0.18582152379103395, "brier": 0.6192979758743122, "mean_conf": 0.5793723024548392, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.33660001788844596, "error_rate_at_0_9": 0.1, "confidence_bias": 0.05437230245483915, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 2, "errors": 1, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.575, "confidence_cutoff": 0.5392079869030716 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.3947196661639164 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.23471739261249, "acc": 0.525, "ece": 0.18582152379103395, "brier": 0.6192979758743122, "mean_conf": 0.5793723024548392, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.33660001788844596, "error_rate_at_0_9": 0.1, "confidence_bias": 0.05437230245483915, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 2, "errors": 1, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.575, "confidence_cutoff": 0.5392079869030716 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.3947196661639164 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 49, "accuracy": 0.245, "full_probability_metrics": { "n": 200, "nll": 2.2693252699323936, "acc": 0.245, "ece": 0.16497611884882069, "brier": 0.8963486136156271, "mean_conf": 0.4099761188488207, "confident_error_rate": 0.03, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 0.4, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6783353024576317, "error_rate_at_0_9": 0.6, "confidence_bias": 0.16497611884882069, "top_bins": { "0.9": { "n": 10, "errors": 6, "error_rate": 0.6 }, "0.95": { "n": 7, "errors": 5, "error_rate": 0.7142857142857143 }, "0.99": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.32, "confidence_cutoff": 0.34133724051721653 }, "0.8": { "coverage": 0.8, "accuracy": 0.26875, "confidence_cutoff": 0.21106107961638057 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.2693252699323936, "acc": 0.245, "ece": 0.16497611884882069, "brier": 0.8963486136156271, "mean_conf": 0.4099761188488207, "confident_error_rate": 0.03, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 0.4, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6783353024576317, "error_rate_at_0_9": 0.6, "confidence_bias": 0.16497611884882069, "top_bins": { "0.9": { "n": 10, "errors": 6, "error_rate": 0.6 }, "0.95": { "n": 7, "errors": 5, "error_rate": 0.7142857142857143 }, "0.99": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.32, "confidence_cutoff": 0.34133724051721653 }, "0.8": { "coverage": 0.8, "accuracy": 0.26875, "confidence_cutoff": 0.21106107961638057 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 64, "accuracy": 0.8, "full_probability_metrics": { "n": 80, "nll": 0.5661373475757354, "acc": 0.8, "ece": 0.10567826189611622, "brier": 0.30890201038028947, "mean_conf": 0.892224182678748, "confident_error_rate": 0.075, "coverage_at_0_9": 0.6125, "accuracy_at_0_9": 0.8775510204081632, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.0625, "aurc": 0.10755527351815734, "error_rate_at_0_9": 0.12244897959183673, "confidence_bias": 0.09222418267874799, "top_bins": { "0.9": { "n": 49, "errors": 6, "error_rate": 0.12244897959183673 }, "0.95": { "n": 43, "errors": 5, "error_rate": 0.11627906976744186 }, "0.99": { "n": 30, "errors": 2, "error_rate": 0.06666666666666667 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9520265453142998 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.8105479954101444 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.5661373475757354, "acc": 0.8, "ece": 0.10567826189611622, "brier": 0.30890201038028947, "mean_conf": 0.892224182678748, "confident_error_rate": 0.075, "coverage_at_0_9": 0.6125, "accuracy_at_0_9": 0.8775510204081632, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.0625, "aurc": 0.10755527351815734, "error_rate_at_0_9": 0.12244897959183673, "confidence_bias": 0.09222418267874799, "top_bins": { "0.9": { "n": 49, "errors": 6, "error_rate": 0.12244897959183673 }, "0.95": { "n": 43, "errors": 5, "error_rate": 0.11627906976744186 }, "0.99": { "n": 30, "errors": 2, "error_rate": 0.06666666666666667 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9520265453142998 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.8105479954101444 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 67, "accuracy": 0.8375, "full_probability_metrics": { "n": 80, "nll": 0.4076515968956551, "acc": 0.8375, "ece": 0.0756398255350947, "brier": 0.22752732490574462, "mean_conf": 0.8883877349277547, "confident_error_rate": 0.05, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.9259259259259259, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1875, "aurc": 0.07403760414455118, "error_rate_at_0_9": 0.07407407407407407, "confidence_bias": 0.05088773492775467, "top_bins": { "0.9": { "n": 54, "errors": 4, "error_rate": 0.07407407407407407 }, "0.95": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.99": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9523864111731861 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.7659589305836441 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4076515968956551, "acc": 0.8375, "ece": 0.0756398255350947, "brier": 0.22752732490574462, "mean_conf": 0.8883877349277547, "confident_error_rate": 0.05, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.9259259259259259, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1875, "aurc": 0.07403760414455118, "error_rate_at_0_9": 0.07407407407407407, "confidence_bias": 0.05088773492775467, "top_bins": { "0.9": { "n": 54, "errors": 4, "error_rate": 0.07407407407407407 }, "0.95": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.99": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9523864111731861 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.7659589305836441 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 78, "accuracy": 0.975, "full_probability_metrics": { "n": 80, "nll": 0.062454219003902765, "acc": 0.975, "ece": 0.02641125791610669, "brier": 0.03417738364052335, "mean_conf": 0.9707201592541216, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0007933219206953384, "error_rate_at_0_9": 0.0, "confidence_bias": -0.004279840745878394, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 58, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9986122076749188 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9717978086208217 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.062454219003902765, "acc": 0.975, "ece": 0.02641125791610669, "brier": 0.03417738364052335, "mean_conf": 0.9707201592541216, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0007933219206953384, "error_rate_at_0_9": 0.0, "confidence_bias": -0.004279840745878394, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 58, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9986122076749188 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9717978086208217 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 60, "accuracy": 0.75, "full_probability_metrics": { "n": 80, "nll": 0.5511454012787074, "acc": 0.75, "ece": 0.11477706250282642, "brier": 0.3647436647514901, "mean_conf": 0.83706629073572, "confident_error_rate": 0.05, "coverage_at_0_9": 0.4625, "accuracy_at_0_9": 0.8918918918918919, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1, "aurc": 0.1309566953433553, "error_rate_at_0_9": 0.10810810810810811, "confidence_bias": 0.08706629073572003, "top_bins": { "0.9": { "n": 37, "errors": 4, "error_rate": 0.10810810810810811 }, "0.95": { "n": 23, "errors": 1, "error_rate": 0.043478260869565216 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.892565773538265 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7286303260514237 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.5511454012787074, "acc": 0.75, "ece": 0.11477706250282642, "brier": 0.3647436647514901, "mean_conf": 0.83706629073572, "confident_error_rate": 0.05, "coverage_at_0_9": 0.4625, "accuracy_at_0_9": 0.8918918918918919, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1, "aurc": 0.1309566953433553, "error_rate_at_0_9": 0.10810810810810811, "confidence_bias": 0.08706629073572003, "top_bins": { "0.9": { "n": 37, "errors": 4, "error_rate": 0.10810810810810811 }, "0.95": { "n": 23, "errors": 1, "error_rate": 0.043478260869565216 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.892565773538265 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7286303260514237 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0005845648811941865, "acc": 1.0, "ece": 0.0005842383088866532, "brier": 1.304027872644646e-06, "mean_conf": 0.9994157616911133, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0005842383088866532, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998575963908811 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9990510293331177 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0005845648811941865, "acc": 1.0, "ece": 0.0005842383088866532, "brier": 1.304027872644646e-06, "mean_conf": 0.9994157616911133, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0005842383088866532, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998575963908811 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9990510293331177 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.6911943948170873, "acc": 0.5, "ece": 0.4553804476966105, "brier": 0.8471454278292263, "mean_conf": 0.9553804476966106, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.45538044769661057, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 8, "errors": 3, "error_rate": 0.375 }, "0.99": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9954959066051886 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.9772900433958512 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.6911943948170873, "acc": 0.5, "ece": 0.4553804476966105, "brier": 0.8471454278292263, "mean_conf": 0.9553804476966106, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.45538044769661057, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 8, "errors": 3, "error_rate": 0.375 }, "0.99": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9954959066051886 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.9772900433958512 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.1693817003431033, "acc": 0.3, "ece": 0.3398580581986238, "brier": 0.7399961597163618, "mean_conf": 0.6094493989992678, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.49964285714285717, "error_rate_at_0_9": null, "confidence_bias": 0.30944939899926777, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6593740778974914 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4723504011944295 } }, "score_mae": 0.7350904137985406, "ranked_probability_score": 0.24737668377996697 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.1693817003431033, "acc": 0.3, "ece": 0.3398580581986238, "brier": 0.7399961597163618, "mean_conf": 0.6094493989992678, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.49964285714285717, "error_rate_at_0_9": null, "confidence_bias": 0.30944939899926777, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6593740778974914 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4723504011944295 } }, "score_mae": 0.7350904137985406, "ranked_probability_score": 0.24737668377996697 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.5439971492431627, "acc": 0.8, "ece": 0.25153839838104586, "brier": 0.3527875110545306, "mean_conf": 0.8464554809497382, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.10567460317460317, "error_rate_at_0_9": 0.2, "confidence_bias": 0.04645548094973817, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9056080745663384 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8324020504951477 } }, "score_mae": 0.27575610821343904, "ranked_probability_score": 0.08679179616392349 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5439971492431627, "acc": 0.8, "ece": 0.25153839838104586, "brier": 0.3527875110545306, "mean_conf": 0.8464554809497382, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.10567460317460317, "error_rate_at_0_9": 0.2, "confidence_bias": 0.04645548094973817, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9056080745663384 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8324020504951477 } }, "score_mae": 0.27575610821343904, "ranked_probability_score": 0.08679179616392349 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.4366799884996023, "acc": 0.7, "ece": 0.15686187971376453, "brier": 0.28055433866227236, "mean_conf": 0.7592900199679372, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.09567460317460316, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05929001996793726, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9616237928056446 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5392077799672632 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.4366799884996023, "acc": 0.7, "ece": 0.15686187971376453, "brier": 0.28055433866227236, "mean_conf": 0.7592900199679372, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.09567460317460316, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05929001996793726, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9616237928056446 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5392077799672632 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 2.1505827963903323, "acc": 0.4, "ece": 0.5512749767384495, "brier": 1.0617122577412619, "mean_conf": 0.9512749767384495, "confident_error_rate": 0.4, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.566031746031746, "error_rate_at_0_9": 0.5, "confidence_bias": 0.5512749767384495, "top_bins": { "0.9": { "n": 8, "errors": 4, "error_rate": 0.5 }, "0.95": { "n": 6, "errors": 3, "error_rate": 0.5 }, "0.99": { "n": 4, "errors": 2, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9858661164684005 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9132759758855711 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 2.1505827963903323, "acc": 0.4, "ece": 0.5512749767384495, "brier": 1.0617122577412619, "mean_conf": 0.9512749767384495, "confident_error_rate": 0.4, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.566031746031746, "error_rate_at_0_9": 0.5, "confidence_bias": 0.5512749767384495, "top_bins": { "0.9": { "n": 8, "errors": 4, "error_rate": 0.5 }, "0.95": { "n": 6, "errors": 3, "error_rate": 0.5 }, "0.99": { "n": 4, "errors": 2, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9858661164684005 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9132759758855711 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.7886911033169917, "acc": 0.8, "ece": 0.2896506054668469, "brier": 0.3994661328488577, "mean_conf": 0.6445000222968975, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.0815079365079365, "error_rate_at_0_9": 0.0, "confidence_bias": -0.15549997770310253, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.648382513117147 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.552017942593003 } }, "score_mae": 0.4868300532873704, "ranked_probability_score": 0.13092902422713157 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7886911033169917, "acc": 0.8, "ece": 0.2896506054668469, "brier": 0.3994661328488577, "mean_conf": 0.6445000222968975, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.0815079365079365, "error_rate_at_0_9": 0.0, "confidence_bias": -0.15549997770310253, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.648382513117147 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.552017942593003 } }, "score_mae": 0.4868300532873704, "ranked_probability_score": 0.13092902422713157 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.3767287899849792, "acc": 0.4, "ece": 0.4694014705279312, "brier": 0.8709872364995602, "mean_conf": 0.8694014705279312, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.35674603174603176, "error_rate_at_0_9": 0.4, "confidence_bias": 0.46940147052793113, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.923675406898987 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7918749261210868 } }, "score_mae": 0.5388245841471331, "ranked_probability_score": 0.2184530701194963 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3767287899849792, "acc": 0.4, "ece": 0.4694014705279312, "brier": 0.8709872364995602, "mean_conf": 0.8694014705279312, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.35674603174603176, "error_rate_at_0_9": 0.4, "confidence_bias": 0.46940147052793113, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.923675406898987 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7918749261210868 } }, "score_mae": 0.5388245841471331, "ranked_probability_score": 0.2184530701194963 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.2773724243294436, "acc": 0.9, "ece": 0.0605966355742682, "brier": 0.1662618860174146, "mean_conf": 0.8859863344431146, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.03361111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": -0.014013665556885435, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9952971577913517 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7437572104858521 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.2773724243294436, "acc": 0.9, "ece": 0.0605966355742682, "brier": 0.1662618860174146, "mean_conf": 0.8859863344431146, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.03361111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": -0.014013665556885435, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9952971577913517 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7437572104858521 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.16566479833477588, "acc": 0.9, "ece": 0.12898559554996386, "brier": 0.09361033752433574, "mean_conf": 0.8801334624102457, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01986653758975432, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9932803743463842 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7429449440631419 } }, "score_mae": 0.1350966897771858, "ranked_probability_score": 0.02416145776928678 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.16566479833477588, "acc": 0.9, "ece": 0.12898559554996386, "brier": 0.09361033752433574, "mean_conf": 0.8801334624102457, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01986653758975432, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9932803743463842 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7429449440631419 } }, "score_mae": 0.1350966897771858, "ranked_probability_score": 0.02416145776928678 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.3214204814939223, "acc": 0.5, "ece": 0.3065079840881984, "brier": 0.7582577058995394, "mean_conf": 0.7142191160459859, "confident_error_rate": 0.1, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5693650793650793, "error_rate_at_0_9": 1.0, "confidence_bias": 0.21421911604598587, "top_bins": { "0.9": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7826788367131439 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5244126915931702 } }, "score_mae": 0.5574423361752243, "ranked_probability_score": 0.20795141558841884 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3214204814939223, "acc": 0.5, "ece": 0.3065079840881984, "brier": 0.7582577058995394, "mean_conf": 0.7142191160459859, "confident_error_rate": 0.1, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5693650793650793, "error_rate_at_0_9": 1.0, "confidence_bias": 0.21421911604598587, "top_bins": { "0.9": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7826788367131439 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5244126915931702 } }, "score_mae": 0.5574423361752243, "ranked_probability_score": 0.20795141558841884 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.6176388888888888, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.4166666666666667, "flip_rate": 0.3888888888888889, "covered_only_flip_rate": 0.3888888888888889, "half_l1": 0.06940253904197548 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.7719315952361762, "share_at_0_9": 0.24545454545454545, "normalized_entropy": 0.5423840449379791 }, "covered_only_confidence_metrics": { "mean_max_p": 0.7719315952361762, "share_at_0_9": 0.24545454545454545, "normalized_entropy": 0.5423840449379791 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 72, "accuracy": 0.6545454545454545, "full_probability_metrics": { "n": 110, "nll": 0.9020271083304178, "acc": 0.6545454545454545, "ece": 0.17413695379702643, "brier": 0.506434572529203, "mean_conf": 0.8286824083424811, "confident_error_rate": 0.10909090909090909, "coverage_at_0_9": 0.509090909090909, "accuracy_at_0_9": 0.7857142857142857, "coverage_at_5pct_error": 0.2636363636363636, "coverage_at_1pct_error": 0.19090909090909092, "aurc": 0.17842599087654085, "error_rate_at_0_9": 0.21428571428571427, "confidence_bias": 0.17413695379702654, "top_bins": { "0.9": { "n": 56, "errors": 12, "error_rate": 0.21428571428571427 }, "0.95": { "n": 47, "errors": 7, "error_rate": 0.14893617021276595 }, "0.99": { "n": 31, "errors": 3, "error_rate": 0.0967741935483871 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9132759758855711 }, "0.8": { "coverage": 0.8, "accuracy": 0.7045454545454546, "confidence_cutoff": 0.6514657396058725 } }, "score_mae": 0.4548400308998156, "ranked_probability_score": 0.15261057460803734 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.9020271083304178, "acc": 0.6545454545454545, "ece": 0.17413695379702643, "brier": 0.506434572529203, "mean_conf": 0.8286824083424811, "confident_error_rate": 0.10909090909090909, "coverage_at_0_9": 0.509090909090909, "accuracy_at_0_9": 0.7857142857142857, "coverage_at_5pct_error": 0.2636363636363636, "coverage_at_1pct_error": 0.19090909090909092, "aurc": 0.17842599087654085, "error_rate_at_0_9": 0.21428571428571427, "confidence_bias": 0.17413695379702654, "top_bins": { "0.9": { "n": 56, "errors": 12, "error_rate": 0.21428571428571427 }, "0.95": { "n": 47, "errors": 7, "error_rate": 0.14893617021276595 }, "0.99": { "n": 31, "errors": 3, "error_rate": 0.0967741935483871 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9132759758855711 }, "0.8": { "coverage": 0.8, "accuracy": 0.7045454545454546, "confidence_cutoff": 0.6514657396058725 } }, "score_mae": 0.4548400308998156, "ranked_probability_score": 0.15261057460803734 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.05675081310630485 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.9397539608859695, "paired_flip": { "pairs": 64, "flip_rate": 0.28125, "both_correct_rate": 0.1875, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.7916666666666666 }, "unknowable": { "n": 110, "mean_max_p": 0.7719315952361762, "share_at_0_9": 0.24545454545454545, "control_mean_max_p": 0.8286824083424811, "control_share_at_0_9": 0.509090909090909, "control_acc": 0.6545454545454545, "paired_confidence_drop": 0.05675081310630485, "share_less_confident_than_control": 0.7090909090909091 }, "clean": { "n": 1046, "nll": 1.1093572870411794, "acc": 0.5707456978967496, "ece": 0.11253234387750352, "brier": 0.5454244231030179, "mean_conf": 0.67847434299176, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.3508604206500956, "accuracy_at_0_9": 0.8637602179836512, "coverage_at_5pct_error": 0.1615678776290631, "coverage_at_1pct_error": 0.07839388145315487, "aurc": 0.2073272193865173, "error_rate_at_0_9": 0.1362397820163488, "confidence_bias": 0.10772864509501046, "top_bins": { "0.9": { "n": 367, "errors": 50, "error_rate": 0.1362397820163488 }, "0.95": { "n": 291, "errors": 31, "error_rate": 0.10652920962199312 }, "0.99": { "n": 182, "errors": 12, "error_rate": 0.06593406593406594 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7820267686424475, "confidence_cutoff": 0.7346880435943604 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6618876941457587, "confidence_cutoff": 0.34225571665548854 } }, "score_mae": 0.5620276509000492, "ranked_probability_score": 0.18985219074479165 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.7908087714131653, "acc": 0.2, "ece": 0.028528881168032223, "brier": 0.8313516934182543, "mean_conf": 0.22852888116803224, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8366744776980845, "error_rate_at_0_9": null, "confidence_bias": 0.028528881168032233, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.23856641064834516 }, "0.8": { "coverage": 0.8, "accuracy": 0.1875, "confidence_cutoff": 0.20732088641399823 } } }, "buried_paws": { "n": 20, "nll": 0.8029379214431758, "acc": 0.7, "ece": 0.29940200432752617, "brier": 0.4942866453297709, "mean_conf": 0.7994499991746407, "confident_error_rate": 0.15, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.7, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.2710809305933145, "error_rate_at_0_9": 0.3, "confidence_bias": 0.0994499991746407, "top_bins": { "0.9": { "n": 10, "errors": 3, "error_rate": 0.3 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9074558886127438 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6426825736354248 } } }, "buried_qnli": { "n": 20, "nll": 0.9929085140388691, "acc": 0.75, "ece": 0.22639238954441143, "brier": 0.46928063274900966, "mean_conf": 0.9208617071387074, "confident_error_rate": 0.2, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.7142857142857143, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.22706332753933373, "error_rate_at_0_9": 0.2857142857142857, "confidence_bias": 0.1708617071387074, "top_bins": { "0.9": { "n": 14, "errors": 4, "error_rate": 0.2857142857142857 }, "0.95": { "n": 11, "errors": 3, "error_rate": 0.2727272727272727 }, "0.99": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9770263437961588 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.8613669954915646 } } }, "buried_tweet_offensive": { "n": 20, "nll": 1.0708164346340534, "acc": 0.35, "ece": 0.37229141976580093, "brier": 0.788094537410754, "mean_conf": 0.7222914197658008, "confident_error_rate": 0.05, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6951221076395224, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3722914197658008, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.7836121916770935 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5308689037224323 } } }, "composition_final_combination": { "n": 32, "nll": 0.4592510513559308, "acc": 0.78125, "ece": 0.11873426027196524, "brier": 0.31564664422642535, "mean_conf": 0.7480866799274459, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.40625, "coverage_at_1pct_error": 0.40625, "aurc": 0.11490055820497895, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03316332007255407, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8125, "confidence_cutoff": 0.6918026002704323 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7692307692307693, "confidence_cutoff": 0.6016458689675966 } } }, "composition_final_exception": { "n": 32, "nll": 0.4905876266422741, "acc": 0.71875, "ece": 0.14358990496568, "brier": 0.34488071613522836, "mean_conf": 0.7820237940462751, "confident_error_rate": 0.0, "coverage_at_0_9": 0.28125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3125, "coverage_at_1pct_error": 0.3125, "aurc": 0.11050093611777564, "error_rate_at_0_9": 0.0, "confidence_bias": 0.06327379404627509, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.7355679054389028 }, "0.8": { "coverage": 0.84375, "accuracy": 0.8148148148148148, "confidence_cutoff": 0.6881686654141148 } } }, "composition_final_negation": { "n": 32, "nll": 0.5520341586586174, "acc": 0.71875, "ece": 0.03756021464975888, "brier": 0.3781696546081076, "mean_conf": 0.6811897853502411, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1875, "coverage_at_1pct_error": 0.1875, "aurc": 0.1752653126264595, "error_rate_at_0_9": 0.0, "confidence_bias": -0.037560214649758894, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8235294117647058, "confidence_cutoff": 0.6485482647582371 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8076923076923077, "confidence_cutoff": 0.5773103833198547 } } }, "contrastive_authorization": { "n": 40, "nll": 1.1639439615681977, "acc": 0.5, "ece": 0.42154747051796204, "brier": 0.7285618108946876, "mean_conf": 0.921547470517962, "confident_error_rate": 0.225, "coverage_at_0_9": 0.725, "accuracy_at_0_9": 0.6896551724137931, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.475, "aurc": 0.16084830910365294, "error_rate_at_0_9": 0.3103448275862069, "confidence_bias": 0.42154747051796204, "top_bins": { "0.9": { "n": 29, "errors": 9, "error_rate": 0.3103448275862069 }, "0.95": { "n": 23, "errors": 3, "error_rate": 0.13043478260869565 }, "0.99": { "n": 22, "errors": 2, "error_rate": 0.09090909090909091 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9948383029261881 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8330378532409668 } } }, "contrastive_deadline": { "n": 40, "nll": 1.2285759783018704, "acc": 0.4, "ece": 0.3437115542867167, "brier": 0.7503070890573962, "mean_conf": 0.6328909652012455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.4849932405076257, "error_rate_at_0_9": 0.0, "confidence_bias": 0.23289096520124553, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6242572293128235 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5133316669860979 } }, "score_mae": 0.7228090809003997, "ranked_probability_score": 0.2457146149499231 }, "emotion": { "n": 80, "nll": 1.8077631069315436, "acc": 0.225, "ece": 0.019276751630152564, "brier": 0.8388902680619832, "mean_conf": 0.20938043257213207, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8358816544241062, "error_rate_at_0_9": null, "confidence_bias": -0.015619567427867936, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.20345203578472137 }, "0.8": { "coverage": 0.8, "accuracy": 0.21875, "confidence_cutoff": 0.18823534898897937 } } }, "mmlu": { "n": 80, "nll": 1.23471739261249, "acc": 0.525, "ece": 0.18582152379103395, "brier": 0.6192979758743122, "mean_conf": 0.5793723024548392, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.33660001788844596, "error_rate_at_0_9": 0.1, "confidence_bias": 0.05437230245483915, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.99": { "n": 2, "errors": 1, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.575, "confidence_cutoff": 0.5392079869030716 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.3947196661639164 } } }, "mmlu_pro": { "n": 200, "nll": 2.2693252699323936, "acc": 0.245, "ece": 0.16497611884882069, "brier": 0.8963486136156271, "mean_conf": 0.4099761188488207, "confident_error_rate": 0.03, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 0.4, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6783353024576317, "error_rate_at_0_9": 0.6, "confidence_bias": 0.16497611884882069, "top_bins": { "0.9": { "n": 10, "errors": 6, "error_rate": 0.6 }, "0.95": { "n": 7, "errors": 5, "error_rate": 0.7142857142857143 }, "0.99": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.32, "confidence_cutoff": 0.34133724051721653 }, "0.8": { "coverage": 0.8, "accuracy": 0.26875, "confidence_cutoff": 0.21106107961638057 } } }, "paws": { "n": 80, "nll": 0.5661373475757354, "acc": 0.8, "ece": 0.10567826189611622, "brier": 0.30890201038028947, "mean_conf": 0.892224182678748, "confident_error_rate": 0.075, "coverage_at_0_9": 0.6125, "accuracy_at_0_9": 0.8775510204081632, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.0625, "aurc": 0.10755527351815734, "error_rate_at_0_9": 0.12244897959183673, "confidence_bias": 0.09222418267874799, "top_bins": { "0.9": { "n": 49, "errors": 6, "error_rate": 0.12244897959183673 }, "0.95": { "n": 43, "errors": 5, "error_rate": 0.11627906976744186 }, "0.99": { "n": 30, "errors": 2, "error_rate": 0.06666666666666667 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9520265453142998 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.8105479954101444 } } }, "qnli": { "n": 80, "nll": 0.4076515968956551, "acc": 0.8375, "ece": 0.0756398255350947, "brier": 0.22752732490574462, "mean_conf": 0.8883877349277547, "confident_error_rate": 0.05, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.9259259259259259, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1875, "aurc": 0.07403760414455118, "error_rate_at_0_9": 0.07407407407407407, "confidence_bias": 0.05088773492775467, "top_bins": { "0.9": { "n": 54, "errors": 4, "error_rate": 0.07407407407407407 }, "0.95": { "n": 40, "errors": 4, "error_rate": 0.1 }, "0.99": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9523864111731861 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.7659589305836441 } } }, "sciq": { "n": 80, "nll": 0.062454219003902765, "acc": 0.975, "ece": 0.02641125791610669, "brier": 0.03417738364052335, "mean_conf": 0.9707201592541216, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0007933219206953384, "error_rate_at_0_9": 0.0, "confidence_bias": -0.004279840745878394, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 58, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9986122076749188 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9717978086208217 } } }, "tweet_offensive": { "n": 80, "nll": 0.5511454012787074, "acc": 0.75, "ece": 0.11477706250282642, "brier": 0.3647436647514901, "mean_conf": 0.83706629073572, "confident_error_rate": 0.05, "coverage_at_0_9": 0.4625, "accuracy_at_0_9": 0.8918918918918919, "coverage_at_5pct_error": 0.2875, "coverage_at_1pct_error": 0.1, "aurc": 0.1309566953433553, "error_rate_at_0_9": 0.10810810810810811, "confidence_bias": 0.08706629073572003, "top_bins": { "0.9": { "n": 37, "errors": 4, "error_rate": 0.10810810810810811 }, "0.95": { "n": 23, "errors": 1, "error_rate": 0.043478260869565216 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.892565773538265 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7286303260514237 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 1.150595176850467, "acc": 0.5, "ece": 0.36639941897625383, "brier": 0.7786613200677833, "mean_conf": 0.8663994189762538, "confident_error_rate": 0.2, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.36639941897625383, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8720413110268507 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8464412436949836 } } }, "unknowable_authorization": { "n": 10, "nll": 2.3289831794753795, "acc": 0.5, "ece": 0.48978197200304496, "brier": 0.9781717316757443, "mean_conf": 0.989781972003045, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.3806349206349206, "error_rate_at_0_9": 0.5, "confidence_bias": 0.48978197200304496, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 6, "errors": 3, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9934877356141678 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9826076773197603 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.0005845648811941865, "acc": 1.0, "ece": 0.0005842383088866532, "brier": 1.304027872644646e-06, "mean_conf": 0.9994157616911133, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0005842383088866532, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9998575963908811 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9990510293331177 } } }, "unknowable_control_authorization": { "n": 10, "nll": 1.6911943948170873, "acc": 0.5, "ece": 0.4553804476966105, "brier": 0.8471454278292263, "mean_conf": 0.9553804476966106, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.45538044769661057, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 8, "errors": 3, "error_rate": 0.375 }, "0.99": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9954959066051886 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.9772900433958512 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.1693817003431033, "acc": 0.3, "ece": 0.3398580581986238, "brier": 0.7399961597163618, "mean_conf": 0.6094493989992678, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.49964285714285717, "error_rate_at_0_9": null, "confidence_bias": 0.30944939899926777, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6593740778974914 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4723504011944295 } }, "score_mae": 0.7350904137985406, "ranked_probability_score": 0.24737668377996697 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.5439971492431627, "acc": 0.8, "ece": 0.25153839838104586, "brier": 0.3527875110545306, "mean_conf": 0.8464554809497382, "confident_error_rate": 0.1, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.10567460317460317, "error_rate_at_0_9": 0.2, "confidence_bias": 0.04645548094973817, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9056080745663384 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8324020504951477 } }, "score_mae": 0.27575610821343904, "ranked_probability_score": 0.08679179616392349 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.4366799884996023, "acc": 0.7, "ece": 0.15686187971376453, "brier": 0.28055433866227236, "mean_conf": 0.7592900199679372, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.09567460317460316, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05929001996793726, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9616237928056446 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5392077799672632 } } }, "unknowable_control_return_window": { "n": 10, "nll": 2.1505827963903323, "acc": 0.4, "ece": 0.5512749767384495, "brier": 1.0617122577412619, "mean_conf": 0.9512749767384495, "confident_error_rate": 0.4, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.566031746031746, "error_rate_at_0_9": 0.5, "confidence_bias": 0.5512749767384495, "top_bins": { "0.9": { "n": 8, "errors": 4, "error_rate": 0.5 }, "0.95": { "n": 6, "errors": 3, "error_rate": 0.5 }, "0.99": { "n": 4, "errors": 2, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.9858661164684005 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9132759758855711 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.7886911033169917, "acc": 0.8, "ece": 0.2896506054668469, "brier": 0.3994661328488577, "mean_conf": 0.6445000222968975, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.0815079365079365, "error_rate_at_0_9": 0.0, "confidence_bias": -0.15549997770310253, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.648382513117147 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.552017942593003 } }, "score_mae": 0.4868300532873704, "ranked_probability_score": 0.13092902422713157 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.3767287899849792, "acc": 0.4, "ece": 0.4694014705279312, "brier": 0.8709872364995602, "mean_conf": 0.8694014705279312, "confident_error_rate": 0.2, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.35674603174603176, "error_rate_at_0_9": 0.4, "confidence_bias": 0.46940147052793113, "top_bins": { "0.9": { "n": 5, "errors": 2, "error_rate": 0.4 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.923675406898987 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7918749261210868 } }, "score_mae": 0.5388245841471331, "ranked_probability_score": 0.2184530701194963 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.2773724243294436, "acc": 0.9, "ece": 0.0605966355742682, "brier": 0.1662618860174146, "mean_conf": 0.8859863344431146, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7, "coverage_at_1pct_error": 0.7, "aurc": 0.03361111111111111, "error_rate_at_0_9": 0.0, "confidence_bias": -0.014013665556885435, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9952971577913517 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7437572104858521 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.16566479833477588, "acc": 0.9, "ece": 0.12898559554996386, "brier": 0.09361033752433574, "mean_conf": 0.8801334624102457, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.01986653758975432, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9932803743463842 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7429449440631419 } }, "score_mae": 0.1350966897771858, "ranked_probability_score": 0.02416145776928678 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 1.3214204814939223, "acc": 0.5, "ece": 0.3065079840881984, "brier": 0.7582577058995394, "mean_conf": 0.7142191160459859, "confident_error_rate": 0.1, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5693650793650793, "error_rate_at_0_9": 1.0, "confidence_bias": 0.21421911604598587, "top_bins": { "0.9": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7826788367131439 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5244126915931702 } }, "score_mae": 0.5574423361752243, "ranked_probability_score": 0.20795141558841884 }, "unknowable_deadline": { "n": 10, "nll": 1.2589820879625544, "acc": 0.2, "ece": 0.36893212959265914, "brier": 0.7762560356612945, "mean_conf": 0.5097437216967753, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7859920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.3097437216967753, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.5106021165847778 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.4759919288760477 } }, "score_mae": 0.8118511356119033, "ranked_probability_score": 0.26426470087142107 }, "unknowable_late_fee": { "n": 10, "nll": 1.9578345610229668, "acc": 0.4, "ece": 0.4384146774720468, "brier": 0.9793263466647053, "mean_conf": 0.8384146774720467, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6283333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.43841467747204665, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.8627116784595436 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.8466766844004171 } }, "score_mae": 0.662707380736345, "ranked_probability_score": 0.268898413080004 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.515203921848674, "acc": 0.5, "ece": 0.31447616124116395, "brier": 0.8038644135384884, "mean_conf": 0.8144761612411638, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.31447616124116384, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8138489925114926 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7556996772654426 } } }, "unknowable_return_window": { "n": 10, "nll": 1.694044636439969, "acc": 0.4, "ece": 0.5148449356458633, "brier": 0.9566284686121673, "mean_conf": 0.9148449356458634, "confident_error_rate": 0.3, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6350793650793649, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.5148449356458634, "top_bins": { "0.9": { "n": 7, "errors": 3, "error_rate": 0.42857142857142855 }, "0.95": { "n": 6, "errors": 3, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.967044117030852 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8380123972892761 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.1194738420203219, "acc": 0.5, "ece": 0.2546849224845308, "brier": 0.6575953797863413, "mean_conf": 0.5462813230269099, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6518650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.046281323026909904, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5414409202766692 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.520635806946567 } }, "score_mae": 0.7322382927912229, "ranked_probability_score": 0.22746992587855241 }, "unknowable_sla_response": { "n": 10, "nll": 1.1960696670275208, "acc": 0.5, "ece": 0.18761947071685223, "brier": 0.6993817812842769, "mean_conf": 0.6876194707168523, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.18761947071685225, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.7513925825040644 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6173521922331466 } }, "score_mae": 0.554925169425867, "ranked_probability_score": 0.20240047584226323 }, "unknowable_spend_threshold": { "n": 10, "nll": 1.0922459601552457, "acc": 0.5, "ece": 0.31737103725452986, "brier": 0.7370823460068499, "mean_conf": 0.81737103725453, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.31737103725453, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8832213187677872 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8020878440095673 } } }, "unknowable_volume_discount": { "n": 10, "nll": 2.3273289940649007, "acc": 0.2, "ece": 0.6154863532731728, "brier": 1.1837011966577526, "mean_conf": 0.815486353273173, "confident_error_rate": 0.2, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6433333333333333, "error_rate_at_0_9": 0.5, "confidence_bias": 0.6154863532731729, "top_bins": { "0.9": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.8768707758804068 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7493708701827717 } }, "score_mae": 0.8027666225204879, "ranked_probability_score": 0.3367818215315178 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.28473718474473, "acc": 0.5, "ece": 0.19082847629132516, "brier": 0.7295898959694858, "mean_conf": 0.6908284762913252, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.543968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.19082847629132516, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6865294396664721 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6286217500050916 } }, "score_mae": 0.6235539458680864, "ranked_probability_score": 0.21525890331388647 } }, "variants": { "clean": { "n": 1156, "nll": 1.1502099605200702, "acc": 0.5570934256055363, "ece": 0.13559979178950876, "brier": 0.5738032315960255, "mean_conf": 0.6873673341222841, "confident_error_rate": 0.054498269896193774, "coverage_at_0_9": 0.3408304498269896, "accuracy_at_0_9": 0.8401015228426396, "coverage_at_5pct_error": 0.14100346020761245, "coverage_at_1pct_error": 0.07093425605536333, "aurc": 0.22989539635433062, "error_rate_at_0_9": 0.1598984771573604, "confidence_bias": 0.1302739085167478, "top_bins": { "0.9": { "n": 394, "errors": 63, "error_rate": 0.1598984771573604 }, "0.95": { "n": 307, "errors": 39, "error_rate": 0.1270358306188925 }, "0.99": { "n": 188, "errors": 15, "error_rate": 0.0797872340425532 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7422145328719724, "confidence_cutoff": 0.7429449440631419 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.6410810810810811, "confidence_cutoff": 0.37698230880604333 } }, "score_mae": 0.6130199409971503, "ranked_probability_score": 0.21334975924784758 }, "none_absent": { "n": 36, "nll": 2.187539254947994, "acc": 0.2222222222222222, "ece": 0.28443496560900716, "brier": 0.9724764837876059, "mean_conf": 0.5066571878312294, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.1111111111111111, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7092819437819445, "error_rate_at_0_9": 0.75, "confidence_bias": 0.28443496560900716, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.5341000105150978 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.27586206896551724, "confidence_cutoff": 0.21830812233001906 } } }, "none_present": { "n": 36, "nll": 1.2924956121691369, "acc": 0.4444444444444444, "ece": 0.1417091708915361, "brier": 0.5914869948105144, "mean_conf": 0.5484486185005655, "confident_error_rate": 0.0, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3333333333333333, "coverage_at_1pct_error": 0.3333333333333333, "aurc": 0.25535215090013996, "error_rate_at_0_9": 0.0, "confidence_bias": 0.10400417405612106, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7222222222222222, "confidence_cutoff": 0.49119879514578146 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5517241379310345, "confidence_cutoff": 0.18630012073740812 } } }, "permuted": { "n": 36, "nll": 1.2623952400668164, "acc": 0.4444444444444444, "ece": 0.1775192428970765, "brier": 0.6046423841974604, "mean_conf": 0.5753934539626718, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3055555555555556, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3333333333333333, "coverage_at_1pct_error": 0.3333333333333333, "aurc": 0.26065563244601503, "error_rate_at_0_9": 0.0, "confidence_bias": 0.1309490095182274, "top_bins": { "0.9": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7222222222222222, "confidence_cutoff": 0.5535825501262817 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5172413793103449, "confidence_cutoff": 0.22238437831401825 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.06558975824185498, "flip_rate": 0.3888888888888889 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 1.1093572870411794, "acc": 0.5707456978967496, "ece": 0.11253234387750352, "brier": 0.5454244231030179, "mean_conf": 0.67847434299176, "confident_error_rate": 0.04780114722753346, "coverage_at_0_9": 0.3508604206500956, "accuracy_at_0_9": 0.8637602179836512, "coverage_at_5pct_error": 0.1615678776290631, "coverage_at_1pct_error": 0.07839388145315487, "aurc": 0.2073272193865173, "error_rate_at_0_9": 0.1362397820163488, "confidence_bias": 0.10772864509501046, "top_bins": { "0.9": { "n": 367, "errors": 50, "error_rate": 0.1362397820163488 }, "0.95": { "n": 291, "errors": 31, "error_rate": 0.10652920962199312 }, "0.99": { "n": 182, "errors": 12, "error_rate": 0.06593406593406594 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7820267686424475, "confidence_cutoff": 0.7346880435943604 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6618876941457587, "confidence_cutoff": 0.34225571665548854 } }, "score_mae": 0.5620276509000492, "ranked_probability_score": 0.18985219074479165 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Eos": { "panels": { "old_core": { "point": 0.6594122023809523, "exact_fraction": "3545/5376", "ci95": [ 0.6209068080357142, 0.6971363467261905 ] }, "v3_core": { "point": 0.46041666666666664, "exact_fraction": "221/480", "ci95": [ 0.4341666666666667, 0.48624999999999996 ] }, "v4": { "point": 0.703125, "exact_fraction": "45/64", "ci95": [ 0.6531231174698795, 0.7522440055031446 ] }, "v5": { "point": 0.8166666666666667, "exact_fraction": "49/60", "ci95": [ 0.7841463067895258, 0.8490080670426065 ] }, "transfer_v9_test": { "point": 0.5200764818355641, "exact_fraction": "272/523", "ci95": [ 0.4893601498468416, 0.5501942047994679 ] }, "original_four_panel_mean": { "point": 0.6599051339285714, "exact_fraction": "23651/35840", "ci95": [ 0.6409758694694809, 0.6784141644437998 ] }, "weighted_mean": { "point": 0.618908049656287, "exact_fraction": "87007579/140582400", "ci95": [ 0.6026054104977019, 0.6352887782710095 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 105, "requested": 128 }, "boolean_constraints": { "correct": 53, "requested": 64 }, "dbpedia_14": { "correct": 103, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 19, "requested": 96 }, "ordinal_rubric": { "correct": 54, "requested": 64 }, "relational_composition": { "correct": 62, "requested": 96 }, "scoped_evidence": { "correct": 24, "requested": 96 }, "state_tracking": { "correct": 37, "requested": 96 }, "unknown_rejection": { "correct": 45, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 48, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 20, "requested": 80 }, "massive_en": { "correct": 110, "requested": 120 }, "massive_zh": { "correct": 102, "requested": 120 }, "multiset_reconciliation": { "correct": 21, "requested": 80 }, "ordinal_service_loss": { "correct": 16, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 22, "requested": 80 }, "temporal_exclusion": { "correct": 26, "requested": 80 }, "transaction_recovery": { "correct": 34, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 121, "requested": 160 }, "belebele_en": { "correct": 109, "requested": 160 }, "belebele_zh": { "correct": 99, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 84, "requested": 120 }, "squad2_answerability": { "correct": 98, "requested": 120 }, "snli": { "correct": 101, "requested": 120 }, "qasc": { "correct": 109, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 544, "accuracy": 0.5200764818355641, "full_probability_metrics": { "n": 1046, "nll": 1.140604453192763, "acc": 0.5200764818355641, "ece": 0.13956812478907013, "brier": 0.5847272961307521, "mean_conf": 0.6596446066246342, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.2676864244741874, "accuracy_at_0_9": 0.8464285714285714, "coverage_at_5pct_error": 0.1022944550669216, "coverage_at_1pct_error": 0.06405353728489484, "aurc": 0.2611055042681389, "error_rate_at_0_9": 0.15357142857142858, "confidence_bias": 0.1395681247890701, "top_bins": { "0.9": { "n": 280, "errors": 43, "error_rate": 0.15357142857142858 }, "0.95": { "n": 172, "errors": 17, "error_rate": 0.09883720930232558 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7093690248565966, "confidence_cutoff": 0.6627623778724683 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6164874551971327, "confidence_cutoff": 0.43874029305741846 } }, "score_mae": 0.6646501516822292, "ranked_probability_score": 0.20382515703147194 }, "covered_only_probability_metrics": { "n": 1046, "nll": 1.140604453192763, "acc": 0.5200764818355641, "ece": 0.13956812478907013, "brier": 0.5847272961307521, "mean_conf": 0.6596446066246342, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.2676864244741874, "accuracy_at_0_9": 0.8464285714285714, "coverage_at_5pct_error": 0.1022944550669216, "coverage_at_1pct_error": 0.06405353728489484, "aurc": 0.2611055042681389, "error_rate_at_0_9": 0.15357142857142858, "confidence_bias": 0.1395681247890701, "top_bins": { "0.9": { "n": 280, "errors": 43, "error_rate": 0.15357142857142858 }, "0.95": { "n": 172, "errors": 17, "error_rate": 0.09883720930232558 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7093690248565966, "confidence_cutoff": 0.6627623778724683 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6164874551971327, "confidence_cutoff": 0.43874029305741846 } }, "score_mae": 0.6646501516822292, "ranked_probability_score": 0.20382515703147194 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 8, "accuracy": 0.4, "full_probability_metrics": { "n": 20, "nll": 1.6922680067235276, "acc": 0.4, "ece": 0.19980236035018145, "brier": 0.7677341836497671, "mean_conf": 0.47849831189284203, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5530819797247816, "error_rate_at_0_9": null, "confidence_bias": 0.07849831189284201, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.4973419665436558 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.3228425991641082 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.6922680067235276, "acc": 0.4, "ece": 0.19980236035018145, "brier": 0.7677341836497671, "mean_conf": 0.47849831189284203, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5530819797247816, "error_rate_at_0_9": null, "confidence_bias": 0.07849831189284201, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.4973419665436558 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.3228425991641082 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 14, "accuracy": 0.7, "full_probability_metrics": { "n": 20, "nll": 0.5982875413119898, "acc": 0.7, "ece": 0.11075831991651361, "brier": 0.4001615387031127, "mean_conf": 0.8107583199165134, "confident_error_rate": 0.05, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.8571428571428571, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.1632702252168196, "error_rate_at_0_9": 0.14285714285714285, "confidence_bias": 0.11075831991651341, "top_bins": { "0.9": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8505409829237911 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7491870969049392 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5982875413119898, "acc": 0.7, "ece": 0.11075831991651361, "brier": 0.4001615387031127, "mean_conf": 0.8107583199165134, "confident_error_rate": 0.05, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.8571428571428571, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.1632702252168196, "error_rate_at_0_9": 0.14285714285714285, "confidence_bias": 0.11075831991651341, "top_bins": { "0.9": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8505409829237911 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7491870969049392 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 17, "accuracy": 0.85, "full_probability_metrics": { "n": 20, "nll": 0.4599992511278768, "acc": 0.85, "ece": 0.1158713711282571, "brier": 0.26410424468040555, "mean_conf": 0.8705393336106015, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.10557834408453606, "error_rate_at_0_9": 0.1, "confidence_bias": 0.020539333610601562, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9002058908435355 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8476532137135575 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.4599992511278768, "acc": 0.85, "ece": 0.1158713711282571, "brier": 0.26410424468040555, "mean_conf": 0.8705393336106015, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.10557834408453606, "error_rate_at_0_9": 0.1, "confidence_bias": 0.020539333610601562, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9002058908435355 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8476532137135575 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 3, "accuracy": 0.15, "full_probability_metrics": { "n": 20, "nll": 1.9649514082589004, "acc": 0.15, "ece": 0.7461943399452262, "brier": 1.3496773276376004, "mean_conf": 0.8961943399452263, "confident_error_rate": 0.45, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7559541307935272, "error_rate_at_0_9": 0.75, "confidence_bias": 0.7461943399452263, "top_bins": { "0.9": { "n": 12, "errors": 9, "error_rate": 0.75 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.9223600304830083 }, "0.8": { "coverage": 0.8, "accuracy": 0.1875, "confidence_cutoff": 0.8789992463439591 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.9649514082589004, "acc": 0.15, "ece": 0.7461943399452262, "brier": 1.3496773276376004, "mean_conf": 0.8961943399452263, "confident_error_rate": 0.45, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7559541307935272, "error_rate_at_0_9": 0.75, "confidence_bias": 0.7461943399452263, "top_bins": { "0.9": { "n": 12, "errors": 9, "error_rate": 0.75 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.9223600304830083 }, "0.8": { "coverage": 0.8, "accuracy": 0.1875, "confidence_cutoff": 0.8789992463439591 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 12, "accuracy": 0.375, "full_probability_metrics": { "n": 32, "nll": 0.7103396907796085, "acc": 0.375, "ece": 0.2518146996729819, "brier": 0.5195104923922356, "mean_conf": 0.5991911717131082, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.42419622085140174, "error_rate_at_0_9": null, "confidence_bias": 0.2241911717131082, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5294117647058824, "confidence_cutoff": 0.5852385584938956 }, "0.8": { "coverage": 0.8125, "accuracy": 0.38461538461538464, "confidence_cutoff": 0.5485769510269165 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.7103396907796085, "acc": 0.375, "ece": 0.2518146996729819, "brier": 0.5195104923922356, "mean_conf": 0.5991911717131082, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.42419622085140174, "error_rate_at_0_9": null, "confidence_bias": 0.2241911717131082, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5294117647058824, "confidence_cutoff": 0.5852385584938956 }, "0.8": { "coverage": 0.8125, "accuracy": 0.38461538461538464, "confidence_cutoff": 0.5485769510269165 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 17, "accuracy": 0.53125, "full_probability_metrics": { "n": 32, "nll": 0.6119872736170107, "acc": 0.53125, "ece": 0.1663285734801442, "brier": 0.4250681326420055, "mean_conf": 0.6087256435466561, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.21875, "coverage_at_1pct_error": 0.21875, "aurc": 0.22280994275960814, "error_rate_at_0_9": null, "confidence_bias": 0.07747564354665615, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.6103642400990653 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6153846153846154, "confidence_cutoff": 0.5236621499061584 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.6119872736170107, "acc": 0.53125, "ece": 0.1663285734801442, "brier": 0.4250681326420055, "mean_conf": 0.6087256435466561, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.21875, "coverage_at_1pct_error": 0.21875, "aurc": 0.22280994275960814, "error_rate_at_0_9": null, "confidence_bias": 0.07747564354665615, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.6103642400990653 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6153846153846154, "confidence_cutoff": 0.5236621499061584 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 22, "accuracy": 0.6875, "full_probability_metrics": { "n": 32, "nll": 0.6214072129897903, "acc": 0.6875, "ece": 0.09700088834713139, "brier": 0.43061617531605, "mean_conf": 0.5904991116528687, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.21277250791235114, "error_rate_at_0_9": null, "confidence_bias": -0.09700088834713128, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.6470588235294118, "confidence_cutoff": 0.5689421892166138 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7692307692307693, "confidence_cutoff": 0.5137548752877041 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.6214072129897903, "acc": 0.6875, "ece": 0.09700088834713139, "brier": 0.43061617531605, "mean_conf": 0.5904991116528687, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.21277250791235114, "error_rate_at_0_9": null, "confidence_bias": -0.09700088834713128, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.6470588235294118, "confidence_cutoff": 0.5689421892166138 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7692307692307693, "confidence_cutoff": 0.5137548752877041 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 20, "accuracy": 0.5, "full_probability_metrics": { "n": 40, "nll": 0.8723859824290997, "acc": 0.5, "ece": 0.4244367108871019, "brier": 0.6501338236771121, "mean_conf": 0.8518532337953731, "confident_error_rate": 0.0, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.425, "coverage_at_1pct_error": 0.425, "aurc": 0.1772797029704015, "error_rate_at_0_9": 0.0, "confidence_bias": 0.35185323379537314, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.8627102760692065 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7737385745903638 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.8723859824290997, "acc": 0.5, "ece": 0.4244367108871019, "brier": 0.6501338236771121, "mean_conf": 0.8518532337953731, "confident_error_rate": 0.0, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.425, "coverage_at_1pct_error": 0.425, "aurc": 0.1772797029704015, "error_rate_at_0_9": 0.0, "confidence_bias": 0.35185323379537314, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.8627102760692065 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7737385745903638 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 20, "accuracy": 0.5, "full_probability_metrics": { "n": 40, "nll": 1.1619693513204439, "acc": 0.5, "ece": 0.08339102781151485, "brier": 0.6603451502681464, "mean_conf": 0.5457050910932919, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.42314987081059563, "error_rate_at_0_9": null, "confidence_bias": 0.045705091093291905, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.55, "confidence_cutoff": 0.5472279598733752 }, "0.8": { "coverage": 0.8, "accuracy": 0.46875, "confidence_cutoff": 0.49062062096801606 } }, "score_mae": 0.7331951468157973, "ranked_probability_score": 0.227748073203181 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.1619693513204439, "acc": 0.5, "ece": 0.08339102781151485, "brier": 0.6603451502681464, "mean_conf": 0.5457050910932919, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.42314987081059563, "error_rate_at_0_9": null, "confidence_bias": 0.045705091093291905, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.55, "confidence_cutoff": 0.5472279598733752 }, "0.8": { "coverage": 0.8, "accuracy": 0.46875, "confidence_cutoff": 0.49062062096801606 } }, "score_mae": 0.7331951468157973, "ranked_probability_score": 0.227748073203181 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 36, "accuracy": 0.45, "full_probability_metrics": { "n": 80, "nll": 1.4384755112737106, "acc": 0.45, "ece": 0.23354828672781014, "brier": 0.6782082601971992, "mean_conf": 0.6835482867278101, "confident_error_rate": 0.05, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 0.8095238095238095, "coverage_at_5pct_error": 0.1125, "coverage_at_1pct_error": 0.1125, "aurc": 0.3075475611515972, "error_rate_at_0_9": 0.19047619047619047, "confidence_bias": 0.23354828672781008, "top_bins": { "0.9": { "n": 21, "errors": 4, "error_rate": 0.19047619047619047 }, "0.95": { "n": 13, "errors": 3, "error_rate": 0.23076923076923078 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.7035340538594081 }, "0.8": { "coverage": 0.8, "accuracy": 0.546875, "confidence_cutoff": 0.4568873397079616 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.4384755112737106, "acc": 0.45, "ece": 0.23354828672781014, "brier": 0.6782082601971992, "mean_conf": 0.6835482867278101, "confident_error_rate": 0.05, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 0.8095238095238095, "coverage_at_5pct_error": 0.1125, "coverage_at_1pct_error": 0.1125, "aurc": 0.3075475611515972, "error_rate_at_0_9": 0.19047619047619047, "confidence_bias": 0.23354828672781008, "top_bins": { "0.9": { "n": 21, "errors": 4, "error_rate": 0.19047619047619047 }, "0.95": { "n": 13, "errors": 3, "error_rate": 0.23076923076923078 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.7035340538594081 }, "0.8": { "coverage": 0.8, "accuracy": 0.546875, "confidence_cutoff": 0.4568873397079616 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 38, "accuracy": 0.475, "full_probability_metrics": { "n": 80, "nll": 1.3090704914241156, "acc": 0.475, "ece": 0.14470752412362756, "brier": 0.6651548126894188, "mean_conf": 0.5401188879032786, "confident_error_rate": 0.025, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3692044013257334, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.06511888790327858, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.4982062829825442 }, "0.8": { "coverage": 0.8, "accuracy": 0.53125, "confidence_cutoff": 0.3532145619392395 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.3090704914241156, "acc": 0.475, "ece": 0.14470752412362756, "brier": 0.6651548126894188, "mean_conf": 0.5401188879032786, "confident_error_rate": 0.025, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3692044013257334, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.06511888790327858, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.4982062829825442 }, "0.8": { "coverage": 0.8, "accuracy": 0.53125, "confidence_cutoff": 0.3532145619392395 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 34, "accuracy": 0.17, "full_probability_metrics": { "n": 200, "nll": 2.3089932657368935, "acc": 0.17, "ece": 0.15466899840526752, "brier": 0.8998449734534026, "mean_conf": 0.324351039449046, "confident_error_rate": 0.0, "coverage_at_0_9": 0.01, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.01, "coverage_at_1pct_error": 0.01, "aurc": 0.7036049462279607, "error_rate_at_0_9": 0.0, "confidence_bias": 0.15435103944904596, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.26, "confidence_cutoff": 0.2645310173352473 }, "0.8": { "coverage": 0.8, "accuracy": 0.2, "confidence_cutoff": 0.15071202306993456 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.3089932657368935, "acc": 0.17, "ece": 0.15466899840526752, "brier": 0.8998449734534026, "mean_conf": 0.324351039449046, "confident_error_rate": 0.0, "coverage_at_0_9": 0.01, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.01, "coverage_at_1pct_error": 0.01, "aurc": 0.7036049462279607, "error_rate_at_0_9": 0.0, "confidence_bias": 0.15435103944904596, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.26, "confidence_cutoff": 0.2645310173352473 }, "0.8": { "coverage": 0.8, "accuracy": 0.2, "confidence_cutoff": 0.15071202306993456 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 57, "accuracy": 0.7125, "full_probability_metrics": { "n": 80, "nll": 0.6836634797207242, "acc": 0.7125, "ece": 0.17934968602751228, "brier": 0.4134595515140756, "mean_conf": 0.8918496860275124, "confident_error_rate": 0.125, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.8214285714285714, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.3375, "aurc": 0.1169251381554202, "error_rate_at_0_9": 0.17857142857142858, "confidence_bias": 0.17934968602751233, "top_bins": { "0.9": { "n": 56, "errors": 10, "error_rate": 0.17857142857142858 }, "0.95": { "n": 43, "errors": 7, "error_rate": 0.16279069767441862 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.9672752130844654 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7933746339865138 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.6836634797207242, "acc": 0.7125, "ece": 0.17934968602751228, "brier": 0.4134595515140756, "mean_conf": 0.8918496860275124, "confident_error_rate": 0.125, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.8214285714285714, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.3375, "aurc": 0.1169251381554202, "error_rate_at_0_9": 0.17857142857142858, "confidence_bias": 0.17934968602751233, "top_bins": { "0.9": { "n": 56, "errors": 10, "error_rate": 0.17857142857142858 }, "0.95": { "n": 43, "errors": 7, "error_rate": 0.16279069767441862 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.9672752130844654 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7933746339865138 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 65, "accuracy": 0.8125, "full_probability_metrics": { "n": 80, "nll": 0.4234729256173562, "acc": 0.8125, "ece": 0.08154498996045036, "brier": 0.26487978917079796, "mean_conf": 0.875528983544843, "confident_error_rate": 0.05, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.92, "coverage_at_5pct_error": 0.55, "coverage_at_1pct_error": 0.025, "aurc": 0.09236898219002079, "error_rate_at_0_9": 0.08, "confidence_bias": 0.06302898354484299, "top_bins": { "0.9": { "n": 50, "errors": 4, "error_rate": 0.08 }, "0.95": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9245217259759483 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7813270571630142 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4234729256173562, "acc": 0.8125, "ece": 0.08154498996045036, "brier": 0.26487978917079796, "mean_conf": 0.875528983544843, "confident_error_rate": 0.05, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.92, "coverage_at_5pct_error": 0.55, "coverage_at_1pct_error": 0.025, "aurc": 0.09236898219002079, "error_rate_at_0_9": 0.08, "confidence_bias": 0.06302898354484299, "top_bins": { "0.9": { "n": 50, "errors": 4, "error_rate": 0.08 }, "0.95": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9245217259759483 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7813270571630142 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 78, "accuracy": 0.975, "full_probability_metrics": { "n": 80, "nll": 0.12773807478479932, "acc": 0.975, "ece": 0.042135866947253076, "brier": 0.05761418509406378, "mean_conf": 0.9330920038477387, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.8125, "aurc": 0.003545759269761683, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04190799615226126, "top_bins": { "0.9": { "n": 61, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 30, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9766970841042327 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8963993749335851 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.12773807478479932, "acc": 0.975, "ece": 0.042135866947253076, "brier": 0.05761418509406378, "mean_conf": 0.9330920038477387, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.8125, "aurc": 0.003545759269761683, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04190799615226126, "top_bins": { "0.9": { "n": 61, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 30, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9766970841042327 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8963993749335851 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 42, "accuracy": 0.525, "full_probability_metrics": { "n": 80, "nll": 1.076663463803308, "acc": 0.525, "ece": 0.33069550075925347, "brier": 0.7108048153709255, "mean_conf": 0.8147667972496968, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.48164792381461224, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.2897667972496968, "top_bins": { "0.9": { "n": 28, "errors": 12, "error_rate": 0.42857142857142855 }, "0.95": { "n": 11, "errors": 5, "error_rate": 0.45454545454545453 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.525, "confidence_cutoff": 0.8547389634659732 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6671240131332882 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.076663463803308, "acc": 0.525, "ece": 0.33069550075925347, "brier": 0.7108048153709255, "mean_conf": 0.8147667972496968, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.48164792381461224, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.2897667972496968, "top_bins": { "0.9": { "n": 28, "errors": 12, "error_rate": 0.42857142857142855 }, "0.95": { "n": 11, "errors": 5, "error_rate": 0.45454545454545453 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.525, "confidence_cutoff": 0.8547389634659732 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6671240131332882 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.0804363010151486, "acc": 1.0, "ece": 0.07499668897844983, "brier": 0.019488772524919224, "mean_conf": 0.9250033110215503, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07499668897844969, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9773614188128392 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8755277134320821 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.0804363010151486, "acc": 1.0, "ece": 0.07499668897844983, "brier": 0.019488772524919224, "mean_conf": 0.9250033110215503, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07499668897844969, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9773614188128392 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8755277134320821 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.8993057373066659, "acc": 0.5, "ece": 0.37620665693587, "brier": 0.6694469865764338, "mean_conf": 0.8433104243341202, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.22218253968253965, "error_rate_at_0_9": 0.0, "confidence_bias": 0.3433104243341202, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8683824331335321 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8090215211523486 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.8993057373066659, "acc": 0.5, "ece": 0.37620665693587, "brier": 0.6694469865764338, "mean_conf": 0.8433104243341202, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.22218253968253965, "error_rate_at_0_9": 0.0, "confidence_bias": 0.3433104243341202, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8683824331335321 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8090215211523486 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.078318425147064, "acc": 0.4, "ece": 0.21827502538583887, "brier": 0.6392985173942919, "mean_conf": 0.5448525953261908, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4484126984126983, "error_rate_at_0_9": null, "confidence_bias": 0.14485259532619077, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5694281501280378 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5023539141171264 } }, "score_mae": 0.6744406501165433, "ranked_probability_score": 0.20361680207298805 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.078318425147064, "acc": 0.4, "ece": 0.21827502538583887, "brier": 0.6392985173942919, "mean_conf": 0.5448525953261908, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4484126984126983, "error_rate_at_0_9": null, "confidence_bias": 0.14485259532619077, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5694281501280378 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5023539141171264 } }, "score_mae": 0.6744406501165433, "ranked_probability_score": 0.20361680207298805 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 0.8257367555082981, "acc": 0.6, "ece": 0.20315558166625602, "brier": 0.5368300664727356, "mean_conf": 0.6927817124371142, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.446468253968254, "error_rate_at_0_9": null, "confidence_bias": 0.0927817124371142, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7525878724127334 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6318264509465329 } }, "score_mae": 0.4942852151824952, "ranked_probability_score": 0.13674801772209272 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.8257367555082981, "acc": 0.6, "ece": 0.20315558166625602, "brier": 0.5368300664727356, "mean_conf": 0.6927817124371142, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.446468253968254, "error_rate_at_0_9": null, "confidence_bias": 0.0927817124371142, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7525878724127334 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6318264509465329 } }, "score_mae": 0.4942852151824952, "ranked_probability_score": 0.13674801772209272 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.7411287197432239, "acc": 0.5, "ece": 0.33488537025880927, "brier": 0.41978723877464386, "mean_conf": 0.5404037240333779, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": null, "confidence_bias": 0.040403724033377886, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.6493193153038366 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.3707034179358656 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7411287197432239, "acc": 0.5, "ece": 0.33488537025880927, "brier": 0.41978723877464386, "mean_conf": 0.5404037240333779, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": null, "confidence_bias": 0.040403724033377886, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.6493193153038366 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.3707034179358656 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.7304189222342764, "acc": 0.5, "ece": 0.12032863124216035, "brier": 0.5349363083274995, "mean_conf": 0.6203286312421604, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.543968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.12032863124216042, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6419200301170349 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5817150300230077 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7304189222342764, "acc": 0.5, "ece": 0.12032863124216035, "brier": 0.5349363083274995, "mean_conf": 0.6203286312421604, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.543968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.12032863124216042, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6419200301170349 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5817150300230077 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 2, "accuracy": 0.2, "full_probability_metrics": { "n": 10, "nll": 1.3919532483735586, "acc": 0.2, "ece": 0.45759628518374096, "brier": 0.8839505147660234, "mean_conf": 0.5808141287879629, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6859920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.3808141287879629, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6313703938441154 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.4809979283363553 } }, "score_mae": 0.9450878389130155, "ranked_probability_score": 0.32573415308930975 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3919532483735586, "acc": 0.2, "ece": 0.45759628518374096, "brier": 0.8839505147660234, "mean_conf": 0.5808141287879629, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6859920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.3808141287879629, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6313703938441154 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.4809979283363553 } }, "score_mae": 0.9450878389130155, "ranked_probability_score": 0.32573415308930975 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.0889432071009875, "acc": 0.3, "ece": 0.3326331805458419, "brier": 0.7081347505541592, "mean_conf": 0.6326331805458418, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5546428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.33263318054584184, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6303047819571844 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5840732368219906 } }, "score_mae": 0.6461730700376959, "ranked_probability_score": 0.20695672051012787 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0889432071009875, "acc": 0.3, "ece": 0.3326331805458419, "brier": 0.7081347505541592, "mean_conf": 0.6326331805458418, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5546428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.33263318054584184, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6303047819571844 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5840732368219906 } }, "score_mae": 0.6461730700376959, "ranked_probability_score": 0.20695672051012787 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.5415958593526109, "acc": 0.7, "ece": 0.2550459061934608, "brier": 0.3559473859760248, "mean_conf": 0.7142320425289375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.12246031746031745, "error_rate_at_0_9": 0.0, "confidence_bias": 0.014232042528937505, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7415914077608177 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5424330133491566 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5415958593526109, "acc": 0.7, "ece": 0.2550459061934608, "brier": 0.3559473859760248, "mean_conf": 0.7142320425289375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.12246031746031745, "error_rate_at_0_9": 0.0, "confidence_bias": 0.014232042528937505, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7415914077608177 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5424330133491566 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.5665938867164622, "acc": 0.9, "ece": 0.30137184591871985, "brier": 0.3096533035516891, "mean_conf": 0.5986281540812801, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.3013718459187199, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.5679470258359087 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5116761999081999 } }, "score_mae": 0.4469346863442409, "ranked_probability_score": 0.09709592599237986 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5665938867164622, "acc": 0.9, "ece": 0.30137184591871985, "brier": 0.3096533035516891, "mean_conf": 0.5986281540812801, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.3013718459187199, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.5679470258359087 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5116761999081999 } }, "score_mae": 0.4469346863442409, "ranked_probability_score": 0.09709592599237986 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.9218747362554651, "acc": 0.5, "ece": 0.04956487919521275, "brier": 0.5612367027546188, "mean_conf": 0.4504351208047873, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.356468253968254, "error_rate_at_0_9": null, "confidence_bias": -0.049564879195212685, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.44516182609143595 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.42480437681661914 } }, "score_mae": 0.506799468965112, "ranked_probability_score": 0.1571076581150969 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.9218747362554651, "acc": 0.5, "ece": 0.04956487919521275, "brier": 0.5612367027546188, "mean_conf": 0.4504351208047873, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.356468253968254, "error_rate_at_0_9": null, "confidence_bias": -0.049564879195212685, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.44516182609143595 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.42480437681661914 } }, "score_mae": 0.506799468965112, "ranked_probability_score": 0.1571076581150969 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.5523611111111111, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.5555555555555556, "flip_rate": 0.1111111111111111, "covered_only_flip_rate": 0.1111111111111111, "half_l1": 0.15052847452945783 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.6301195789558026, "share_at_0_9": 0.09090909090909091, "normalized_entropy": 0.7772340839901362 }, "covered_only_confidence_metrics": { "mean_max_p": 0.6301195789558026, "share_at_0_9": 0.09090909090909091, "normalized_entropy": 0.7772340839901362 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 61, "accuracy": 0.5545454545454546, "full_probability_metrics": { "n": 110, "nll": 0.8060277998867056, "acc": 0.5545454545454546, "ece": 0.10295063900590565, "brier": 0.5126100497884581, "mean_conf": 0.6494020931948475, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09090909090909091, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.13636363636363635, "coverage_at_1pct_error": 0.13636363636363635, "aurc": 0.2641604332676767, "error_rate_at_0_9": 0.0, "confidence_bias": 0.09485663864939298, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7272727272727273, "confidence_cutoff": 0.639728143209103 }, "0.8": { "coverage": 0.8, "accuracy": 0.6136363636363636, "confidence_cutoff": 0.4829787701923171 } }, "score_mae": 0.6189534882598504, "ranked_probability_score": 0.18787654625033254 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.8060277998867056, "acc": 0.5545454545454546, "ece": 0.10295063900590565, "brier": 0.5126100497884581, "mean_conf": 0.6494020931948475, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09090909090909091, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.13636363636363635, "coverage_at_1pct_error": 0.13636363636363635, "aurc": 0.2641604332676767, "error_rate_at_0_9": 0.0, "confidence_bias": 0.09485663864939298, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7272727272727273, "confidence_cutoff": 0.639728143209103 }, "0.8": { "coverage": 0.8, "accuracy": 0.6136363636363636, "confidence_cutoff": 0.4829787701923171 } }, "score_mae": 0.6189534882598504, "ranked_probability_score": 0.18787654625033254 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.019282514239044705 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.9232584714693672, "paired_flip": { "pairs": 64, "flip_rate": 0.046875, "both_correct_rate": 0.03125, "invariant_pairs": 24, "invariance_rate": 0.9583333333333334, "invariant_both_correct_rate": 0.5 }, "unknowable": { "n": 110, "mean_max_p": 0.6301195789558026, "share_at_0_9": 0.09090909090909091, "control_mean_max_p": 0.6494020931948475, "control_share_at_0_9": 0.09090909090909091, "control_acc": 0.5545454545454546, "paired_confidence_drop": 0.019282514239044705, "share_less_confident_than_control": 0.5818181818181818 }, "clean": { "n": 1046, "nll": 1.140604453192763, "acc": 0.5200764818355641, "ece": 0.13956812478907013, "brier": 0.5847272961307521, "mean_conf": 0.6596446066246342, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.2676864244741874, "accuracy_at_0_9": 0.8464285714285714, "coverage_at_5pct_error": 0.1022944550669216, "coverage_at_1pct_error": 0.06405353728489484, "aurc": 0.2611055042681389, "error_rate_at_0_9": 0.15357142857142858, "confidence_bias": 0.1395681247890701, "top_bins": { "0.9": { "n": 280, "errors": 43, "error_rate": 0.15357142857142858 }, "0.95": { "n": 172, "errors": 17, "error_rate": 0.09883720930232558 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7093690248565966, "confidence_cutoff": 0.6627623778724683 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6164874551971327, "confidence_cutoff": 0.43874029305741846 } }, "score_mae": 0.6646501516822292, "ranked_probability_score": 0.20382515703147194 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.6922680067235276, "acc": 0.4, "ece": 0.19980236035018145, "brier": 0.7677341836497671, "mean_conf": 0.47849831189284203, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5530819797247816, "error_rate_at_0_9": null, "confidence_bias": 0.07849831189284201, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.4973419665436558 }, "0.8": { "coverage": 0.8, "accuracy": 0.4375, "confidence_cutoff": 0.3228425991641082 } } }, "buried_paws": { "n": 20, "nll": 0.5982875413119898, "acc": 0.7, "ece": 0.11075831991651361, "brier": 0.4001615387031127, "mean_conf": 0.8107583199165134, "confident_error_rate": 0.05, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.8571428571428571, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.1632702252168196, "error_rate_at_0_9": 0.14285714285714285, "confidence_bias": 0.11075831991651341, "top_bins": { "0.9": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8505409829237911 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7491870969049392 } } }, "buried_qnli": { "n": 20, "nll": 0.4599992511278768, "acc": 0.85, "ece": 0.1158713711282571, "brier": 0.26410424468040555, "mean_conf": 0.8705393336106015, "confident_error_rate": 0.05, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.10557834408453606, "error_rate_at_0_9": 0.1, "confidence_bias": 0.020539333610601562, "top_bins": { "0.9": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.95": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9002058908435355 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8476532137135575 } } }, "buried_tweet_offensive": { "n": 20, "nll": 1.9649514082589004, "acc": 0.15, "ece": 0.7461943399452262, "brier": 1.3496773276376004, "mean_conf": 0.8961943399452263, "confident_error_rate": 0.45, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7559541307935272, "error_rate_at_0_9": 0.75, "confidence_bias": 0.7461943399452263, "top_bins": { "0.9": { "n": 12, "errors": 9, "error_rate": 0.75 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.9223600304830083 }, "0.8": { "coverage": 0.8, "accuracy": 0.1875, "confidence_cutoff": 0.8789992463439591 } } }, "composition_final_combination": { "n": 32, "nll": 0.7103396907796085, "acc": 0.375, "ece": 0.2518146996729819, "brier": 0.5195104923922356, "mean_conf": 0.5991911717131082, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.42419622085140174, "error_rate_at_0_9": null, "confidence_bias": 0.2241911717131082, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5294117647058824, "confidence_cutoff": 0.5852385584938956 }, "0.8": { "coverage": 0.8125, "accuracy": 0.38461538461538464, "confidence_cutoff": 0.5485769510269165 } } }, "composition_final_exception": { "n": 32, "nll": 0.6119872736170107, "acc": 0.53125, "ece": 0.1663285734801442, "brier": 0.4250681326420055, "mean_conf": 0.6087256435466561, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.21875, "coverage_at_1pct_error": 0.21875, "aurc": 0.22280994275960814, "error_rate_at_0_9": null, "confidence_bias": 0.07747564354665615, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.6103642400990653 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6153846153846154, "confidence_cutoff": 0.5236621499061584 } } }, "composition_final_negation": { "n": 32, "nll": 0.6214072129897903, "acc": 0.6875, "ece": 0.09700088834713139, "brier": 0.43061617531605, "mean_conf": 0.5904991116528687, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.21277250791235114, "error_rate_at_0_9": null, "confidence_bias": -0.09700088834713128, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.6470588235294118, "confidence_cutoff": 0.5689421892166138 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7692307692307693, "confidence_cutoff": 0.5137548752877041 } } }, "contrastive_authorization": { "n": 40, "nll": 0.8723859824290997, "acc": 0.5, "ece": 0.4244367108871019, "brier": 0.6501338236771121, "mean_conf": 0.8518532337953731, "confident_error_rate": 0.0, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.425, "coverage_at_1pct_error": 0.425, "aurc": 0.1772797029704015, "error_rate_at_0_9": 0.0, "confidence_bias": 0.35185323379537314, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.8627102760692065 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7737385745903638 } } }, "contrastive_deadline": { "n": 40, "nll": 1.1619693513204439, "acc": 0.5, "ece": 0.08339102781151485, "brier": 0.6603451502681464, "mean_conf": 0.5457050910932919, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.42314987081059563, "error_rate_at_0_9": null, "confidence_bias": 0.045705091093291905, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.55, "confidence_cutoff": 0.5472279598733752 }, "0.8": { "coverage": 0.8, "accuracy": 0.46875, "confidence_cutoff": 0.49062062096801606 } }, "score_mae": 0.7331951468157973, "ranked_probability_score": 0.227748073203181 }, "emotion": { "n": 80, "nll": 1.4384755112737106, "acc": 0.45, "ece": 0.23354828672781014, "brier": 0.6782082601971992, "mean_conf": 0.6835482867278101, "confident_error_rate": 0.05, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 0.8095238095238095, "coverage_at_5pct_error": 0.1125, "coverage_at_1pct_error": 0.1125, "aurc": 0.3075475611515972, "error_rate_at_0_9": 0.19047619047619047, "confidence_bias": 0.23354828672781008, "top_bins": { "0.9": { "n": 21, "errors": 4, "error_rate": 0.19047619047619047 }, "0.95": { "n": 13, "errors": 3, "error_rate": 0.23076923076923078 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.7035340538594081 }, "0.8": { "coverage": 0.8, "accuracy": 0.546875, "confidence_cutoff": 0.4568873397079616 } } }, "mmlu": { "n": 80, "nll": 1.3090704914241156, "acc": 0.475, "ece": 0.14470752412362756, "brier": 0.6651548126894188, "mean_conf": 0.5401188879032786, "confident_error_rate": 0.025, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3692044013257334, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.06511888790327858, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.4982062829825442 }, "0.8": { "coverage": 0.8, "accuracy": 0.53125, "confidence_cutoff": 0.3532145619392395 } } }, "mmlu_pro": { "n": 200, "nll": 2.3089932657368935, "acc": 0.17, "ece": 0.15466899840526752, "brier": 0.8998449734534026, "mean_conf": 0.324351039449046, "confident_error_rate": 0.0, "coverage_at_0_9": 0.01, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.01, "coverage_at_1pct_error": 0.01, "aurc": 0.7036049462279607, "error_rate_at_0_9": 0.0, "confidence_bias": 0.15435103944904596, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.26, "confidence_cutoff": 0.2645310173352473 }, "0.8": { "coverage": 0.8, "accuracy": 0.2, "confidence_cutoff": 0.15071202306993456 } } }, "paws": { "n": 80, "nll": 0.6836634797207242, "acc": 0.7125, "ece": 0.17934968602751228, "brier": 0.4134595515140756, "mean_conf": 0.8918496860275124, "confident_error_rate": 0.125, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 0.8214285714285714, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.3375, "aurc": 0.1169251381554202, "error_rate_at_0_9": 0.17857142857142858, "confidence_bias": 0.17934968602751233, "top_bins": { "0.9": { "n": 56, "errors": 10, "error_rate": 0.17857142857142858 }, "0.95": { "n": 43, "errors": 7, "error_rate": 0.16279069767441862 }, "0.99": { "n": 17, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.9672752130844654 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7933746339865138 } } }, "qnli": { "n": 80, "nll": 0.4234729256173562, "acc": 0.8125, "ece": 0.08154498996045036, "brier": 0.26487978917079796, "mean_conf": 0.875528983544843, "confident_error_rate": 0.05, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.92, "coverage_at_5pct_error": 0.55, "coverage_at_1pct_error": 0.025, "aurc": 0.09236898219002079, "error_rate_at_0_9": 0.08, "confidence_bias": 0.06302898354484299, "top_bins": { "0.9": { "n": 50, "errors": 4, "error_rate": 0.08 }, "0.95": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9245217259759483 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7813270571630142 } } }, "sciq": { "n": 80, "nll": 0.12773807478479932, "acc": 0.975, "ece": 0.042135866947253076, "brier": 0.05761418509406378, "mean_conf": 0.9330920038477387, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.8125, "aurc": 0.003545759269761683, "error_rate_at_0_9": 0.0, "confidence_bias": -0.04190799615226126, "top_bins": { "0.9": { "n": 61, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 30, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9766970841042327 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8963993749335851 } } }, "tweet_offensive": { "n": 80, "nll": 1.076663463803308, "acc": 0.525, "ece": 0.33069550075925347, "brier": 0.7108048153709255, "mean_conf": 0.8147667972496968, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.48164792381461224, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.2897667972496968, "top_bins": { "0.9": { "n": 28, "errors": 12, "error_rate": 0.42857142857142855 }, "0.95": { "n": 11, "errors": 5, "error_rate": 0.45454545454545453 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.525, "confidence_cutoff": 0.8547389634659732 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6671240131332882 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.9609863049688819, "acc": 0.5, "ece": 0.31785332132940547, "brier": 0.7041868887416481, "mean_conf": 0.8178533213294055, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.31785332132940547, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8065210460644805 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7894791725433119 } } }, "unknowable_authorization": { "n": 10, "nll": 1.388098909264548, "acc": 0.5, "ece": 0.4332876898053106, "brier": 0.8717328054350422, "mean_conf": 0.9332876898053106, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.3023015873015873, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4332876898053106, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9469728120744589 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9118526630489431 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.0804363010151486, "acc": 1.0, "ece": 0.07499668897844983, "brier": 0.019488772524919224, "mean_conf": 0.9250033110215503, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07499668897844969, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9773614188128392 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8755277134320821 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.8993057373066659, "acc": 0.5, "ece": 0.37620665693587, "brier": 0.6694469865764338, "mean_conf": 0.8433104243341202, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.22218253968253965, "error_rate_at_0_9": 0.0, "confidence_bias": 0.3433104243341202, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8683824331335321 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8090215211523486 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.078318425147064, "acc": 0.4, "ece": 0.21827502538583887, "brier": 0.6392985173942919, "mean_conf": 0.5448525953261908, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4484126984126983, "error_rate_at_0_9": null, "confidence_bias": 0.14485259532619077, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5694281501280378 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5023539141171264 } }, "score_mae": 0.6744406501165433, "ranked_probability_score": 0.20361680207298805 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.8257367555082981, "acc": 0.6, "ece": 0.20315558166625602, "brier": 0.5368300664727356, "mean_conf": 0.6927817124371142, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.446468253968254, "error_rate_at_0_9": null, "confidence_bias": 0.0927817124371142, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7525878724127334 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6318264509465329 } }, "score_mae": 0.4942852151824952, "ranked_probability_score": 0.13674801772209272 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.7411287197432239, "acc": 0.5, "ece": 0.33488537025880927, "brier": 0.41978723877464386, "mean_conf": 0.5404037240333779, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": null, "confidence_bias": 0.040403724033377886, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.6493193153038366 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.3707034179358656 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.7304189222342764, "acc": 0.5, "ece": 0.12032863124216035, "brier": 0.5349363083274995, "mean_conf": 0.6203286312421604, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.543968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.12032863124216042, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6419200301170349 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5817150300230077 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 1.3919532483735586, "acc": 0.2, "ece": 0.45759628518374096, "brier": 0.8839505147660234, "mean_conf": 0.5808141287879629, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6859920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.3808141287879629, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6313703938441154 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.4809979283363553 } }, "score_mae": 0.9450878389130155, "ranked_probability_score": 0.32573415308930975 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.0889432071009875, "acc": 0.3, "ece": 0.3326331805458419, "brier": 0.7081347505541592, "mean_conf": 0.6326331805458418, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5546428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.33263318054584184, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6303047819571844 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5840732368219906 } }, "score_mae": 0.6461730700376959, "ranked_probability_score": 0.20695672051012787 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.5415958593526109, "acc": 0.7, "ece": 0.2550459061934608, "brier": 0.3559473859760248, "mean_conf": 0.7142320425289375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.12246031746031745, "error_rate_at_0_9": 0.0, "confidence_bias": 0.014232042528937505, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7415914077608177 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5424330133491566 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.5665938867164622, "acc": 0.9, "ece": 0.30137184591871985, "brier": 0.3096533035516891, "mean_conf": 0.5986281540812801, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.3013718459187199, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.5679470258359087 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5116761999081999 } }, "score_mae": 0.4469346863442409, "ranked_probability_score": 0.09709592599237986 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.9218747362554651, "acc": 0.5, "ece": 0.04956487919521275, "brier": 0.5612367027546188, "mean_conf": 0.4504351208047873, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.356468253968254, "error_rate_at_0_9": null, "confidence_bias": -0.049564879195212685, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.44516182609143595 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.42480437681661914 } }, "score_mae": 0.506799468965112, "ranked_probability_score": 0.1571076581150969 }, "unknowable_deadline": { "n": 10, "nll": 1.082707747553807, "acc": 0.5, "ece": 0.03381085261679436, "brier": 0.6491368001068862, "mean_conf": 0.5294463211234628, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.506031746031746, "error_rate_at_0_9": null, "confidence_bias": 0.029446321123462793, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5246076466603358 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5076066066471261 } }, "score_mae": 0.7040350904437757, "ranked_probability_score": 0.21270430486937442 }, "unknowable_late_fee": { "n": 10, "nll": 1.1418831420477165, "acc": 0.3, "ece": 0.18283314476114515, "brier": 0.7003310846546063, "mean_conf": 0.48283314476114514, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6483333333333332, "error_rate_at_0_9": null, "confidence_bias": 0.18283314476114515, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.4698658603509748 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4627956977070019 } }, "score_mae": 0.6946627285010155, "ranked_probability_score": 0.2243192482129038 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.3048501231231846, "acc": 0.5, "ece": 0.1529036906369674, "brier": 0.7109024251322064, "mean_conf": 0.6529036906369674, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.1529036906369674, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.6568431731901472 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6244806824600239 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7178510440432224, "acc": 0.5, "ece": 0.05941208480838296, "brier": 0.5238767655412675, "mean_conf": 0.5594120848083829, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.556031746031746, "error_rate_at_0_9": null, "confidence_bias": 0.05941208480838289, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5750499963760376 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5056763887405396 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.3677402293388377, "acc": 0.2, "ece": 0.3531587173814433, "brier": 0.8504291162505948, "mean_conf": 0.5531587173814433, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9452777777777779, "error_rate_at_0_9": null, "confidence_bias": 0.3531587173814433, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.5547798193278936 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.5371175488400094 } }, "score_mae": 0.9322957627053814, "ranked_probability_score": 0.3190046118260125 }, "unknowable_sla_response": { "n": 10, "nll": 1.0440307232779806, "acc": 0.5, "ece": 0.029419786680393203, "brier": 0.6281251960756025, "mean_conf": 0.47512296635230966, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": -0.02487703364769034, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.4739007141932526 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.45972721398689703 } }, "score_mae": 0.5767468477958181, "ranked_probability_score": 0.18848971696738578 }, "unknowable_spend_threshold": { "n": 10, "nll": 1.1013959726826534, "acc": 0.5, "ece": 0.3585677756918152, "brier": 0.7661584305100013, "mean_conf": 0.8585677756918152, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.3585677756918152, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8537230833255326 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.8512976844915905 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.3228914817126447, "acc": 0.4, "ece": 0.21114800528522598, "brier": 0.7804776520064267, "mean_conf": 0.6111480052852262, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6283333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.21114800528522615, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.5865577880573932 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5731940694691519 } }, "score_mae": 0.9094505438830824, "ranked_probability_score": 0.3070379097899052 }, "unknowable_warranty_claim": { "n": 10, "nll": 0.9823894531209312, "acc": 0.6, "ece": 0.14241834866163683, "brier": 0.5967797358535363, "mean_conf": 0.45758165133836315, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.2885714285714286, "error_rate_at_0_9": null, "confidence_bias": -0.14241834866163683, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.4573594170021837 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.44940969649495505 } }, "score_mae": 0.5626008598210509, "ranked_probability_score": 0.17631253045428857 } }, "variants": { "clean": { "n": 1156, "nll": 1.1394641084350987, "acc": 0.513840830449827, "ece": 0.14299430122362078, "brier": 0.5964066788545371, "mean_conf": 0.6568351316734479, "confident_error_rate": 0.04152249134948097, "coverage_at_0_9": 0.2508650519031142, "accuracy_at_0_9": 0.8344827586206897, "coverage_at_5pct_error": 0.092560553633218, "coverage_at_1pct_error": 0.05795847750865052, "aurc": 0.2768537179213685, "error_rate_at_0_9": 0.16551724137931034, "confidence_bias": 0.1429943012236209, "top_bins": { "0.9": { "n": 290, "errors": 48, "error_rate": 0.16551724137931034 }, "0.95": { "n": 172, "errors": 17, "error_rate": 0.09883720930232558 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6955017301038062, "confidence_cutoff": 0.6518979396927796 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.6054054054054054, "confidence_cutoff": 0.45206127917949196 } }, "score_mae": 0.689143334373276, "ranked_probability_score": 0.21663249327716186 }, "none_absent": { "n": 36, "nll": 2.547862059997515, "acc": 0.08333333333333333, "ece": 0.5146576308063049, "brier": 1.2141787126381738, "mean_conf": 0.5979909641396383, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.08333333333333333, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9752938918722884, "error_rate_at_0_9": 1.0, "confidence_bias": 0.5146576308063049, "top_bins": { "0.9": { "n": 3, "errors": 3, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.5676298420338048 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.034482758620689655, "confidence_cutoff": 0.4325112096414036 } } }, "none_present": { "n": 36, "nll": 1.073902026229569, "acc": 0.6111111111111112, "ece": 0.12866874273559475, "brier": 0.4586288165507695, "mean_conf": 0.6564114048685927, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3333333333333333, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4166666666666667, "coverage_at_1pct_error": 0.4166666666666667, "aurc": 0.14084691504659977, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04530029375748157, "top_bins": { "0.9": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8888888888888888, "confidence_cutoff": 0.6629634187522506 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.6896551724137931, "confidence_cutoff": 0.4065709841538935 } } }, "permuted": { "n": 36, "nll": 1.0680460421561149, "acc": 0.5833333333333334, "ece": 0.17834202420725892, "brier": 0.494763519004633, "mean_conf": 0.6811602518824487, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3333333333333333, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3888888888888889, "coverage_at_1pct_error": 0.3888888888888889, "aurc": 0.16128607585183194, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0978269185491153, "top_bins": { "0.9": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8888888888888888, "confidence_cutoff": 0.704864758752485 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.6896551724137931, "confidence_cutoff": 0.41936809449446344 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.139509735643348, "flip_rate": 0.1111111111111111 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 1.140604453192763, "acc": 0.5200764818355641, "ece": 0.13956812478907013, "brier": 0.5847272961307521, "mean_conf": 0.6596446066246342, "confident_error_rate": 0.04110898661567878, "coverage_at_0_9": 0.2676864244741874, "accuracy_at_0_9": 0.8464285714285714, "coverage_at_5pct_error": 0.1022944550669216, "coverage_at_1pct_error": 0.06405353728489484, "aurc": 0.2611055042681389, "error_rate_at_0_9": 0.15357142857142858, "confidence_bias": 0.1395681247890701, "top_bins": { "0.9": { "n": 280, "errors": 43, "error_rate": 0.15357142857142858 }, "0.95": { "n": 172, "errors": 17, "error_rate": 0.09883720930232558 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7093690248565966, "confidence_cutoff": 0.6627623778724683 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6164874551971327, "confidence_cutoff": 0.43874029305741846 } }, "score_mae": 0.6646501516822292, "ranked_probability_score": 0.20382515703147194 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "kev-0.8b": { "panels": { "old_core": { "point": 0.6013764880952381, "exact_fraction": "3233/5376", "ci95": [ 0.5636532738095238, 0.6389518229166667 ] }, "v3_core": { "point": 0.42291666666666666, "exact_fraction": "203/480", "ci95": [ 0.40041666666666664, 0.4458333333333333 ] }, "v4": { "point": 0.678125, "exact_fraction": "217/320", "ci95": [ 0.6290165213112032, 0.726565036525974 ] }, "v5": { "point": 0.6875, "exact_fraction": "11/16", "ci95": [ 0.6506416866656668, 0.7240803290265815 ] }, "transfer_v9_test": { "point": 0.6118546845124283, "exact_fraction": "320/523", "ci95": [ 0.5782116164448567, 0.6448523883329422 ] }, "original_four_panel_mean": { "point": 0.5974795386904762, "exact_fraction": "64241/107520", "ci95": [ 0.5786786699999984, 0.6161705292117233 ] }, "weighted_mean": { "point": 0.5827640657721024, "exact_fraction": "81926371/140582400", "ci95": [ 0.5663895425667974, 0.598946310408207 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 111, "requested": 128 }, "boolean_constraints": { "correct": 52, "requested": 64 }, "dbpedia_14": { "correct": 108, "requested": 112 }, "natural_intents": { "correct": 64, "requested": 64 }, "option_carrier": { "correct": 32, "requested": 96 }, "ordinal_rubric": { "correct": 38, "requested": 64 }, "relational_composition": { "correct": 43, "requested": 96 }, "scoped_evidence": { "correct": 10, "requested": 96 }, "state_tracking": { "correct": 30, "requested": 96 }, "unknown_rejection": { "correct": 37, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 40, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 16, "requested": 80 }, "massive_en": { "correct": 100, "requested": 120 }, "massive_zh": { "correct": 103, "requested": 120 }, "multiset_reconciliation": { "correct": 20, "requested": 80 }, "ordinal_service_loss": { "correct": 22, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 22, "requested": 80 }, "temporal_exclusion": { "correct": 17, "requested": 80 }, "transaction_recovery": { "correct": 26, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 119, "requested": 160 }, "belebele_en": { "correct": 102, "requested": 160 }, "belebele_zh": { "correct": 94, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 51, "requested": 120 }, "squad2_answerability": { "correct": 80, "requested": 120 }, "snli": { "correct": 93, "requested": 120 }, "qasc": { "correct": 106, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 640, "accuracy": 0.6118546845124283, "full_probability_metrics": { "n": 1046, "nll": 0.9489348241911458, "acc": 0.6118546845124283, "ece": 0.025818167713742683, "brier": 0.4839082184775719, "mean_conf": 0.6222309763026364, "confident_error_rate": 0.0076481835564053535, "coverage_at_0_9": 0.1826003824091778, "accuracy_at_0_9": 0.9581151832460733, "coverage_at_5pct_error": 0.21606118546845124, "coverage_at_1pct_error": 0.13097514340344169, "aurc": 0.17821722891968061, "error_rate_at_0_9": 0.041884816753926704, "confidence_bias": 0.010376291790208048, "top_bins": { "0.9": { "n": 191, "errors": 8, "error_rate": 0.041884816753926704 }, "0.95": { "n": 122, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8183556405353728, "confidence_cutoff": 0.6650095781678212 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7084826762246117, "confidence_cutoff": 0.3603672867152023 } }, "score_mae": 0.41780077899809037, "ranked_probability_score": 0.12404296786366242 }, "covered_only_probability_metrics": { "n": 1046, "nll": 0.9489348241911458, "acc": 0.6118546845124283, "ece": 0.025818167713742683, "brier": 0.4839082184775719, "mean_conf": 0.6222309763026364, "confident_error_rate": 0.0076481835564053535, "coverage_at_0_9": 0.1826003824091778, "accuracy_at_0_9": 0.9581151832460733, "coverage_at_5pct_error": 0.21606118546845124, "coverage_at_1pct_error": 0.13097514340344169, "aurc": 0.17821722891968061, "error_rate_at_0_9": 0.041884816753926704, "confidence_bias": 0.010376291790208048, "top_bins": { "0.9": { "n": 191, "errors": 8, "error_rate": 0.041884816753926704 }, "0.95": { "n": 122, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8183556405353728, "confidence_cutoff": 0.6650095781678212 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7084826762246117, "confidence_cutoff": 0.3603672867152023 } }, "score_mae": 0.41780077899809037, "ranked_probability_score": 0.12404296786366242 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 8, "accuracy": 0.4, "full_probability_metrics": { "n": 20, "nll": 1.6328041429043239, "acc": 0.4, "ece": 0.11281607470751408, "brier": 0.7304192172416464, "mean_conf": 0.43283199529103183, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.38751183500022507, "error_rate_at_0_9": null, "confidence_bias": 0.03283199529103181, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.4774935988311551 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.2867279382801943 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.6328041429043239, "acc": 0.4, "ece": 0.11281607470751408, "brier": 0.7304192172416464, "mean_conf": 0.43283199529103183, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.38751183500022507, "error_rate_at_0_9": null, "confidence_bias": 0.03283199529103181, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.4774935988311551 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.2867279382801943 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 12, "accuracy": 0.6, "full_probability_metrics": { "n": 20, "nll": 0.7297084552328041, "acc": 0.6, "ece": 0.15529728035791762, "brier": 0.5279158075089577, "mean_conf": 0.7446538834640174, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.3164015922587905, "error_rate_at_0_9": null, "confidence_bias": 0.14465388346401742, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7581421845874516 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.7255959890605215 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.7297084552328041, "acc": 0.6, "ece": 0.15529728035791762, "brier": 0.5279158075089577, "mean_conf": 0.7446538834640174, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.3164015922587905, "error_rate_at_0_9": null, "confidence_bias": 0.14465388346401742, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7581421845874516 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.7255959890605215 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 17, "accuracy": 0.85, "full_probability_metrics": { "n": 20, "nll": 0.5106882179688628, "acc": 0.85, "ece": 0.16705020228547476, "brier": 0.33370132013000237, "mean_conf": 0.718156051319082, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.15138533529593898, "error_rate_at_0_9": 0.0, "confidence_bias": -0.13184394868091798, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7218795252966598 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5940769733763643 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5106882179688628, "acc": 0.85, "ece": 0.16705020228547476, "brier": 0.33370132013000237, "mean_conf": 0.718156051319082, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.15138533529593898, "error_rate_at_0_9": 0.0, "confidence_bias": -0.13184394868091798, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7218795252966598 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5940769733763643 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 12, "accuracy": 0.6, "full_probability_metrics": { "n": 20, "nll": 0.7450604547358973, "acc": 0.6, "ece": 0.15737830290473837, "brier": 0.5257647370594005, "mean_conf": 0.7323312665114348, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.43053427029626723, "error_rate_at_0_9": null, "confidence_bias": 0.1323312665114348, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7442948000343204 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.6650095781678212 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.7450604547358973, "acc": 0.6, "ece": 0.15737830290473837, "brier": 0.5257647370594005, "mean_conf": 0.7323312665114348, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.43053427029626723, "error_rate_at_0_9": null, "confidence_bias": 0.1323312665114348, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7442948000343204 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.6650095781678212 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 13, "accuracy": 0.40625, "full_probability_metrics": { "n": 32, "nll": 1.0320353927282957, "acc": 0.40625, "ece": 0.37978321392588305, "brier": 0.7410070756907946, "mean_conf": 0.7860332139258831, "confident_error_rate": 0.125, "coverage_at_0_9": 0.21875, "accuracy_at_0_9": 0.42857142857142855, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5425929023002614, "error_rate_at_0_9": 0.5714285714285714, "confidence_bias": 0.3797832139258831, "top_bins": { "0.9": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.47058823529411764, "confidence_cutoff": 0.7948651554495771 }, "0.8": { "coverage": 0.8125, "accuracy": 0.46153846153846156, "confidence_cutoff": 0.6413344954648915 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 1.0320353927282957, "acc": 0.40625, "ece": 0.37978321392588305, "brier": 0.7410070756907946, "mean_conf": 0.7860332139258831, "confident_error_rate": 0.125, "coverage_at_0_9": 0.21875, "accuracy_at_0_9": 0.42857142857142855, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5425929023002614, "error_rate_at_0_9": 0.5714285714285714, "confidence_bias": 0.3797832139258831, "top_bins": { "0.9": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.47058823529411764, "confidence_cutoff": 0.7948651554495771 }, "0.8": { "coverage": 0.8125, "accuracy": 0.46153846153846156, "confidence_cutoff": 0.6413344954648915 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 21, "accuracy": 0.65625, "full_probability_metrics": { "n": 32, "nll": 0.717299870586857, "acc": 0.65625, "ece": 0.30315644031633326, "brier": 0.5011811626132091, "mean_conf": 0.7164429849804473, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.48329043683112194, "error_rate_at_0_9": null, "confidence_bias": 0.06019298498044734, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7472231984138489 }, "0.8": { "coverage": 0.84375, "accuracy": 0.7037037037037037, "confidence_cutoff": 0.6238813624452733 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.717299870586857, "acc": 0.65625, "ece": 0.30315644031633326, "brier": 0.5011811626132091, "mean_conf": 0.7164429849804473, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.48329043683112194, "error_rate_at_0_9": null, "confidence_bias": 0.06019298498044734, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7472231984138489 }, "0.8": { "coverage": 0.84375, "accuracy": 0.7037037037037037, "confidence_cutoff": 0.6238813624452733 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 23, "accuracy": 0.71875, "full_probability_metrics": { "n": 32, "nll": 0.5322047320655654, "acc": 0.71875, "ece": 0.17014622114123046, "brier": 0.3473982110584136, "mean_conf": 0.6488173186822086, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.14944154536939344, "error_rate_at_0_9": null, "confidence_bias": -0.0699326813177914, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.6580008660555402 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8846153846153846, "confidence_cutoff": 0.5313978791236877 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.5322047320655654, "acc": 0.71875, "ece": 0.17014622114123046, "brier": 0.3473982110584136, "mean_conf": 0.6488173186822086, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.14944154536939344, "error_rate_at_0_9": null, "confidence_bias": -0.0699326813177914, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.6580008660555402 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8846153846153846, "confidence_cutoff": 0.5313978791236877 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 39, "accuracy": 0.975, "full_probability_metrics": { "n": 40, "nll": 0.15131752514826352, "acc": 0.975, "ece": 0.11294708422717381, "brier": 0.05668953176003259, "mean_conf": 0.8674847096952952, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10751529030470475, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9185473391305456 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8089489099696603 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.15131752514826352, "acc": 0.975, "ece": 0.11294708422717381, "brier": 0.05668953176003259, "mean_conf": 0.8674847096952952, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10751529030470475, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9185473391305456 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8089489099696603 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 18, "accuracy": 0.45, "full_probability_metrics": { "n": 40, "nll": 1.0345684773009012, "acc": 0.45, "ece": 0.16219795036058843, "brier": 0.6620965862152072, "mean_conf": 0.5732693792864124, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5082073760882855, "error_rate_at_0_9": null, "confidence_bias": 0.1232693792864124, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5608630305721544 }, "0.8": { "coverage": 0.8, "accuracy": 0.40625, "confidence_cutoff": 0.4430796504020691 } }, "score_mae": 0.6340324900423988, "ranked_probability_score": 0.19847476556704904 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.0345684773009012, "acc": 0.45, "ece": 0.16219795036058843, "brier": 0.6620965862152072, "mean_conf": 0.5732693792864124, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5082073760882855, "error_rate_at_0_9": null, "confidence_bias": 0.1232693792864124, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5608630305721544 }, "0.8": { "coverage": 0.8, "accuracy": 0.40625, "confidence_cutoff": 0.4430796504020691 } }, "score_mae": 0.6340324900423988, "ranked_probability_score": 0.19847476556704904 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 48, "accuracy": 0.6, "full_probability_metrics": { "n": 80, "nll": 1.1265468044675306, "acc": 0.6, "ece": 0.11443615014249534, "brier": 0.5217501476585913, "mean_conf": 0.5179636746526439, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2125, "coverage_at_1pct_error": 0.2125, "aurc": 0.173451131532144, "error_rate_at_0_9": 0.0, "confidence_bias": -0.08203632534735605, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.4884979135693363 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.3357400992071569 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.1265468044675306, "acc": 0.6, "ece": 0.11443615014249534, "brier": 0.5217501476585913, "mean_conf": 0.5179636746526439, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2125, "coverage_at_1pct_error": 0.2125, "aurc": 0.173451131532144, "error_rate_at_0_9": 0.0, "confidence_bias": -0.08203632534735605, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.4884979135693363 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.3357400992071569 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 41, "accuracy": 0.5125, "full_probability_metrics": { "n": 80, "nll": 1.1885759361017207, "acc": 0.5125, "ece": 0.10398958292861613, "brier": 0.6384140927539121, "mean_conf": 0.4705273931820525, "confident_error_rate": 0.0, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.39327722144608956, "error_rate_at_0_9": 0.0, "confidence_bias": -0.041972606817947455, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.65, "confidence_cutoff": 0.42177510104412175 }, "0.8": { "coverage": 0.8, "accuracy": 0.515625, "confidence_cutoff": 0.33762807660186517 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.1885759361017207, "acc": 0.5125, "ece": 0.10398958292861613, "brier": 0.6384140927539121, "mean_conf": 0.4705273931820525, "confident_error_rate": 0.0, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.39327722144608956, "error_rate_at_0_9": 0.0, "confidence_bias": -0.041972606817947455, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.65, "confidence_cutoff": 0.42177510104412175 }, "0.8": { "coverage": 0.8, "accuracy": 0.515625, "confidence_cutoff": 0.33762807660186517 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 45, "accuracy": 0.225, "full_probability_metrics": { "n": 200, "nll": 2.1558072536966413, "acc": 0.225, "ece": 0.07299126860376964, "brier": 0.8654828504835517, "mean_conf": 0.27562993573256056, "confident_error_rate": 0.0, "coverage_at_0_9": 0.005, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.02, "coverage_at_1pct_error": 0.02, "aurc": 0.6635123360876309, "error_rate_at_0_9": 0.0, "confidence_bias": 0.050629935732560555, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.21746287632371644 }, "0.8": { "coverage": 0.8, "accuracy": 0.25625, "confidence_cutoff": 0.1625811746572265 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.1558072536966413, "acc": 0.225, "ece": 0.07299126860376964, "brier": 0.8654828504835517, "mean_conf": 0.27562993573256056, "confident_error_rate": 0.0, "coverage_at_0_9": 0.005, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.02, "coverage_at_1pct_error": 0.02, "aurc": 0.6635123360876309, "error_rate_at_0_9": 0.0, "confidence_bias": 0.050629935732560555, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.21746287632371644 }, "0.8": { "coverage": 0.8, "accuracy": 0.25625, "confidence_cutoff": 0.1625811746572265 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 47, "accuracy": 0.5875, "full_probability_metrics": { "n": 80, "nll": 0.6944234783738545, "acc": 0.5875, "ece": 0.1262200499385823, "brier": 0.49963081595517106, "mean_conf": 0.6750599829681917, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3527140864591778, "error_rate_at_0_9": null, "confidence_bias": 0.08755998296819167, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.68075031042099 }, "0.8": { "coverage": 0.8, "accuracy": 0.59375, "confidence_cutoff": 0.6360945512028172 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.6944234783738545, "acc": 0.5875, "ece": 0.1262200499385823, "brier": 0.49963081595517106, "mean_conf": 0.6750599829681917, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3527140864591778, "error_rate_at_0_9": null, "confidence_bias": 0.08755998296819167, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.68075031042099 }, "0.8": { "coverage": 0.8, "accuracy": 0.59375, "confidence_cutoff": 0.6360945512028172 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 65, "accuracy": 0.8125, "full_probability_metrics": { "n": 80, "nll": 0.4555073999657404, "acc": 0.8125, "ece": 0.0915795153688172, "brier": 0.2870652400868212, "mean_conf": 0.7690534034702379, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.875, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.09462801816327258, "error_rate_at_0_9": 0.125, "confidence_bias": -0.04344659652976213, "top_bins": { "0.9": { "n": 24, "errors": 3, "error_rate": 0.125 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.7596169114112854 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.6030593277847366 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4555073999657404, "acc": 0.8125, "ece": 0.0915795153688172, "brier": 0.2870652400868212, "mean_conf": 0.7690534034702379, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.875, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.09462801816327258, "error_rate_at_0_9": 0.125, "confidence_bias": -0.04344659652976213, "top_bins": { "0.9": { "n": 24, "errors": 3, "error_rate": 0.125 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.7596169114112854 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.6030593277847366 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 77, "accuracy": 0.9625, "full_probability_metrics": { "n": 80, "nll": 0.18834000915162102, "acc": 0.9625, "ece": 0.08085216985910534, "brier": 0.09050740969997009, "mean_conf": 0.8816478301408945, "confident_error_rate": 0.0, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.7625, "aurc": 0.005139395903322137, "error_rate_at_0_9": 0.0, "confidence_bias": -0.08085216985910548, "top_bins": { "0.9": { "n": 54, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 45, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9593347088320329 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.792910008619996 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.18834000915162102, "acc": 0.9625, "ece": 0.08085216985910534, "brier": 0.09050740969997009, "mean_conf": 0.8816478301408945, "confident_error_rate": 0.0, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.7625, "aurc": 0.005139395903322137, "error_rate_at_0_9": 0.0, "confidence_bias": -0.08085216985910548, "top_bins": { "0.9": { "n": 54, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 45, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9593347088320329 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.792910008619996 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 58, "accuracy": 0.725, "full_probability_metrics": { "n": 80, "nll": 0.5194802188489087, "acc": 0.725, "ece": 0.04920898242311043, "brier": 0.34811439847415704, "mean_conf": 0.7606741796687964, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.13595768124624372, "error_rate_at_0_9": 0.0625, "confidence_bias": 0.03567417966879638, "top_bins": { "0.9": { "n": 16, "errors": 1, "error_rate": 0.0625 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.7611014133527665 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.6431154204343384 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.5194802188489087, "acc": 0.725, "ece": 0.04920898242311043, "brier": 0.34811439847415704, "mean_conf": 0.7606741796687964, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.13595768124624372, "error_rate_at_0_9": 0.0625, "confidence_bias": 0.03567417966879638, "top_bins": { "0.9": { "n": 16, "errors": 1, "error_rate": 0.0625 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.7611014133527665 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.6431154204343384 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.010614667252794148, "acc": 1.0, "ece": 0.0105551277111825, "brier": 0.0002362723484120104, "mean_conf": 0.9894448722888175, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0105551277111825, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9910509466345773 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9867438874859482 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.010614667252794148, "acc": 1.0, "ece": 0.0105551277111825, "brier": 0.0002362723484120104, "mean_conf": 0.9894448722888175, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0105551277111825, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9910509466345773 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9867438874859482 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.18762674207375457, "acc": 1.0, "ece": 0.1630505554036273, "brier": 0.0785331260905723, "mean_conf": 0.8369494445963725, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.16305055540362745, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9289364218711853 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7272492863908198 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.18762674207375457, "acc": 1.0, "ece": 0.1630505554036273, "brier": 0.0785331260905723, "mean_conf": 0.8369494445963725, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.16305055540362745, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9289364218711853 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7272492863908198 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.0819812812391447, "acc": 0.4, "ece": 0.2179296430245531, "brier": 0.6983678640867412, "mean_conf": 0.55364404768203, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7146428571428571, "error_rate_at_0_9": null, "confidence_bias": 0.15364404768203, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5413114428520203 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4340435266494751 } }, "score_mae": 0.6230918518689158, "ranked_probability_score": 0.2015582690795208 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0819812812391447, "acc": 0.4, "ece": 0.2179296430245531, "brier": 0.6983678640867412, "mean_conf": 0.55364404768203, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7146428571428571, "error_rate_at_0_9": null, "confidence_bias": 0.15364404768203, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5413114428520203 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4340435266494751 } }, "score_mae": 0.6230918518689158, "ranked_probability_score": 0.2015582690795208 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.24211831689569624, "acc": 0.9, "ece": 0.08309738728193819, "brier": 0.1504702562673398, "mean_conf": 0.9563502338155283, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05635023381552828, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9855445265994944 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9685846991808067 } }, "score_mae": 0.09716835579799854, "ranked_probability_score": 0.03528106133634211 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.24211831689569624, "acc": 0.9, "ece": 0.08309738728193819, "brier": 0.1504702562673398, "mean_conf": 0.9563502338155283, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05635023381552828, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9855445265994944 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9685846991808067 } }, "score_mae": 0.09716835579799854, "ranked_probability_score": 0.03528106133634211 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.02838703360000571, "acc": 1.0, "ece": 0.027635333708992738, "brier": 0.002268429428419828, "mean_conf": 0.9723646662910073, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027635333708992738, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9916285715087849 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9531984667826858 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.02838703360000571, "acc": 1.0, "ece": 0.027635333708992738, "brier": 0.002268429428419828, "mean_conf": 0.9723646662910073, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027635333708992738, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9916285715087849 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9531984667826858 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 8, "accuracy": 0.8, "full_probability_metrics": { "n": 10, "nll": 0.4967078190499693, "acc": 0.8, "ece": 0.28676134017722765, "brier": 0.32609330633710903, "mean_conf": 0.691431930937867, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.11246031746031745, "error_rate_at_0_9": null, "confidence_bias": -0.10856806906213301, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7418128252029419 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6112156510353088 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.4967078190499693, "acc": 0.8, "ece": 0.28676134017722765, "brier": 0.32609330633710903, "mean_conf": 0.691431930937867, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.11246031746031745, "error_rate_at_0_9": null, "confidence_bias": -0.10856806906213301, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7418128252029419 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6112156510353088 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.475859500379213, "acc": 1.0, "ece": 0.3731629842487789, "brier": 0.22978191865644892, "mean_conf": 0.6268370157512211, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": null, "confidence_bias": -0.3731629842487789, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.6530453964160264 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5752129979368151 } }, "score_mae": 0.35415910432790615, "ranked_probability_score": 0.06384588994201582 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.475859500379213, "acc": 1.0, "ece": 0.3731629842487789, "brier": 0.22978191865644892, "mean_conf": 0.6268370157512211, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": null, "confidence_bias": -0.3731629842487789, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.6530453964160264 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5752129979368151 } }, "score_mae": 0.35415910432790615, "ranked_probability_score": 0.06384588994201582 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.03205922369840776, "acc": 1.0, "ece": 0.031276946991082744, "brier": 0.002423337198586589, "mean_conf": 0.9687230530089173, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.031276946991082744, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9820183094843828 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9579578388269809 } }, "score_mae": 0.019461998768093222, "ranked_probability_score": 0.0004910640933817507 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.03205922369840776, "acc": 1.0, "ece": 0.031276946991082744, "brier": 0.002423337198586589, "mean_conf": 0.9687230530089173, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.031276946991082744, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9820183094843828 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9579578388269809 } }, "score_mae": 0.019461998768093222, "ranked_probability_score": 0.0004910640933817507 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.024872545233687492, "acc": 1.0, "ece": 0.024419907698640442, "brier": 0.0015987643049556542, "mean_conf": 0.9755800923013596, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024419907698640442, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9870816919645322 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9560908812669289 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.024872545233687492, "acc": 1.0, "ece": 0.024419907698640442, "brier": 0.0015987643049556542, "mean_conf": 0.9755800923013596, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024419907698640442, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9870816919645322 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9560908812669289 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.033222898291474914, "acc": 1.0, "ece": 0.03256888767009658, "brier": 0.001958310993865444, "mean_conf": 0.9674311123299034, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03256888767009658, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9742770906196108 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9501113885943084 } }, "score_mae": 0.034394007249953874, "ranked_probability_score": 0.0006179846620626799 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.033222898291474914, "acc": 1.0, "ece": 0.03256888767009658, "brier": 0.001958310993865444, "mean_conf": 0.9674311123299034, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03256888767009658, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9742770906196108 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9501113885943084 } }, "score_mae": 0.034394007249953874, "ranked_probability_score": 0.0006179846620626799 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.8650021967731402, "acc": 0.5, "ece": 0.17404689944827045, "brier": 0.534130946600855, "mean_conf": 0.5314912195576974, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.343968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.0314912195576974, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5749318174997514 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.45559284769882036 } }, "score_mae": 0.5136025117984403, "ranked_probability_score": 0.1447363472551048 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.8650021967731402, "acc": 0.5, "ece": 0.17404689944827045, "brier": 0.534130946600855, "mean_conf": 0.5314912195576974, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.343968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.0314912195576974, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5749318174997514 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.45559284769882036 } }, "score_mae": 0.5136025117984403, "ranked_probability_score": 0.1447363472551048 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.7289351851851852, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.5555555555555556, "flip_rate": 0.16666666666666666, "covered_only_flip_rate": 0.16666666666666666, "half_l1": 0.080796118569043 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.4211497994139846, "share_at_0_9": 0.0, "normalized_entropy": 0.9795540742636577 }, "covered_only_confidence_metrics": { "mean_max_p": 0.4211497994139846, "share_at_0_9": 0.0, "normalized_entropy": 0.9795540742636577 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 96, "accuracy": 0.8727272727272727, "full_probability_metrics": { "n": 110, "nll": 0.3162229294988444, "acc": 0.8727272727272727, "ece": 0.07557999413920456, "brier": 0.18416932111939147, "mean_conf": 0.8245679716873384, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5727272727272728, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7636363636363637, "coverage_at_1pct_error": 0.6, "aurc": 0.025463583898436594, "error_rate_at_0_9": 0.0, "confidence_bias": -0.048159301039934266, "top_bins": { "0.9": { "n": 63, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 52, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9423748009032622 }, "0.8": { "coverage": 0.8, "accuracy": 0.9431818181818182, "confidence_cutoff": 0.612665144175223 } }, "score_mae": 0.27364630496855136, "ranked_probability_score": 0.07442176939473798 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.3162229294988444, "acc": 0.8727272727272727, "ece": 0.07557999413920456, "brier": 0.18416932111939147, "mean_conf": 0.8245679716873384, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5727272727272728, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7636363636363637, "coverage_at_1pct_error": 0.6, "aurc": 0.025463583898436594, "error_rate_at_0_9": 0.0, "confidence_bias": -0.048159301039934266, "top_bins": { "0.9": { "n": 63, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 52, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9423748009032622 }, "0.8": { "coverage": 0.8, "accuracy": 0.9431818181818182, "confidence_cutoff": 0.612665144175223 } }, "score_mae": 0.27364630496855136, "ranked_probability_score": 0.07442176939473798 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.4034181722733537 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -0.6256600219912991, "paired_flip": { "pairs": 64, "flip_rate": 0.5625, "both_correct_rate": 0.453125, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.5833333333333334 }, "unknowable": { "n": 110, "mean_max_p": 0.4211497994139846, "share_at_0_9": 0.0, "control_mean_max_p": 0.8245679716873384, "control_share_at_0_9": 0.5727272727272728, "control_acc": 0.8727272727272727, "paired_confidence_drop": 0.4034181722733537, "share_less_confident_than_control": 0.9545454545454546 }, "clean": { "n": 1046, "nll": 0.9489348241911458, "acc": 0.6118546845124283, "ece": 0.025818167713742683, "brier": 0.4839082184775719, "mean_conf": 0.6222309763026364, "confident_error_rate": 0.0076481835564053535, "coverage_at_0_9": 0.1826003824091778, "accuracy_at_0_9": 0.9581151832460733, "coverage_at_5pct_error": 0.21606118546845124, "coverage_at_1pct_error": 0.13097514340344169, "aurc": 0.17821722891968061, "error_rate_at_0_9": 0.041884816753926704, "confidence_bias": 0.010376291790208048, "top_bins": { "0.9": { "n": 191, "errors": 8, "error_rate": 0.041884816753926704 }, "0.95": { "n": 122, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8183556405353728, "confidence_cutoff": 0.6650095781678212 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7084826762246117, "confidence_cutoff": 0.3603672867152023 } }, "score_mae": 0.41780077899809037, "ranked_probability_score": 0.12404296786366242 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.6328041429043239, "acc": 0.4, "ece": 0.11281607470751408, "brier": 0.7304192172416464, "mean_conf": 0.43283199529103183, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.38751183500022507, "error_rate_at_0_9": null, "confidence_bias": 0.03283199529103181, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.4774935988311551 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.2867279382801943 } } }, "buried_paws": { "n": 20, "nll": 0.7297084552328041, "acc": 0.6, "ece": 0.15529728035791762, "brier": 0.5279158075089577, "mean_conf": 0.7446538834640174, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.3164015922587905, "error_rate_at_0_9": null, "confidence_bias": 0.14465388346401742, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7581421845874516 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.7255959890605215 } } }, "buried_qnli": { "n": 20, "nll": 0.5106882179688628, "acc": 0.85, "ece": 0.16705020228547476, "brier": 0.33370132013000237, "mean_conf": 0.718156051319082, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.15138533529593898, "error_rate_at_0_9": 0.0, "confidence_bias": -0.13184394868091798, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7218795252966598 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5940769733763643 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.7450604547358973, "acc": 0.6, "ece": 0.15737830290473837, "brier": 0.5257647370594005, "mean_conf": 0.7323312665114348, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.43053427029626723, "error_rate_at_0_9": null, "confidence_bias": 0.1323312665114348, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7442948000343204 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.6650095781678212 } } }, "composition_final_combination": { "n": 32, "nll": 1.0320353927282957, "acc": 0.40625, "ece": 0.37978321392588305, "brier": 0.7410070756907946, "mean_conf": 0.7860332139258831, "confident_error_rate": 0.125, "coverage_at_0_9": 0.21875, "accuracy_at_0_9": 0.42857142857142855, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5425929023002614, "error_rate_at_0_9": 0.5714285714285714, "confidence_bias": 0.3797832139258831, "top_bins": { "0.9": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.47058823529411764, "confidence_cutoff": 0.7948651554495771 }, "0.8": { "coverage": 0.8125, "accuracy": 0.46153846153846156, "confidence_cutoff": 0.6413344954648915 } } }, "composition_final_exception": { "n": 32, "nll": 0.717299870586857, "acc": 0.65625, "ece": 0.30315644031633326, "brier": 0.5011811626132091, "mean_conf": 0.7164429849804473, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.48329043683112194, "error_rate_at_0_9": null, "confidence_bias": 0.06019298498044734, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7472231984138489 }, "0.8": { "coverage": 0.84375, "accuracy": 0.7037037037037037, "confidence_cutoff": 0.6238813624452733 } } }, "composition_final_negation": { "n": 32, "nll": 0.5322047320655654, "acc": 0.71875, "ece": 0.17014622114123046, "brier": 0.3473982110584136, "mean_conf": 0.6488173186822086, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.14944154536939344, "error_rate_at_0_9": null, "confidence_bias": -0.0699326813177914, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.8823529411764706, "confidence_cutoff": 0.6580008660555402 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8846153846153846, "confidence_cutoff": 0.5313978791236877 } } }, "contrastive_authorization": { "n": 40, "nll": 0.15131752514826352, "acc": 0.975, "ece": 0.11294708422717381, "brier": 0.05668953176003259, "mean_conf": 0.8674847096952952, "confident_error_rate": 0.0, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000625, "error_rate_at_0_9": 0.0, "confidence_bias": -0.10751529030470475, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9185473391305456 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.8089489099696603 } } }, "contrastive_deadline": { "n": 40, "nll": 1.0345684773009012, "acc": 0.45, "ece": 0.16219795036058843, "brier": 0.6620965862152072, "mean_conf": 0.5732693792864124, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5082073760882855, "error_rate_at_0_9": null, "confidence_bias": 0.1232693792864124, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5608630305721544 }, "0.8": { "coverage": 0.8, "accuracy": 0.40625, "confidence_cutoff": 0.4430796504020691 } }, "score_mae": 0.6340324900423988, "ranked_probability_score": 0.19847476556704904 }, "emotion": { "n": 80, "nll": 1.1265468044675306, "acc": 0.6, "ece": 0.11443615014249534, "brier": 0.5217501476585913, "mean_conf": 0.5179636746526439, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2125, "coverage_at_1pct_error": 0.2125, "aurc": 0.173451131532144, "error_rate_at_0_9": 0.0, "confidence_bias": -0.08203632534735605, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.4884979135693363 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.3357400992071569 } } }, "mmlu": { "n": 80, "nll": 1.1885759361017207, "acc": 0.5125, "ece": 0.10398958292861613, "brier": 0.6384140927539121, "mean_conf": 0.4705273931820525, "confident_error_rate": 0.0, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.39327722144608956, "error_rate_at_0_9": 0.0, "confidence_bias": -0.041972606817947455, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.65, "confidence_cutoff": 0.42177510104412175 }, "0.8": { "coverage": 0.8, "accuracy": 0.515625, "confidence_cutoff": 0.33762807660186517 } } }, "mmlu_pro": { "n": 200, "nll": 2.1558072536966413, "acc": 0.225, "ece": 0.07299126860376964, "brier": 0.8654828504835517, "mean_conf": 0.27562993573256056, "confident_error_rate": 0.0, "coverage_at_0_9": 0.005, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.02, "coverage_at_1pct_error": 0.02, "aurc": 0.6635123360876309, "error_rate_at_0_9": 0.0, "confidence_bias": 0.050629935732560555, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.21746287632371644 }, "0.8": { "coverage": 0.8, "accuracy": 0.25625, "confidence_cutoff": 0.1625811746572265 } } }, "paws": { "n": 80, "nll": 0.6944234783738545, "acc": 0.5875, "ece": 0.1262200499385823, "brier": 0.49963081595517106, "mean_conf": 0.6750599829681917, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3527140864591778, "error_rate_at_0_9": null, "confidence_bias": 0.08755998296819167, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.68075031042099 }, "0.8": { "coverage": 0.8, "accuracy": 0.59375, "confidence_cutoff": 0.6360945512028172 } } }, "qnli": { "n": 80, "nll": 0.4555073999657404, "acc": 0.8125, "ece": 0.0915795153688172, "brier": 0.2870652400868212, "mean_conf": 0.7690534034702379, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.875, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.09462801816327258, "error_rate_at_0_9": 0.125, "confidence_bias": -0.04344659652976213, "top_bins": { "0.9": { "n": 24, "errors": 3, "error_rate": 0.125 }, "0.95": { "n": 15, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.7596169114112854 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.6030593277847366 } } }, "sciq": { "n": 80, "nll": 0.18834000915162102, "acc": 0.9625, "ece": 0.08085216985910534, "brier": 0.09050740969997009, "mean_conf": 0.8816478301408945, "confident_error_rate": 0.0, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.7625, "aurc": 0.005139395903322137, "error_rate_at_0_9": 0.0, "confidence_bias": -0.08085216985910548, "top_bins": { "0.9": { "n": 54, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 45, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9593347088320329 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.792910008619996 } } }, "tweet_offensive": { "n": 80, "nll": 0.5194802188489087, "acc": 0.725, "ece": 0.04920898242311043, "brier": 0.34811439847415704, "mean_conf": 0.7606741796687964, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.13595768124624372, "error_rate_at_0_9": 0.0625, "confidence_bias": 0.03567417966879638, "top_bins": { "0.9": { "n": 16, "errors": 1, "error_rate": 0.0625 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.7611014133527665 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.6431154204343384 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.6939806799515269, "acc": 0.5, "ece": 0.019253725176406, "brier": 0.5008324790562729, "mean_conf": 0.519253725176406, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.019253725176406, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.5163091863301639 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5148934125900269 } } }, "unknowable_authorization": { "n": 10, "nll": 0.8722533704762607, "acc": 0.5, "ece": 0.25652105928649827, "brier": 0.6492987837359321, "mean_conf": 0.7565210592864983, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4726984126984126, "error_rate_at_0_9": null, "confidence_bias": 0.25652105928649827, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7624760752547373 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6944449336992224 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.010614667252794148, "acc": 1.0, "ece": 0.0105551277111825, "brier": 0.0002362723484120104, "mean_conf": 0.9894448722888175, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0105551277111825, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9910509466345773 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9867438874859482 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.18762674207375457, "acc": 1.0, "ece": 0.1630505554036273, "brier": 0.0785331260905723, "mean_conf": 0.8369494445963725, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.16305055540362745, "top_bins": { "0.9": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9289364218711853 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7272492863908198 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.0819812812391447, "acc": 0.4, "ece": 0.2179296430245531, "brier": 0.6983678640867412, "mean_conf": 0.55364404768203, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7146428571428571, "error_rate_at_0_9": null, "confidence_bias": 0.15364404768203, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5413114428520203 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4340435266494751 } }, "score_mae": 0.6230918518689158, "ranked_probability_score": 0.2015582690795208 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.24211831689569624, "acc": 0.9, "ece": 0.08309738728193819, "brier": 0.1504702562673398, "mean_conf": 0.9563502338155283, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": 0.05635023381552828, "top_bins": { "0.9": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9855445265994944 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9685846991808067 } }, "score_mae": 0.09716835579799854, "ranked_probability_score": 0.03528106133634211 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.02838703360000571, "acc": 1.0, "ece": 0.027635333708992738, "brier": 0.002268429428419828, "mean_conf": 0.9723646662910073, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.027635333708992738, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9916285715087849 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9531984667826858 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.4967078190499693, "acc": 0.8, "ece": 0.28676134017722765, "brier": 0.32609330633710903, "mean_conf": 0.691431930937867, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.11246031746031745, "error_rate_at_0_9": null, "confidence_bias": -0.10856806906213301, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7418128252029419 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6112156510353088 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.475859500379213, "acc": 1.0, "ece": 0.3731629842487789, "brier": 0.22978191865644892, "mean_conf": 0.6268370157512211, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": null, "confidence_bias": -0.3731629842487789, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.6530453964160264 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5752129979368151 } }, "score_mae": 0.35415910432790615, "ranked_probability_score": 0.06384588994201582 }, "unknowable_control_sla_response": { "n": 10, "nll": 0.03205922369840776, "acc": 1.0, "ece": 0.031276946991082744, "brier": 0.002423337198586589, "mean_conf": 0.9687230530089173, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.031276946991082744, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9820183094843828 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9579578388269809 } }, "score_mae": 0.019461998768093222, "ranked_probability_score": 0.0004910640933817507 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.024872545233687492, "acc": 1.0, "ece": 0.024419907698640442, "brier": 0.0015987643049556542, "mean_conf": 0.9755800923013596, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.024419907698640442, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9870816919645322 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9560908812669289 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.033222898291474914, "acc": 1.0, "ece": 0.03256888767009658, "brier": 0.001958310993865444, "mean_conf": 0.9674311123299034, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.03256888767009658, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9742770906196108 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9501113885943084 } }, "score_mae": 0.034394007249953874, "ranked_probability_score": 0.0006179846620626799 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.8650021967731402, "acc": 0.5, "ece": 0.17404689944827045, "brier": 0.534130946600855, "mean_conf": 0.5314912195576974, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.343968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.0314912195576974, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5749318174997514 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.45559284769882036 } }, "score_mae": 0.5136025117984403, "ranked_probability_score": 0.1447363472551048 }, "unknowable_deadline": { "n": 10, "nll": 1.1071854553806513, "acc": 0.2, "ece": 0.15279377127692456, "brier": 0.6726584130348836, "mean_conf": 0.3527937712769246, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.894325396825397, "error_rate_at_0_9": null, "confidence_bias": 0.15279377127692456, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3545035229475356 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.3461037679345539 } }, "score_mae": 0.7095588297047374, "ranked_probability_score": 0.2286381551067925 }, "unknowable_late_fee": { "n": 10, "nll": 1.1009222586644083, "acc": 0.3, "ece": 0.05528204508982232, "brier": 0.6684590952643721, "mean_conf": 0.3552820450898223, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6733333333333332, "error_rate_at_0_9": null, "confidence_bias": 0.05528204508982232, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.3528743386268616 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3416363793123847 } }, "score_mae": 0.6159571473108558, "ranked_probability_score": 0.2119415165063617 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.0743875752470067, "acc": 0.5, "ece": 0.12310781351525407, "brier": 0.6498611289876937, "mean_conf": 0.37689218648474593, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": -0.12310781351525407, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.37869871791299337 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3678526282310486 } } }, "unknowable_return_window": { "n": 10, "nll": 0.6959248786816741, "acc": 0.5, "ece": 0.018002226588528014, "brier": 0.5027695062876225, "mean_conf": 0.518002226588528, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5889682539682539, "error_rate_at_0_9": null, "confidence_bias": 0.018002226588528014, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5237695262569472 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5067899227142334 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.1115773013235066, "acc": 0.1, "ece": 0.2563719250278863, "brier": 0.6760369919101811, "mean_conf": 0.3563719250278863, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9788888888888889, "error_rate_at_0_9": null, "confidence_bias": 0.2563719250278863, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.3588156998157501 }, "0.8": { "coverage": 0.8, "accuracy": 0.0, "confidence_cutoff": 0.34798340046356324 } }, "score_mae": 0.718772458822063, "ranked_probability_score": 0.23420919749259364 }, "unknowable_sla_response": { "n": 10, "nll": 1.1113402899320053, "acc": 0.2, "ece": 0.1479866602175886, "brier": 0.6751827510461801, "mean_conf": 0.3479866602175886, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9349999999999999, "error_rate_at_0_9": null, "confidence_bias": 0.1479866602175886, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.0, "confidence_cutoff": 0.3522949814796448 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.341176410983587 } }, "score_mae": 0.5263728017533722, "ranked_probability_score": 0.19933626993753667 }, "unknowable_spend_threshold": { "n": 10, "nll": 1.1042525030071733, "acc": 0.5, "ece": 0.14743567287762616, "brier": 0.6699830116468511, "mean_conf": 0.35256432712237384, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": -0.14743567287762616, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.3511350348253538 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3478470548345761 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.0995896846945556, "acc": 0.4, "ece": 0.056070031766507655, "brier": 0.6671933530468324, "mean_conf": 0.34392996823349237, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5783333333333334, "error_rate_at_0_9": null, "confidence_bias": -0.056070031766507655, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.3414221304231247 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.340835133741192 } }, "score_mae": 0.8008344053973435, "ranked_probability_score": 0.24426572275994105 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.060261845877397, "acc": 0.5, "ece": 0.1469501009504352, "brier": 0.6414409180128442, "mean_conf": 0.3530498990495648, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.3981349206349207, "error_rate_at_0_9": null, "confidence_bias": -0.1469501009504352, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.35052138566970825 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.3451622231352992 } }, "score_mae": 0.4942307719058233, "ranked_probability_score": 0.18405752515186863 } }, "variants": { "clean": { "n": 1156, "nll": 0.954067979702682, "acc": 0.5899653979238755, "ece": 0.024056874289590406, "brier": 0.4981878554047032, "mean_conf": 0.6030969542803597, "confident_error_rate": 0.006920415224913495, "coverage_at_0_9": 0.1652249134948097, "accuracy_at_0_9": 0.9581151832460733, "coverage_at_5pct_error": 0.19550173010380623, "coverage_at_1pct_error": 0.1185121107266436, "aurc": 0.19696485131410457, "error_rate_at_0_9": 0.041884816753926704, "confidence_bias": 0.01313155635648422, "top_bins": { "0.9": { "n": 191, "errors": 8, "error_rate": 0.041884816753926704 }, "0.95": { "n": 122, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.78719723183391, "confidence_cutoff": 0.632773656629731 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.6778378378378378, "confidence_cutoff": 0.3490887388315421 } }, "score_mae": 0.5027333878046937, "ranked_probability_score": 0.1589298790994824 }, "none_absent": { "n": 36, "nll": 0.9102490391546016, "acc": 0.6111111111111112, "ece": 0.13730794500275045, "brier": 0.511483661848222, "mean_conf": 0.5584555794023928, "confident_error_rate": 0.0, "coverage_at_0_9": 0.027777777777777776, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05555555555555555, "coverage_at_1pct_error": 0.05555555555555555, "aurc": 0.31345116069280576, "error_rate_at_0_9": 0.0, "confidence_bias": -0.052655531708718395, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.524643298054909 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.6206896551724138, "confidence_cutoff": 0.4411063210707393 } } }, "none_present": { "n": 36, "nll": 1.4180436210236733, "acc": 0.4444444444444444, "ece": 0.21785299965250632, "brier": 0.6510539151417039, "mean_conf": 0.5328794141147639, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2222222222222222, "coverage_at_1pct_error": 0.2222222222222222, "aurc": 0.30348153066005734, "error_rate_at_0_9": null, "confidence_bias": 0.0884349696703195, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6111111111111112, "confidence_cutoff": 0.5056465883018322 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5517241379310345, "confidence_cutoff": 0.3732667076125099 } } }, "permuted": { "n": 36, "nll": 1.1203347443894875, "acc": 0.5833333333333334, "ece": 0.08938140601905017, "brier": 0.538097155621065, "mean_conf": 0.6092602344710492, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1111111111111111, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.1388888888888889, "coverage_at_1pct_error": 0.1388888888888889, "aurc": 0.2125980554928154, "error_rate_at_0_9": 0.0, "confidence_bias": 0.025926901137715785, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7777777777777778, "confidence_cutoff": 0.600199887159465 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.6551724137931034, "confidence_cutoff": 0.4275105233861428 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.07763010788821195, "flip_rate": 0.16666666666666666 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 0.9489348241911458, "acc": 0.6118546845124283, "ece": 0.025818167713742683, "brier": 0.4839082184775719, "mean_conf": 0.6222309763026364, "confident_error_rate": 0.0076481835564053535, "coverage_at_0_9": 0.1826003824091778, "accuracy_at_0_9": 0.9581151832460733, "coverage_at_5pct_error": 0.21606118546845124, "coverage_at_1pct_error": 0.13097514340344169, "aurc": 0.17821722891968061, "error_rate_at_0_9": 0.041884816753926704, "confidence_bias": 0.010376291790208048, "top_bins": { "0.9": { "n": 191, "errors": 8, "error_rate": 0.041884816753926704 }, "0.95": { "n": 122, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8183556405353728, "confidence_cutoff": 0.6650095781678212 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.7084826762246117, "confidence_cutoff": 0.3603672867152023 } }, "score_mae": 0.41780077899809037, "ranked_probability_score": 0.12404296786366242 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Qwen3.5-2B": { "panels": { "old_core": { "point": 0.5712425595238095, "exact_fraction": "3071/5376", "ci95": [ 0.535453869047619, 0.607627418154762 ] }, "v3_core": { "point": 0.39, "exact_fraction": "39/100", "ci95": [ 0.3733333333333334, 0.40625 ] }, "v4": { "point": 0.7375, "exact_fraction": "59/80", "ci95": [ 0.6928571428571428, 0.782292158018868 ] }, "v5": { "point": 0.7229166666666667, "exact_fraction": "347/480", "ci95": [ 0.6848579953997695, 0.760121271215623 ] }, "transfer_v9_test": { "point": 0.5630975143403442, "exact_fraction": "589/1046", "ci95": [ 0.5326182450604224, 0.5945186030548062 ] }, "original_four_panel_mean": { "point": 0.605414806547619, "exact_fraction": "325471/537600", "ci95": [ 0.5881936025503637, 0.6231347666599296 ] }, "weighted_mean": { "point": 0.5723998950081944, "exact_fraction": "26823117/46860800", "ci95": [ 0.5574004298169121, 0.587603872710617 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 103, "requested": 128 }, "boolean_constraints": { "correct": 24, "requested": 64 }, "dbpedia_14": { "correct": 104, "requested": 112 }, "natural_intents": { "correct": 62, "requested": 64 }, "option_carrier": { "correct": 9, "requested": 96 }, "ordinal_rubric": { "correct": 52, "requested": 64 }, "relational_composition": { "correct": 47, "requested": 96 }, "scoped_evidence": { "correct": 35, "requested": 96 }, "state_tracking": { "correct": 24, "requested": 96 }, "unknown_rejection": { "correct": 40, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 37, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 17, "requested": 80 }, "massive_en": { "correct": 85, "requested": 120 }, "massive_zh": { "correct": 89, "requested": 120 }, "multiset_reconciliation": { "correct": 22, "requested": 80 }, "ordinal_service_loss": { "correct": 16, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 22, "requested": 80 }, "temporal_exclusion": { "correct": 23, "requested": 80 }, "transaction_recovery": { "correct": 19, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 104, "requested": 160 }, "belebele_en": { "correct": 134, "requested": 160 }, "belebele_zh": { "correct": 130, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 68, "requested": 120 }, "squad2_answerability": { "correct": 99, "requested": 120 }, "snli": { "correct": 77, "requested": 120 }, "qasc": { "correct": 103, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 589, "accuracy": 0.5630975143403442, "full_probability_metrics": { "n": 1046, "nll": 1.2327016430228732, "acc": 0.5630975143403442, "ece": 0.12300515062522288, "brier": 0.5708125684679077, "mean_conf": 0.6861026649655669, "confident_error_rate": 0.0497131931166348, "coverage_at_0_9": 0.21510516252390058, "accuracy_at_0_9": 0.7688888888888888, "coverage_at_5pct_error": 0.11759082217973231, "coverage_at_1pct_error": 0.030592734225621414, "aurc": 0.2698851807680104, "error_rate_at_0_9": 0.2311111111111111, "confidence_bias": 0.12300515062522277, "top_bins": { "0.9": { "n": 225, "errors": 52, "error_rate": 0.2311111111111111 }, "0.95": { "n": 163, "errors": 32, "error_rate": 0.19631901840490798 }, "0.99": { "n": 69, "errors": 1, "error_rate": 0.014492753623188406 } }, "selective": { "0.5": { "coverage": 0.5162523900573613, "accuracy": 0.6907407407407408, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8068833652007649, "accuracy": 0.6409952606635071, "confidence_cutoff": 0.5312093496322632 } }, "score_mae": 0.6636140452293933, "ranked_probability_score": 0.2782415897800982 }, "covered_only_probability_metrics": { "n": 1046, "nll": 1.2327016430228732, "acc": 0.5630975143403442, "ece": 0.12300515062522288, "brier": 0.5708125684679077, "mean_conf": 0.6861026649655669, "confident_error_rate": 0.0497131931166348, "coverage_at_0_9": 0.21510516252390058, "accuracy_at_0_9": 0.7688888888888888, "coverage_at_5pct_error": 0.11759082217973231, "coverage_at_1pct_error": 0.030592734225621414, "aurc": 0.2698851807680104, "error_rate_at_0_9": 0.2311111111111111, "confidence_bias": 0.12300515062522277, "top_bins": { "0.9": { "n": 225, "errors": 52, "error_rate": 0.2311111111111111 }, "0.95": { "n": 163, "errors": 32, "error_rate": 0.19631901840490798 }, "0.99": { "n": 69, "errors": 1, "error_rate": 0.014492753623188406 } }, "selective": { "0.5": { "coverage": 0.5162523900573613, "accuracy": 0.6907407407407408, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8068833652007649, "accuracy": 0.6409952606635071, "confidence_cutoff": 0.5312093496322632 } }, "score_mae": 0.6636140452293933, "ranked_probability_score": 0.2782415897800982 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 11, "accuracy": 0.55, "full_probability_metrics": { "n": 20, "nll": 1.7862126355043544, "acc": 0.55, "ece": 0.19388488276953267, "brier": 0.6951362919831459, "mean_conf": 0.7438848827695328, "confident_error_rate": 0.05, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.26371453666732303, "error_rate_at_0_9": 0.2, "confidence_bias": 0.19388488276953275, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.7686943798472075 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6015144894104486 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.7862126355043544, "acc": 0.55, "ece": 0.19388488276953267, "brier": 0.6951362919831459, "mean_conf": 0.7438848827695328, "confident_error_rate": 0.05, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.26371453666732303, "error_rate_at_0_9": 0.2, "confidence_bias": 0.19388488276953275, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.7686943798472075 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6015144894104486 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 14, "accuracy": 0.7, "full_probability_metrics": { "n": 20, "nll": 0.6331344233704024, "acc": 0.7, "ece": 0.10089250057935717, "brier": 0.4404257196027067, "mean_conf": 0.637459222972393, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3118279965648386, "error_rate_at_0_9": null, "confidence_bias": -0.06254077702760696, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5621764957904816 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.6331344233704024, "acc": 0.7, "ece": 0.10089250057935717, "brier": 0.4404257196027067, "mean_conf": 0.637459222972393, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3118279965648386, "error_rate_at_0_9": null, "confidence_bias": -0.06254077702760696, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5621764957904816 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 13, "accuracy": 0.65, "full_probability_metrics": { "n": 20, "nll": 0.5589757006905984, "acc": 0.65, "ece": 0.09610208794474606, "brier": 0.3795811171622817, "mean_conf": 0.6639485664665699, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.16913384989623378, "error_rate_at_0_9": null, "confidence_bias": 0.013948566466569834, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6513548493385315 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5926665663719177 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5589757006905984, "acc": 0.65, "ece": 0.09610208794474606, "brier": 0.3795811171622817, "mean_conf": 0.6639485664665699, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.16913384989623378, "error_rate_at_0_9": null, "confidence_bias": 0.013948566466569834, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6513548493385315 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5926665663719177 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 7, "accuracy": 0.35, "full_probability_metrics": { "n": 20, "nll": 0.8943354779094845, "acc": 0.35, "ece": 0.39750848710536957, "brier": 0.6823503314583276, "mean_conf": 0.6701891958713532, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.6530745037556184, "error_rate_at_0_9": null, "confidence_bias": 0.3201891958713532, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.85, "accuracy": 0.35294117647058826, "confidence_cutoff": 0.5926666259765625 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.8943354779094845, "acc": 0.35, "ece": 0.39750848710536957, "brier": 0.6823503314583276, "mean_conf": 0.6701891958713532, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.6530745037556184, "error_rate_at_0_9": null, "confidence_bias": 0.3201891958713532, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.85, "accuracy": 0.35294117647058826, "confidence_cutoff": 0.5926666259765625 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 20, "accuracy": 0.625, "full_probability_metrics": { "n": 32, "nll": 0.6618922668537142, "acc": 0.625, "ece": 0.14041897832029454, "brier": 0.46968893830708225, "mean_conf": 0.6462984525836305, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.3406844839689173, "error_rate_at_0_9": null, "confidence_bias": 0.0212984525836305, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 0.5789473684210527, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.6224593334657152 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.6618922668537142, "acc": 0.625, "ece": 0.14041897832029454, "brier": 0.46968893830708225, "mean_conf": 0.6462984525836305, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.3406844839689173, "error_rate_at_0_9": null, "confidence_bias": 0.0212984525836305, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 0.5789473684210527, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.6224593334657152 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 13, "accuracy": 0.40625, "full_probability_metrics": { "n": 32, "nll": 0.7153033490192675, "acc": 0.40625, "ece": 0.21517161386448355, "brier": 0.522471670249089, "mean_conf": 0.6111393350254657, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.46973315762862006, "error_rate_at_0_9": null, "confidence_bias": 0.2048893350254657, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5882352941176471, "confidence_cutoff": 0.6224593334657152 }, "0.8": { "coverage": 0.9375, "accuracy": 0.43333333333333335, "confidence_cutoff": 0.5312093775743617 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.7153033490192675, "acc": 0.40625, "ece": 0.21517161386448355, "brier": 0.522471670249089, "mean_conf": 0.6111393350254657, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.46973315762862006, "error_rate_at_0_9": null, "confidence_bias": 0.2048893350254657, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5882352941176471, "confidence_cutoff": 0.6224593334657152 }, "0.8": { "coverage": 0.9375, "accuracy": 0.43333333333333335, "confidence_cutoff": 0.5312093775743617 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 15, "accuracy": 0.46875, "full_probability_metrics": { "n": 32, "nll": 0.83213993101933, "acc": 0.46875, "ece": 0.2163413860392832, "brier": 0.6165937780295263, "mean_conf": 0.6509422762795869, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6491686379227497, "error_rate_at_0_9": null, "confidence_bias": 0.1821922762795869, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 0.47368421052631576, "confidence_cutoff": 0.6224593334657152 }, "0.8": { "coverage": 0.8125, "accuracy": 0.4230769230769231, "confidence_cutoff": 0.5926665906508809 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.83213993101933, "acc": 0.46875, "ece": 0.2163413860392832, "brier": 0.6165937780295263, "mean_conf": 0.6509422762795869, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6491686379227497, "error_rate_at_0_9": null, "confidence_bias": 0.1821922762795869, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 0.47368421052631576, "confidence_cutoff": 0.6224593334657152 }, "0.8": { "coverage": 0.8125, "accuracy": 0.4230769230769231, "confidence_cutoff": 0.5926665906508809 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 25, "accuracy": 0.625, "full_probability_metrics": { "n": 40, "nll": 0.6593153134313047, "acc": 0.625, "ece": 0.08166578561067585, "brier": 0.4667643843708243, "mean_conf": 0.5433342143893242, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.225, "coverage_at_1pct_error": 0.225, "aurc": 0.26776101532567054, "error_rate_at_0_9": null, "confidence_bias": -0.08166578561067583, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.675, "accuracy": 0.5925925925925926, "confidence_cutoff": 0.531209409236908 }, "0.8": { "coverage": 1.0, "accuracy": 0.625, "confidence_cutoff": 0.5 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.6593153134313047, "acc": 0.625, "ece": 0.08166578561067585, "brier": 0.4667643843708243, "mean_conf": 0.5433342143893242, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.225, "coverage_at_1pct_error": 0.225, "aurc": 0.26776101532567054, "error_rate_at_0_9": null, "confidence_bias": -0.08166578561067583, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.675, "accuracy": 0.5925925925925926, "confidence_cutoff": 0.531209409236908 }, "0.8": { "coverage": 1.0, "accuracy": 0.625, "confidence_cutoff": 0.5 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 10, "accuracy": 0.25, "full_probability_metrics": { "n": 40, "nll": 3.286383395467695, "acc": 0.25, "ece": 0.7053335182903989, "brier": 1.3877339965960604, "mean_conf": 0.9553335182903989, "confident_error_rate": 0.75, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7027467460002794, "error_rate_at_0_9": 0.75, "confidence_bias": 0.7053335182903989, "top_bins": { "0.9": { "n": 40, "errors": 30, "error_rate": 0.75 }, "0.95": { "n": 28, "errors": 20, "error_rate": 0.7142857142857143 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.2727272727272727, "confidence_cutoff": 0.9569650579822465 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.945548670444282 } }, "score_mae": 0.7458722048324027, "ranked_probability_score": 0.35191218221250375 }, "covered_only_probability_metrics": { "n": 40, "nll": 3.286383395467695, "acc": 0.25, "ece": 0.7053335182903989, "brier": 1.3877339965960604, "mean_conf": 0.9553335182903989, "confident_error_rate": 0.75, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7027467460002794, "error_rate_at_0_9": 0.75, "confidence_bias": 0.7053335182903989, "top_bins": { "0.9": { "n": 40, "errors": 30, "error_rate": 0.75 }, "0.95": { "n": 28, "errors": 20, "error_rate": 0.7142857142857143 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.2727272727272727, "confidence_cutoff": 0.9569650579822465 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.945548670444282 } }, "score_mae": 0.7458722048324027, "ranked_probability_score": 0.35191218221250375 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 54, "accuracy": 0.675, "full_probability_metrics": { "n": 80, "nll": 1.1245782403213567, "acc": 0.675, "ece": 0.1627690131291551, "brier": 0.4951019737328922, "mean_conf": 0.8318246605638224, "confident_error_rate": 0.1, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 0.8181818181818182, "coverage_at_5pct_error": 0.325, "coverage_at_1pct_error": 0.3, "aurc": 0.1474651102262961, "error_rate_at_0_9": 0.18181818181818182, "confidence_bias": 0.15682466056382238, "top_bins": { "0.9": { "n": 44, "errors": 8, "error_rate": 0.18181818181818182 }, "0.95": { "n": 32, "errors": 4, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.9245011200082578 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6621073781398983 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.1245782403213567, "acc": 0.675, "ece": 0.1627690131291551, "brier": 0.4951019737328922, "mean_conf": 0.8318246605638224, "confident_error_rate": 0.1, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 0.8181818181818182, "coverage_at_5pct_error": 0.325, "coverage_at_1pct_error": 0.3, "aurc": 0.1474651102262961, "error_rate_at_0_9": 0.18181818181818182, "confidence_bias": 0.15682466056382238, "top_bins": { "0.9": { "n": 44, "errors": 8, "error_rate": 0.18181818181818182 }, "0.95": { "n": 32, "errors": 4, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.9245011200082578 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6621073781398983 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 43, "accuracy": 0.5375, "full_probability_metrics": { "n": 80, "nll": 1.0384180174537831, "acc": 0.5375, "ece": 0.17784433133642866, "brier": 0.5325961245420057, "mean_conf": 0.6733467293898716, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.2125, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.1875, "aurc": 0.19166611346696732, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.13584672938987163, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6864975264622741 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.4595814217271822 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.0384180174537831, "acc": 0.5375, "ece": 0.17784433133642866, "brier": 0.5325961245420057, "mean_conf": 0.6733467293898716, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.2125, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.1875, "aurc": 0.19166611346696732, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.13584672938987163, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6864975264622741 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.4595814217271822 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 56, "accuracy": 0.28, "full_probability_metrics": { "n": 200, "nll": 2.7325578057868087, "acc": 0.28, "ece": 0.2510208181758569, "brier": 0.9069599340991101, "mean_conf": 0.5310208181758569, "confident_error_rate": 0.015, "coverage_at_0_9": 0.09, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.045, "coverage_at_1pct_error": 0.045, "aurc": 0.5176505235238105, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.25102081817585686, "top_bins": { "0.9": { "n": 18, "errors": 3, "error_rate": 0.16666666666666666 }, "0.95": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.43, "confidence_cutoff": 0.4757429015413205 }, "0.8": { "coverage": 0.8, "accuracy": 0.325, "confidence_cutoff": 0.3472072904424558 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.7325578057868087, "acc": 0.28, "ece": 0.2510208181758569, "brier": 0.9069599340991101, "mean_conf": 0.5310208181758569, "confident_error_rate": 0.015, "coverage_at_0_9": 0.09, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.045, "coverage_at_1pct_error": 0.045, "aurc": 0.5176505235238105, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.25102081817585686, "top_bins": { "0.9": { "n": 18, "errors": 3, "error_rate": 0.16666666666666666 }, "0.95": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.43, "confidence_cutoff": 0.4757429015413205 }, "0.8": { "coverage": 0.8, "accuracy": 0.325, "confidence_cutoff": 0.3472072904424558 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 56, "accuracy": 0.7, "full_probability_metrics": { "n": 80, "nll": 0.5677938717923453, "acc": 0.7, "ece": 0.03018935378640889, "brier": 0.38503109456130885, "mean_conf": 0.7081807853654027, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.20397905911269057, "error_rate_at_0_9": null, "confidence_bias": 0.008180785365402787, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.8181818181818182, "confidence_cutoff": 0.7057850360870361 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7384615384615385, "confidence_cutoff": 0.5926665663719177 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.5677938717923453, "acc": 0.7, "ece": 0.03018935378640889, "brier": 0.38503109456130885, "mean_conf": 0.7081807853654027, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.20397905911269057, "error_rate_at_0_9": null, "confidence_bias": 0.008180785365402787, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.8181818181818182, "confidence_cutoff": 0.7057850360870361 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7384615384615385, "confidence_cutoff": 0.5926665663719177 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 51, "accuracy": 0.6375, "full_probability_metrics": { "n": 80, "nll": 0.6399010127286031, "acc": 0.6375, "ece": 0.0663282605819404, "brier": 0.44992281619168484, "mean_conf": 0.5870338861830533, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0875, "coverage_at_1pct_error": 0.0875, "aurc": 0.27883439689308287, "error_rate_at_0_9": null, "confidence_bias": -0.050466113816946656, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5875, "accuracy": 0.6808510638297872, "confidence_cutoff": 0.562176525592804 }, "0.8": { "coverage": 0.875, "accuracy": 0.6571428571428571, "confidence_cutoff": 0.5312093496322632 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.6399010127286031, "acc": 0.6375, "ece": 0.0663282605819404, "brier": 0.44992281619168484, "mean_conf": 0.5870338861830533, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0875, "coverage_at_1pct_error": 0.0875, "aurc": 0.27883439689308287, "error_rate_at_0_9": null, "confidence_bias": -0.050466113816946656, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5875, "accuracy": 0.6808510638297872, "confidence_cutoff": 0.562176525592804 }, "0.8": { "coverage": 0.875, "accuracy": 0.6571428571428571, "confidence_cutoff": 0.5312093496322632 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 77, "accuracy": 0.9625, "full_probability_metrics": { "n": 80, "nll": 0.15196097469307052, "acc": 0.9625, "ece": 0.027357528447452366, "brier": 0.06394141718449492, "mean_conf": 0.9656630985306636, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.9125, "accuracy_at_0_9": 0.9863013698630136, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.3875, "aurc": 0.013004205907125995, "error_rate_at_0_9": 0.0136986301369863, "confidence_bias": 0.003163098530663544, "top_bins": { "0.9": { "n": 73, "errors": 1, "error_rate": 0.0136986301369863 }, "0.95": { "n": 72, "errors": 1, "error_rate": 0.013888888888888888 }, "0.99": { "n": 57, "errors": 1, "error_rate": 0.017543859649122806 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9968472076726742 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9809843211553804 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.15196097469307052, "acc": 0.9625, "ece": 0.027357528447452366, "brier": 0.06394141718449492, "mean_conf": 0.9656630985306636, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.9125, "accuracy_at_0_9": 0.9863013698630136, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.3875, "aurc": 0.013004205907125995, "error_rate_at_0_9": 0.0136986301369863, "confidence_bias": 0.003163098530663544, "top_bins": { "0.9": { "n": 73, "errors": 1, "error_rate": 0.0136986301369863 }, "0.95": { "n": 72, "errors": 1, "error_rate": 0.013888888888888888 }, "0.99": { "n": 57, "errors": 1, "error_rate": 0.017543859649122806 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9968472076726742 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9809843211553804 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 67, "accuracy": 0.8375, "full_probability_metrics": { "n": 80, "nll": 0.4128277789148231, "acc": 0.8375, "ece": 0.11258694771677256, "brier": 0.2553380112280364, "mean_conf": 0.7249130522832274, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.2875, "aurc": 0.06034373794472666, "error_rate_at_0_9": 0.0, "confidence_bias": -0.11258694771677258, "top_bins": { "0.9": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.575, "accuracy": 0.9347826086956522, "confidence_cutoff": 0.7310585677623749 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.5926665663719177 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4128277789148231, "acc": 0.8375, "ece": 0.11258694771677256, "brier": 0.2553380112280364, "mean_conf": 0.7249130522832274, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.2875, "aurc": 0.06034373794472666, "error_rate_at_0_9": 0.0, "confidence_bias": -0.11258694771677258, "top_bins": { "0.9": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.575, "accuracy": 0.9347826086956522, "confidence_cutoff": 0.7310585677623749 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.5926665663719177 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.5737342276934214, "acc": 1.0, "ece": 0.43513118624687197, "brier": 0.3818724823450289, "mean_conf": 0.564868813753128, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": null, "confidence_bias": -0.43513118624687197, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.5926665663719177 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5312093496322632 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5737342276934214, "acc": 1.0, "ece": 0.43513118624687197, "brier": 0.3818724823450289, "mean_conf": 0.564868813753128, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": null, "confidence_bias": -0.43513118624687197, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.5926665663719177 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5312093496322632 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.6683156254987609, "acc": 0.5, "ece": 0.046549105644226096, "brier": 0.47556249999673667, "mean_conf": 0.5465491056442261, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.31, "error_rate_at_0_9": null, "confidence_bias": 0.046549105644226096, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.562176525592804 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.531209409236908 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.6683156254987609, "acc": 0.5, "ece": 0.046549105644226096, "brier": 0.47556249999673667, "mean_conf": 0.5465491056442261, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.31, "error_rate_at_0_9": null, "confidence_bias": 0.046549105644226096, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.562176525592804 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.531209409236908 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 2.9213554199896956, "acc": 0.3, "ece": 0.6547563158153, "brier": 1.3000704884490109, "mean_conf": 0.9547563158153001, "confident_error_rate": 0.7, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.3, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6690476190476191, "error_rate_at_0_9": 0.7, "confidence_bias": 0.6547563158153, "top_bins": { "0.9": { "n": 10, "errors": 7, "error_rate": 0.7 }, "0.95": { "n": 8, "errors": 6, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.9572610626650837 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.9505933323983569 } }, "score_mae": 0.7023828513699522, "ranked_probability_score": 0.33006472716063306 }, "covered_only_probability_metrics": { "n": 10, "nll": 2.9213554199896956, "acc": 0.3, "ece": 0.6547563158153, "brier": 1.3000704884490109, "mean_conf": 0.9547563158153001, "confident_error_rate": 0.7, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.3, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6690476190476191, "error_rate_at_0_9": 0.7, "confidence_bias": 0.6547563158153, "top_bins": { "0.9": { "n": 10, "errors": 7, "error_rate": 0.7 }, "0.95": { "n": 8, "errors": 6, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.9572610626650837 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.9505933323983569 } }, "score_mae": 0.7023828513699522, "ranked_probability_score": 0.33006472716063306 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.7330163732489625, "acc": 0.7, "ece": 0.1482540632041076, "brier": 0.4451413587721745, "mean_conf": 0.7021622002406106, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.17857142857142855, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0021622002406106455, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7362475450057354 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5883927323356541 } }, "score_mae": 0.4211218097506838, "ranked_probability_score": 0.11729643003023708 }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7330163732489625, "acc": 0.7, "ece": 0.1482540632041076, "brier": 0.4451413587721745, "mean_conf": 0.7021622002406106, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.17857142857142855, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0021622002406106455, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7362475450057354 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5883927323356541 } }, "score_mae": 0.4211218097506838, "ranked_probability_score": 0.11729643003023708 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 2, "accuracy": 0.2, "full_probability_metrics": { "n": 10, "nll": 1.1096857314005688, "acc": 0.2, "ece": 0.3814912517804449, "brier": 0.7079188069347107, "mean_conf": 0.5814912517804449, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5642063492063492, "error_rate_at_0_9": null, "confidence_bias": 0.3814912517804449, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.598909419807511 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5224289142361607 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.1096857314005688, "acc": 0.2, "ece": 0.3814912517804449, "brier": 0.7079188069347107, "mean_conf": 0.5814912517804449, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5642063492063492, "error_rate_at_0_9": null, "confidence_bias": 0.3814912517804449, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.598909419807511 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5224289142361607 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.7456403458065914, "acc": 0.5, "ece": 0.16947978138923647, "brier": 0.5499934526273271, "mean_conf": 0.6250797569751739, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6214285714285714, "error_rate_at_0_9": null, "confidence_bias": 0.12507975697517393, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.7, "accuracy": 0.42857142857142855, "confidence_cutoff": 0.622459352016449 }, "0.8": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.5926666259765625 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7456403458065914, "acc": 0.5, "ece": 0.16947978138923647, "brier": 0.5499934526273271, "mean_conf": 0.6250797569751739, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6214285714285714, "error_rate_at_0_9": null, "confidence_bias": 0.12507975697517393, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.7, "accuracy": 0.42857142857142855, "confidence_cutoff": 0.622459352016449 }, "0.8": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.5926666259765625 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.459165154023861, "acc": 0.3, "ece": 0.38370459231587195, "brier": 0.8650992018323784, "mean_conf": 0.6837045923158721, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.37666666666666665, "error_rate_at_0_9": null, "confidence_bias": 0.3837045923158721, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7158687604089267 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6573608917784425 } }, "score_mae": 0.7480355975722548, "ranked_probability_score": 0.2686501397214868 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.459165154023861, "acc": 0.3, "ece": 0.38370459231587195, "brier": 0.8650992018323784, "mean_conf": 0.6837045923158721, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.37666666666666665, "error_rate_at_0_9": null, "confidence_bias": 0.3837045923158721, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7158687604089267 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6573608917784425 } }, "score_mae": 0.7480355975722548, "ranked_probability_score": 0.2686501397214868 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.2945180507825598, "acc": 0.5, "ece": 0.3660619498101235, "brier": 0.7955644942063342, "mean_conf": 0.8660619498101235, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.37480158730158725, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3660619498101235, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.8661572595285553 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.858034541554363 } }, "score_mae": 0.49130664669541435, "ranked_probability_score": 0.21094552795222593 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.2945180507825598, "acc": 0.5, "ece": 0.3660619498101235, "brier": 0.7955644942063342, "mean_conf": 0.8660619498101235, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.37480158730158725, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3660619498101235, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.8661572595285553 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.858034541554363 } }, "score_mae": 0.49130664669541435, "ranked_probability_score": 0.21094552795222593 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 10, "accuracy": 1.0, "full_probability_metrics": { "n": 10, "nll": 0.18369367567457523, "acc": 1.0, "ece": 0.1604828644401017, "brier": 0.0722314247265077, "mean_conf": 0.8395171355598983, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.16048286444010174, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8791032774305659 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7745119146570109 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.18369367567457523, "acc": 1.0, "ece": 0.1604828644401017, "brier": 0.0722314247265077, "mean_conf": 0.8395171355598983, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.16048286444010174, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8791032774305659 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7745119146570109 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 2, "accuracy": 0.2, "full_probability_metrics": { "n": 10, "nll": 1.301242281351381, "acc": 0.2, "ece": 0.5072509144820883, "brier": 0.8680037613035105, "mean_conf": 0.7072509144820883, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5975396825396825, "error_rate_at_0_9": null, "confidence_bias": 0.5072509144820883, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7678657998998701 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.554495151679999 } }, "score_mae": 0.6802767949532421, "ranked_probability_score": 0.2402083862014263 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.301242281351381, "acc": 0.2, "ece": 0.5072509144820883, "brier": 0.8680037613035105, "mean_conf": 0.7072509144820883, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5975396825396825, "error_rate_at_0_9": null, "confidence_bias": 0.5072509144820883, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7678657998998701 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.554495151679999 } }, "score_mae": 0.6802767949532421, "ranked_probability_score": 0.2402083862014263 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.2172446311530223, "acc": 0.5, "ece": 0.21492532874165532, "brier": 0.6898920276891446, "mean_conf": 0.614331785653604, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5393650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.11433178565360402, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6182967173846468 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5851253779958877 } }, "score_mae": 0.6095279326227744, "ranked_probability_score": 0.20760195788495833 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.2172446311530223, "acc": 0.5, "ece": 0.21492532874165532, "brier": 0.6898920276891446, "mean_conf": 0.614331785653604, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5393650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.11433178565360402, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6182967173846468 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5851253779958877 } }, "score_mae": 0.6095279326227744, "ranked_probability_score": 0.20760195788495833 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.5538888888888889, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.5555555555555556, "flip_rate": 0.3055555555555556, "covered_only_flip_rate": 0.3055555555555556, "half_l1": 0.2163438723610137 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.6518942289023482, "share_at_0_9": 0.09090909090909091, "normalized_entropy": 0.7370137851796369 }, "covered_only_confidence_metrics": { "mean_max_p": 0.6518942289023482, "share_at_0_9": 0.09090909090909091, "normalized_entropy": 0.7370137851796369 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 57, "accuracy": 0.5181818181818182, "full_probability_metrics": { "n": 110, "nll": 1.1097828651475816, "acc": 0.5181818181818182, "ece": 0.18065919044671863, "brier": 0.6501227271711696, "mean_conf": 0.6987067110936791, "confident_error_rate": 0.07272727272727272, "coverage_at_0_9": 0.15454545454545454, "accuracy_at_0_9": 0.5294117647058824, "coverage_at_5pct_error": 0.00909090909090909, "coverage_at_1pct_error": 0.00909090909090909, "aurc": 0.47269702863847524, "error_rate_at_0_9": 0.47058823529411764, "confidence_bias": 0.1805248929118609, "top_bins": { "0.9": { "n": 17, "errors": 8, "error_rate": 0.47058823529411764 }, "0.95": { "n": 8, "errors": 6, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5181818181818182, "accuracy": 0.5087719298245614, "confidence_cutoff": 0.6513549089431763 }, "0.8": { "coverage": 0.8272727272727273, "accuracy": 0.5604395604395604, "confidence_cutoff": 0.562176525592804 } }, "score_mae": 0.6087752721607202, "ranked_probability_score": 0.2291278614918279 }, "covered_only_probability_metrics": { "n": 110, "nll": 1.1097828651475816, "acc": 0.5181818181818182, "ece": 0.18065919044671863, "brier": 0.6501227271711696, "mean_conf": 0.6987067110936791, "confident_error_rate": 0.07272727272727272, "coverage_at_0_9": 0.15454545454545454, "accuracy_at_0_9": 0.5294117647058824, "coverage_at_5pct_error": 0.00909090909090909, "coverage_at_1pct_error": 0.00909090909090909, "aurc": 0.47269702863847524, "error_rate_at_0_9": 0.47058823529411764, "confidence_bias": 0.1805248929118609, "top_bins": { "0.9": { "n": 17, "errors": 8, "error_rate": 0.47058823529411764 }, "0.95": { "n": 8, "errors": 6, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5181818181818182, "accuracy": 0.5087719298245614, "confidence_cutoff": 0.6513549089431763 }, "0.8": { "coverage": 0.8272727272727273, "accuracy": 0.5604395604395604, "confidence_cutoff": 0.562176525592804 } }, "score_mae": 0.6087752721607202, "ranked_probability_score": 0.2291278614918279 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.04681248219133087 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -1.0704941374659387, "paired_flip": { "pairs": 64, "flip_rate": 0.171875, "both_correct_rate": 0.109375, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.5416666666666666 }, "unknowable": { "n": 110, "mean_max_p": 0.6518942289023482, "share_at_0_9": 0.09090909090909091, "control_mean_max_p": 0.6987067110936791, "control_share_at_0_9": 0.15454545454545454, "control_acc": 0.5181818181818182, "paired_confidence_drop": 0.04681248219133087, "share_less_confident_than_control": 0.5636363636363636 }, "clean": { "n": 1046, "nll": 1.2327016430228732, "acc": 0.5630975143403442, "ece": 0.12300515062522288, "brier": 0.5708125684679077, "mean_conf": 0.6861026649655669, "confident_error_rate": 0.0497131931166348, "coverage_at_0_9": 0.21510516252390058, "accuracy_at_0_9": 0.7688888888888888, "coverage_at_5pct_error": 0.11759082217973231, "coverage_at_1pct_error": 0.030592734225621414, "aurc": 0.2698851807680104, "error_rate_at_0_9": 0.2311111111111111, "confidence_bias": 0.12300515062522277, "top_bins": { "0.9": { "n": 225, "errors": 52, "error_rate": 0.2311111111111111 }, "0.95": { "n": 163, "errors": 32, "error_rate": 0.19631901840490798 }, "0.99": { "n": 69, "errors": 1, "error_rate": 0.014492753623188406 } }, "selective": { "0.5": { "coverage": 0.5162523900573613, "accuracy": 0.6907407407407408, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8068833652007649, "accuracy": 0.6409952606635071, "confidence_cutoff": 0.5312093496322632 } }, "score_mae": 0.6636140452293933, "ranked_probability_score": 0.2782415897800982 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.7862126355043544, "acc": 0.55, "ece": 0.19388488276953267, "brier": 0.6951362919831459, "mean_conf": 0.7438848827695328, "confident_error_rate": 0.05, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.26371453666732303, "error_rate_at_0_9": 0.2, "confidence_bias": 0.19388488276953275, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.7686943798472075 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6015144894104486 } } }, "buried_paws": { "n": 20, "nll": 0.6331344233704024, "acc": 0.7, "ece": 0.10089250057935717, "brier": 0.4404257196027067, "mean_conf": 0.637459222972393, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3118279965648386, "error_rate_at_0_9": null, "confidence_bias": -0.06254077702760696, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5621764957904816 } } }, "buried_qnli": { "n": 20, "nll": 0.5589757006905984, "acc": 0.65, "ece": 0.09610208794474606, "brier": 0.3795811171622817, "mean_conf": 0.6639485664665699, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.25, "coverage_at_1pct_error": 0.25, "aurc": 0.16913384989623378, "error_rate_at_0_9": null, "confidence_bias": 0.013948566466569834, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6513548493385315 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5926665663719177 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.8943354779094845, "acc": 0.35, "ece": 0.39750848710536957, "brier": 0.6823503314583276, "mean_conf": 0.6701891958713532, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.6530745037556184, "error_rate_at_0_9": null, "confidence_bias": 0.3201891958713532, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6791787147521973 }, "0.8": { "coverage": 0.85, "accuracy": 0.35294117647058826, "confidence_cutoff": 0.5926666259765625 } } }, "composition_final_combination": { "n": 32, "nll": 0.6618922668537142, "acc": 0.625, "ece": 0.14041897832029454, "brier": 0.46968893830708225, "mean_conf": 0.6462984525836305, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.3406844839689173, "error_rate_at_0_9": null, "confidence_bias": 0.0212984525836305, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 0.5789473684210527, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.6224593334657152 } } }, "composition_final_exception": { "n": 32, "nll": 0.7153033490192675, "acc": 0.40625, "ece": 0.21517161386448355, "brier": 0.522471670249089, "mean_conf": 0.6111393350254657, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.46973315762862006, "error_rate_at_0_9": null, "confidence_bias": 0.2048893350254657, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5882352941176471, "confidence_cutoff": 0.6224593334657152 }, "0.8": { "coverage": 0.9375, "accuracy": 0.43333333333333335, "confidence_cutoff": 0.5312093775743617 } } }, "composition_final_negation": { "n": 32, "nll": 0.83213993101933, "acc": 0.46875, "ece": 0.2163413860392832, "brier": 0.6165937780295263, "mean_conf": 0.6509422762795869, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6491686379227497, "error_rate_at_0_9": null, "confidence_bias": 0.1821922762795869, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 0.47368421052631576, "confidence_cutoff": 0.6224593334657152 }, "0.8": { "coverage": 0.8125, "accuracy": 0.4230769230769231, "confidence_cutoff": 0.5926665906508809 } } }, "contrastive_authorization": { "n": 40, "nll": 0.6593153134313047, "acc": 0.625, "ece": 0.08166578561067585, "brier": 0.4667643843708243, "mean_conf": 0.5433342143893242, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.225, "coverage_at_1pct_error": 0.225, "aurc": 0.26776101532567054, "error_rate_at_0_9": null, "confidence_bias": -0.08166578561067583, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.675, "accuracy": 0.5925925925925926, "confidence_cutoff": 0.531209409236908 }, "0.8": { "coverage": 1.0, "accuracy": 0.625, "confidence_cutoff": 0.5 } } }, "contrastive_deadline": { "n": 40, "nll": 3.286383395467695, "acc": 0.25, "ece": 0.7053335182903989, "brier": 1.3877339965960604, "mean_conf": 0.9553335182903989, "confident_error_rate": 0.75, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7027467460002794, "error_rate_at_0_9": 0.75, "confidence_bias": 0.7053335182903989, "top_bins": { "0.9": { "n": 40, "errors": 30, "error_rate": 0.75 }, "0.95": { "n": 28, "errors": 20, "error_rate": 0.7142857142857143 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.2727272727272727, "confidence_cutoff": 0.9569650579822465 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.945548670444282 } }, "score_mae": 0.7458722048324027, "ranked_probability_score": 0.35191218221250375 }, "emotion": { "n": 80, "nll": 1.1245782403213567, "acc": 0.675, "ece": 0.1627690131291551, "brier": 0.4951019737328922, "mean_conf": 0.8318246605638224, "confident_error_rate": 0.1, "coverage_at_0_9": 0.55, "accuracy_at_0_9": 0.8181818181818182, "coverage_at_5pct_error": 0.325, "coverage_at_1pct_error": 0.3, "aurc": 0.1474651102262961, "error_rate_at_0_9": 0.18181818181818182, "confidence_bias": 0.15682466056382238, "top_bins": { "0.9": { "n": 44, "errors": 8, "error_rate": 0.18181818181818182 }, "0.95": { "n": 32, "errors": 4, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.9245011200082578 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.6621073781398983 } } }, "mmlu": { "n": 80, "nll": 1.0384180174537831, "acc": 0.5375, "ece": 0.17784433133642866, "brier": 0.5325961245420057, "mean_conf": 0.6733467293898716, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.2125, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.1875, "aurc": 0.19166611346696732, "error_rate_at_0_9": 0.058823529411764705, "confidence_bias": 0.13584672938987163, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.6864975264622741 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.4595814217271822 } } }, "mmlu_pro": { "n": 200, "nll": 2.7325578057868087, "acc": 0.28, "ece": 0.2510208181758569, "brier": 0.9069599340991101, "mean_conf": 0.5310208181758569, "confident_error_rate": 0.015, "coverage_at_0_9": 0.09, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.045, "coverage_at_1pct_error": 0.045, "aurc": 0.5176505235238105, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.25102081817585686, "top_bins": { "0.9": { "n": 18, "errors": 3, "error_rate": 0.16666666666666666 }, "0.95": { "n": 13, "errors": 1, "error_rate": 0.07692307692307693 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.43, "confidence_cutoff": 0.4757429015413205 }, "0.8": { "coverage": 0.8, "accuracy": 0.325, "confidence_cutoff": 0.3472072904424558 } } }, "paws": { "n": 80, "nll": 0.5677938717923453, "acc": 0.7, "ece": 0.03018935378640889, "brier": 0.38503109456130885, "mean_conf": 0.7081807853654027, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.20397905911269057, "error_rate_at_0_9": null, "confidence_bias": 0.008180785365402787, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.8181818181818182, "confidence_cutoff": 0.7057850360870361 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7384615384615385, "confidence_cutoff": 0.5926665663719177 } } }, "qnli": { "n": 80, "nll": 0.6399010127286031, "acc": 0.6375, "ece": 0.0663282605819404, "brier": 0.44992281619168484, "mean_conf": 0.5870338861830533, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0875, "coverage_at_1pct_error": 0.0875, "aurc": 0.27883439689308287, "error_rate_at_0_9": null, "confidence_bias": -0.050466113816946656, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5875, "accuracy": 0.6808510638297872, "confidence_cutoff": 0.562176525592804 }, "0.8": { "coverage": 0.875, "accuracy": 0.6571428571428571, "confidence_cutoff": 0.5312093496322632 } } }, "sciq": { "n": 80, "nll": 0.15196097469307052, "acc": 0.9625, "ece": 0.027357528447452366, "brier": 0.06394141718449492, "mean_conf": 0.9656630985306636, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.9125, "accuracy_at_0_9": 0.9863013698630136, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.3875, "aurc": 0.013004205907125995, "error_rate_at_0_9": 0.0136986301369863, "confidence_bias": 0.003163098530663544, "top_bins": { "0.9": { "n": 73, "errors": 1, "error_rate": 0.0136986301369863 }, "0.95": { "n": 72, "errors": 1, "error_rate": 0.013888888888888888 }, "0.99": { "n": 57, "errors": 1, "error_rate": 0.017543859649122806 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9968472076726742 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9809843211553804 } } }, "tweet_offensive": { "n": 80, "nll": 0.4128277789148231, "acc": 0.8375, "ece": 0.11258694771677256, "brier": 0.2553380112280364, "mean_conf": 0.7249130522832274, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.2875, "aurc": 0.06034373794472666, "error_rate_at_0_9": 0.0, "confidence_bias": -0.11258694771677258, "top_bins": { "0.9": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.575, "accuracy": 0.9347826086956522, "confidence_cutoff": 0.7310585677623749 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.5926665663719177 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.6947086643657242, "acc": 0.5, "ece": 0.024967503547668413, "brier": 0.5015584405835085, "mean_conf": 0.5249675035476684, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.024967503547668413, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5312093496322632 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5312093496322632 } } }, "unknowable_authorization": { "n": 10, "nll": 0.6769315558023368, "acc": 0.6, "ece": 0.07196000218391418, "brier": 0.48389166548091467, "mean_conf": 0.5280399978160858, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.32571428571428573, "error_rate_at_0_9": null, "confidence_bias": -0.07196000218391418, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.531209409236908 }, "0.8": { "coverage": 1.0, "accuracy": 0.6, "confidence_cutoff": 0.5 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.5737342276934214, "acc": 1.0, "ece": 0.43513118624687197, "brier": 0.3818724823450289, "mean_conf": 0.564868813753128, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": null, "confidence_bias": -0.43513118624687197, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.5926665663719177 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.5312093496322632 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.6683156254987609, "acc": 0.5, "ece": 0.046549105644226096, "brier": 0.47556249999673667, "mean_conf": 0.5465491056442261, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.31, "error_rate_at_0_9": null, "confidence_bias": 0.046549105644226096, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.562176525592804 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.531209409236908 } } }, "unknowable_control_deadline": { "n": 10, "nll": 2.9213554199896956, "acc": 0.3, "ece": 0.6547563158153, "brier": 1.3000704884490109, "mean_conf": 0.9547563158153001, "confident_error_rate": 0.7, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.3, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6690476190476191, "error_rate_at_0_9": 0.7, "confidence_bias": 0.6547563158153, "top_bins": { "0.9": { "n": 10, "errors": 7, "error_rate": 0.7 }, "0.95": { "n": 8, "errors": 6, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.9572610626650837 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.9505933323983569 } }, "score_mae": 0.7023828513699522, "ranked_probability_score": 0.33006472716063306 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.7330163732489625, "acc": 0.7, "ece": 0.1482540632041076, "brier": 0.4451413587721745, "mean_conf": 0.7021622002406106, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.17857142857142855, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0021622002406106455, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7362475450057354 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.5883927323356541 } }, "score_mae": 0.4211218097506838, "ranked_probability_score": 0.11729643003023708 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 1.1096857314005688, "acc": 0.2, "ece": 0.3814912517804449, "brier": 0.7079188069347107, "mean_conf": 0.5814912517804449, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5642063492063492, "error_rate_at_0_9": null, "confidence_bias": 0.3814912517804449, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.598909419807511 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5224289142361607 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.7456403458065914, "acc": 0.5, "ece": 0.16947978138923647, "brier": 0.5499934526273271, "mean_conf": 0.6250797569751739, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6214285714285714, "error_rate_at_0_9": null, "confidence_bias": 0.12507975697517393, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.7, "accuracy": 0.42857142857142855, "confidence_cutoff": 0.622459352016449 }, "0.8": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.5926666259765625 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 1.459165154023861, "acc": 0.3, "ece": 0.38370459231587195, "brier": 0.8650992018323784, "mean_conf": 0.6837045923158721, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.37666666666666665, "error_rate_at_0_9": null, "confidence_bias": 0.3837045923158721, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7158687604089267 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6573608917784425 } }, "score_mae": 0.7480355975722548, "ranked_probability_score": 0.2686501397214868 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.2945180507825598, "acc": 0.5, "ece": 0.3660619498101235, "brier": 0.7955644942063342, "mean_conf": 0.8660619498101235, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.37480158730158725, "error_rate_at_0_9": 0.5, "confidence_bias": 0.3660619498101235, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.8661572595285553 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.858034541554363 } }, "score_mae": 0.49130664669541435, "ranked_probability_score": 0.21094552795222593 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.18369367567457523, "acc": 1.0, "ece": 0.1604828644401017, "brier": 0.0722314247265077, "mean_conf": 0.8395171355598983, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.16048286444010174, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8791032774305659 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.7745119146570109 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 1.301242281351381, "acc": 0.2, "ece": 0.5072509144820883, "brier": 0.8680037613035105, "mean_conf": 0.7072509144820883, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5975396825396825, "error_rate_at_0_9": null, "confidence_bias": 0.5072509144820883, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7678657998998701 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.554495151679999 } }, "score_mae": 0.6802767949532421, "ranked_probability_score": 0.2402083862014263 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 1.2172446311530223, "acc": 0.5, "ece": 0.21492532874165532, "brier": 0.6898920276891446, "mean_conf": 0.614331785653604, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5393650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.11433178565360402, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6182967173846468 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5851253779958877 } }, "score_mae": 0.6095279326227744, "ranked_probability_score": 0.20760195788495833 }, "unknowable_deadline": { "n": 10, "nll": 2.8963137320066457, "acc": 0.3, "ece": 0.6547488685735938, "brier": 1.292605083197752, "mean_conf": 0.9547488685735939, "confident_error_rate": 0.7, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.3, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.47833333333333333, "error_rate_at_0_9": 0.7, "confidence_bias": 0.6547488685735938, "top_bins": { "0.9": { "n": 10, "errors": 7, "error_rate": 0.7 }, "0.95": { "n": 10, "errors": 7, "error_rate": 0.7 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9569650579822465 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.9515128596587377 } }, "score_mae": 0.6989495530068959, "ranked_probability_score": 0.3281092352053006 }, "unknowable_late_fee": { "n": 10, "nll": 1.2352813127049052, "acc": 0.3, "ece": 0.20876469497319652, "brier": 0.7645989621032641, "mean_conf": 0.5087646949731965, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8066666666666666, "error_rate_at_0_9": null, "confidence_bias": 0.20876469497319655, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.4865780480755539 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.455443338887592 } }, "score_mae": 0.7500844793184518, "ranked_probability_score": 0.25087444214977045 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.4761740996737214, "acc": 0.2, "ece": 0.4369571633258407, "brier": 0.9029347388098353, "mean_conf": 0.6369571633258407, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.71, "error_rate_at_0_9": null, "confidence_bias": 0.4369571633258407, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.6338076355379956 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.6338076355379956 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7441604566299005, "acc": 0.5, "ece": 0.13383814096450805, "brier": 0.548481551510595, "mean_conf": 0.6338381409645081, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6, "error_rate_at_0_9": null, "confidence_bias": 0.13383814096450808, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.6513549089431763 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.622459352016449 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.5094806083275512, "acc": 0.3, "ece": 0.340109757562937, "brier": 0.8847333491579915, "mean_conf": 0.6401097575629371, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.595, "error_rate_at_0_9": null, "confidence_bias": 0.3401097575629371, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.6486542224884033 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.6286793632214976 } }, "score_mae": 0.7848108417509551, "ranked_probability_score": 0.2824946138191086 }, "unknowable_sla_response": { "n": 10, "nll": 1.5525779684374725, "acc": 0.5, "ece": 0.36936887725594936, "brier": 0.8390297583521665, "mean_conf": 0.8693688772559494, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.36936887725594936, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.8634033863500012 }, "0.8": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.8595718241435121 } }, "score_mae": 0.5385530277537973, "ranked_probability_score": 0.22622947383537167 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.7042547315982548, "acc": 0.5, "ece": 0.03989891485337149, "brier": 0.5042721132690919, "mean_conf": 0.5398989148533715, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.03989891485337149, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.5276808140204844 }, "0.8": { "coverage": 1.0, "accuracy": 0.5, "confidence_cutoff": 0.5276808140204844 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.6212674983160316, "acc": 0.2, "ece": 0.4646719712749066, "brier": 0.9724220547126645, "mean_conf": 0.6646719712749066, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7516666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.4646719712749066, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.6573608917784425 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.6569174626010278 } }, "score_mae": 0.8284220695573502, "ranked_probability_score": 0.29885309255970255 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.3658876748653637, "acc": 0.5, "ece": 0.320665966310269, "brier": 0.7016785987000886, "mean_conf": 0.6694706277777724, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.49174603174603176, "error_rate_at_0_9": null, "confidence_bias": 0.16947062777777244, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.6891418361706888 }, "0.8": { "coverage": 0.9, "accuracy": 0.5555555555555556, "confidence_cutoff": 0.6253168396557319 } }, "score_mae": 0.6239943634910664, "ranked_probability_score": 0.21102028740990403 } }, "variants": { "clean": { "n": 1156, "nll": 1.240636939125609, "acc": 0.5475778546712803, "ece": 0.13526968229519157, "brier": 0.5891280361385901, "mean_conf": 0.6828475369664718, "confident_error_rate": 0.05103806228373702, "coverage_at_0_9": 0.20328719723183392, "accuracy_at_0_9": 0.7489361702127659, "coverage_at_5pct_error": 0.10640138408304499, "coverage_at_1pct_error": 0.02768166089965398, "aurc": 0.29318375906234856, "error_rate_at_0_9": 0.251063829787234, "confidence_bias": 0.13526968229519154, "top_bins": { "0.9": { "n": 235, "errors": 59, "error_rate": 0.251063829787234 }, "0.95": { "n": 173, "errors": 39, "error_rate": 0.2254335260115607 }, "0.99": { "n": 69, "errors": 1, "error_rate": 0.014492753623188406 } }, "selective": { "0.5": { "coverage": 0.5051903114186851, "accuracy": 0.666095890410959, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8105536332179931, "accuracy": 0.615795090715048, "confidence_cutoff": 0.5312093496322632 } }, "score_mae": 0.6788096741982781, "ranked_probability_score": 0.2737498151737588 }, "none_absent": { "n": 36, "nll": 1.2249023682701092, "acc": 0.3888888888888889, "ece": 0.20415535274912633, "brier": 0.6886218147427213, "mean_conf": 0.5930442416380152, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.08333333333333333, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5815195819636397, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.20415535274912627, "top_bins": { "0.9": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.5730151704852334 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.4827586206896552, "confidence_cutoff": 0.418417395906474 } } }, "none_present": { "n": 36, "nll": 1.3998282013710819, "acc": 0.6388888888888888, "ece": 0.18459568862188494, "brier": 0.5374461664038009, "mean_conf": 0.7192473473480416, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3333333333333333, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4166666666666667, "coverage_at_1pct_error": 0.4166666666666667, "aurc": 0.15611433904325775, "error_rate_at_0_9": 0.0, "confidence_bias": 0.08035845845915279, "top_bins": { "0.9": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.7928478488439357 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.6896551724137931, "confidence_cutoff": 0.44587551477390686 } } }, "permuted": { "n": 36, "nll": 1.0421663604735372, "acc": 0.6111111111111112, "ece": 0.17513175319367755, "brier": 0.47002828271007985, "mean_conf": 0.7572932013389155, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.4166666666666667, "accuracy_at_0_9": 0.8666666666666667, "coverage_at_5pct_error": 0.1388888888888889, "coverage_at_1pct_error": 0.1388888888888889, "aurc": 0.16438422529613383, "error_rate_at_0_9": 0.13333333333333333, "confidence_bias": 0.14618209022780437, "top_bins": { "0.9": { "n": 15, "errors": 2, "error_rate": 0.13333333333333333 }, "0.95": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8888888888888888, "confidence_cutoff": 0.861844886785514 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7241379310344828, "confidence_cutoff": 0.4865805624528738 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.20130782505453082, "flip_rate": 0.3055555555555556 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 1.2327016430228732, "acc": 0.5630975143403442, "ece": 0.12300515062522288, "brier": 0.5708125684679077, "mean_conf": 0.6861026649655669, "confident_error_rate": 0.0497131931166348, "coverage_at_0_9": 0.21510516252390058, "accuracy_at_0_9": 0.7688888888888888, "coverage_at_5pct_error": 0.11759082217973231, "coverage_at_1pct_error": 0.030592734225621414, "aurc": 0.2698851807680104, "error_rate_at_0_9": 0.2311111111111111, "confidence_bias": 0.12300515062522277, "top_bins": { "0.9": { "n": 225, "errors": 52, "error_rate": 0.2311111111111111 }, "0.95": { "n": 163, "errors": 32, "error_rate": 0.19631901840490798 }, "0.99": { "n": 69, "errors": 1, "error_rate": 0.014492753623188406 } }, "selective": { "0.5": { "coverage": 0.5162523900573613, "accuracy": 0.6907407407407408, "confidence_cutoff": 0.6513548701194006 }, "0.8": { "coverage": 0.8068833652007649, "accuracy": 0.6409952606635071, "confidence_cutoff": 0.5312093496322632 } }, "score_mae": 0.6636140452293933, "ranked_probability_score": 0.2782415897800982 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Kai": { "panels": { "old_core": { "point": 0.5796130952380952, "exact_fraction": "779/1344", "ci95": [ 0.5375372023809525, 0.6222098214285714 ] }, "v3_core": { "point": 0.4083333333333333, "exact_fraction": "49/120", "ci95": [ 0.38583333333333336, 0.43125 ] }, "v4": { "point": 0.546875, "exact_fraction": "35/64", "ci95": [ 0.4987558297706957, 0.5953125 ] }, "v5": { "point": 0.6979166666666666, "exact_fraction": "67/96", "ci95": [ 0.6627755357671038, 0.7312535310734464 ] }, "transfer_v9_test": { "point": 0.4837476099426386, "exact_fraction": "253/523", "ci95": [ 0.4534880873744391, 0.5136407543107796 ] }, "original_four_panel_mean": { "point": 0.5581845238095238, "exact_fraction": "3751/6720", "ci95": [ 0.5391977049977125, 0.5772806794041933 ] }, "weighted_mean": { "point": 0.5352481533961577, "exact_fraction": "7524647/14058240", "ci95": [ 0.5184750126922689, 0.5524634057618808 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 98, "requested": 128 }, "boolean_constraints": { "correct": 34, "requested": 64 }, "dbpedia_14": { "correct": 95, "requested": 112 }, "natural_intents": { "correct": 57, "requested": 64 }, "option_carrier": { "correct": 96, "requested": 96 }, "ordinal_rubric": { "correct": 4, "requested": 64 }, "relational_composition": { "correct": 39, "requested": 96 }, "scoped_evidence": { "correct": 24, "requested": 96 }, "state_tracking": { "correct": 28, "requested": 96 }, "unknown_rejection": { "correct": 48, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 41, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 21, "requested": 80 }, "massive_en": { "correct": 98, "requested": 120 }, "massive_zh": { "correct": 95, "requested": 120 }, "multiset_reconciliation": { "correct": 20, "requested": 80 }, "ordinal_service_loss": { "correct": 16, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 20, "requested": 80 }, "temporal_exclusion": { "correct": 14, "requested": 80 }, "transaction_recovery": { "correct": 26, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 119, "requested": 160 }, "belebele_en": { "correct": 61, "requested": 160 }, "belebele_zh": { "correct": 51, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 56, "requested": 120 }, "squad2_answerability": { "correct": 67, "requested": 120 }, "snli": { "correct": 94, "requested": 120 }, "qasc": { "correct": 118, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 506, "accuracy": 0.4837476099426386, "full_probability_metrics": { "n": 1046, "nll": 1.2391125114356256, "acc": 0.4837476099426386, "ece": 0.14362213022361828, "brier": 0.6391659534279467, "mean_conf": 0.6246297488272995, "confident_error_rate": 0.052581261950286805, "coverage_at_0_9": 0.2074569789674952, "accuracy_at_0_9": 0.7465437788018433, "coverage_at_5pct_error": 0.040152963671128104, "coverage_at_1pct_error": 0.0248565965583174, "aurc": 0.33448141017109756, "error_rate_at_0_9": 0.2534562211981567, "confidence_bias": 0.1408821388846609, "top_bins": { "0.9": { "n": 217, "errors": 55, "error_rate": 0.2534562211981567 }, "0.95": { "n": 125, "errors": 15, "error_rate": 0.12 }, "0.99": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6462715105162524, "confidence_cutoff": 0.626142047277567 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5507765830346476, "confidence_cutoff": 0.34034422991608204 } }, "score_mae": 0.6822981273891354, "ranked_probability_score": 0.22411109211700433 }, "covered_only_probability_metrics": { "n": 1046, "nll": 1.2391125114356256, "acc": 0.4837476099426386, "ece": 0.14362213022361828, "brier": 0.6391659534279467, "mean_conf": 0.6246297488272995, "confident_error_rate": 0.052581261950286805, "coverage_at_0_9": 0.2074569789674952, "accuracy_at_0_9": 0.7465437788018433, "coverage_at_5pct_error": 0.040152963671128104, "coverage_at_1pct_error": 0.0248565965583174, "aurc": 0.33448141017109756, "error_rate_at_0_9": 0.2534562211981567, "confidence_bias": 0.1408821388846609, "top_bins": { "0.9": { "n": 217, "errors": 55, "error_rate": 0.2534562211981567 }, "0.95": { "n": 125, "errors": 15, "error_rate": 0.12 }, "0.99": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6462715105162524, "confidence_cutoff": 0.626142047277567 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5507765830346476, "confidence_cutoff": 0.34034422991608204 } }, "score_mae": 0.6822981273891354, "ranked_probability_score": 0.22411109211700433 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 9, "accuracy": 0.45, "full_probability_metrics": { "n": 20, "nll": 1.6537518068546924, "acc": 0.45, "ece": 0.16356178033417812, "brier": 0.7658320409843673, "mean_conf": 0.4202073978171811, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.540369532857923, "error_rate_at_0_9": null, "confidence_bias": -0.029792602182818928, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.4285053818321599 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3242095757158038 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.6537518068546924, "acc": 0.45, "ece": 0.16356178033417812, "brier": 0.7658320409843673, "mean_conf": 0.4202073978171811, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.540369532857923, "error_rate_at_0_9": null, "confidence_bias": -0.029792602182818928, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.4285053818321599 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3242095757158038 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 7, "accuracy": 0.35, "full_probability_metrics": { "n": 20, "nll": 0.7827316896839417, "acc": 0.35, "ece": 0.24277215003967284, "brier": 0.5858853695879119, "mean_conf": 0.5927721500396729, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6953620351956266, "error_rate_at_0_9": null, "confidence_bias": 0.2427721500396729, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5672407746315002 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5477952659130096 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.7827316896839417, "acc": 0.35, "ece": 0.24277215003967284, "brier": 0.5858853695879119, "mean_conf": 0.5927721500396729, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6953620351956266, "error_rate_at_0_9": null, "confidence_bias": 0.2427721500396729, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5672407746315002 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5477952659130096 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 11, "accuracy": 0.55, "full_probability_metrics": { "n": 20, "nll": 0.8555022618540915, "acc": 0.55, "ece": 0.2654646812006831, "brier": 0.6094519995364962, "mean_conf": 0.7867073891684413, "confident_error_rate": 0.05, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.34032932603172544, "error_rate_at_0_9": 0.2, "confidence_bias": 0.23670738916844125, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8149255514144897 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.6623975336551666 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.8555022618540915, "acc": 0.55, "ece": 0.2654646812006831, "brier": 0.6094519995364962, "mean_conf": 0.7867073891684413, "confident_error_rate": 0.05, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.34032932603172544, "error_rate_at_0_9": 0.2, "confidence_bias": 0.23670738916844125, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8149255514144897 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.6623975336551666 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 15, "accuracy": 0.75, "full_probability_metrics": { "n": 20, "nll": 0.5035467531682788, "acc": 0.75, "ece": 0.19359442964196205, "brier": 0.32708305202234894, "mean_conf": 0.6864339232444763, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.10817796152935473, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06356607675552373, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.6365915834903717 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5576783418655396 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5035467531682788, "acc": 0.75, "ece": 0.19359442964196205, "brier": 0.32708305202234894, "mean_conf": 0.6864339232444763, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.10817796152935473, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06356607675552373, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.6365915834903717 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5576783418655396 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 20, "accuracy": 0.625, "full_probability_metrics": { "n": 32, "nll": 0.7071989721918829, "acc": 0.625, "ece": 0.1490274060111474, "brier": 0.5070841555253245, "mean_conf": 0.6285915295793278, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.35915273066606157, "error_rate_at_0_9": null, "confidence_bias": 0.0035915295793278457, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.6476195068586167 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.5155036449432373 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.7071989721918829, "acc": 0.625, "ece": 0.1490274060111474, "brier": 0.5070841555253245, "mean_conf": 0.6285915295793278, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.35915273066606157, "error_rate_at_0_9": null, "confidence_bias": 0.0035915295793278457, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.6476195068586167 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.5155036449432373 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 17, "accuracy": 0.53125, "full_probability_metrics": { "n": 32, "nll": 0.7698616075669997, "acc": 0.53125, "ece": 0.23369681178224483, "brier": 0.5720272792530656, "mean_conf": 0.6067463536620291, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5686590575565279, "error_rate_at_0_9": null, "confidence_bias": 0.07549635366202911, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.375, "confidence_cutoff": 0.6199085301133552 }, "0.8": { "coverage": 0.84375, "accuracy": 0.48148148148148145, "confidence_cutoff": 0.5331743359565735 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.7698616075669997, "acc": 0.53125, "ece": 0.23369681178224483, "brier": 0.5720272792530656, "mean_conf": 0.6067463536620291, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5686590575565279, "error_rate_at_0_9": null, "confidence_bias": 0.07549635366202911, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.375, "confidence_cutoff": 0.6199085301133552 }, "0.8": { "coverage": 0.84375, "accuracy": 0.48148148148148145, "confidence_cutoff": 0.5331743359565735 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 18, "accuracy": 0.5625, "full_probability_metrics": { "n": 32, "nll": 0.925746127186436, "acc": 0.5625, "ece": 0.23582328519674503, "brier": 0.6420966270281695, "mean_conf": 0.798323285196745, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5213700566016615, "error_rate_at_0_9": 0.75, "confidence_bias": 0.235823285196745, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.4117647058823529, "confidence_cutoff": 0.7928446171535437 }, "0.8": { "coverage": 0.84375, "accuracy": 0.5185185185185185, "confidence_cutoff": 0.7655237088941422 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.925746127186436, "acc": 0.5625, "ece": 0.23582328519674503, "brier": 0.6420966270281695, "mean_conf": 0.798323285196745, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5213700566016615, "error_rate_at_0_9": 0.75, "confidence_bias": 0.235823285196745, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.4117647058823529, "confidence_cutoff": 0.7928446171535437 }, "0.8": { "coverage": 0.84375, "accuracy": 0.5185185185185185, "confidence_cutoff": 0.7655237088941422 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 20, "accuracy": 0.5, "full_probability_metrics": { "n": 40, "nll": 1.4121071983456814, "acc": 0.5, "ece": 0.4394341051578522, "brier": 0.879494291145727, "mean_conf": 0.9394341051578522, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.3325839665024844, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4394341051578522, "top_bins": { "0.9": { "n": 40, "errors": 20, "error_rate": 0.5 }, "0.95": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.55, "confidence_cutoff": 0.9448155164718628 }, "0.8": { "coverage": 0.8, "accuracy": 0.53125, "confidence_cutoff": 0.930177628993988 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 1.4121071983456814, "acc": 0.5, "ece": 0.4394341051578522, "brier": 0.879494291145727, "mean_conf": 0.9394341051578522, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.3325839665024844, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4394341051578522, "top_bins": { "0.9": { "n": 40, "errors": 20, "error_rate": 0.5 }, "0.95": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.55, "confidence_cutoff": 0.9448155164718628 }, "0.8": { "coverage": 0.8, "accuracy": 0.53125, "confidence_cutoff": 0.930177628993988 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 20, "accuracy": 0.5, "full_probability_metrics": { "n": 40, "nll": 1.0939331104726837, "acc": 0.5, "ece": 0.16049458460030175, "brier": 0.6635325488255226, "mean_conf": 0.33950541539969825, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.4541642624276797, "error_rate_at_0_9": null, "confidence_bias": -0.16049458460030175, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.339652419090271 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33875223033143465 } }, "score_mae": 0.7499943084929652, "ranked_probability_score": 0.23488777194331573 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.0939331104726837, "acc": 0.5, "ece": 0.16049458460030175, "brier": 0.6635325488255226, "mean_conf": 0.33950541539969825, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.4541642624276797, "error_rate_at_0_9": null, "confidence_bias": -0.16049458460030175, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.339652419090271 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33875223033143465 } }, "score_mae": 0.7499943084929652, "ranked_probability_score": 0.23488777194331573 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 43, "accuracy": 0.5375, "full_probability_metrics": { "n": 80, "nll": 1.3203244603528614, "acc": 0.5375, "ece": 0.0954288831844616, "brier": 0.6231048759158845, "mean_conf": 0.6141983500053696, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.0375, "coverage_at_1pct_error": 0.0375, "aurc": 0.3275746152390444, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.07669835000536962, "top_bins": { "0.9": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.6244700705076331 }, "0.8": { "coverage": 0.8, "accuracy": 0.578125, "confidence_cutoff": 0.42069205720106523 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.3203244603528614, "acc": 0.5375, "ece": 0.0954288831844616, "brier": 0.6231048759158845, "mean_conf": 0.6141983500053696, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.0375, "coverage_at_1pct_error": 0.0375, "aurc": 0.3275746152390444, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.07669835000536962, "top_bins": { "0.9": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.6244700705076331 }, "0.8": { "coverage": 0.8, "accuracy": 0.578125, "confidence_cutoff": 0.42069205720106523 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 30, "accuracy": 0.375, "full_probability_metrics": { "n": 80, "nll": 1.5752882834112132, "acc": 0.375, "ece": 0.19078021772010062, "brier": 0.8012440739258787, "mean_conf": 0.5657802177201006, "confident_error_rate": 0.05, "coverage_at_0_9": 0.0875, "accuracy_at_0_9": 0.42857142857142855, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6181087704306802, "error_rate_at_0_9": 0.5714285714285714, "confidence_bias": 0.19078021772010056, "top_bins": { "0.9": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5258800905066715 }, "0.8": { "coverage": 0.8, "accuracy": 0.390625, "confidence_cutoff": 0.39601410025379213 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.5752882834112132, "acc": 0.375, "ece": 0.19078021772010062, "brier": 0.8012440739258787, "mean_conf": 0.5657802177201006, "confident_error_rate": 0.05, "coverage_at_0_9": 0.0875, "accuracy_at_0_9": 0.42857142857142855, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6181087704306802, "error_rate_at_0_9": 0.5714285714285714, "confidence_bias": 0.19078021772010056, "top_bins": { "0.9": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5258800905066715 }, "0.8": { "coverage": 0.8, "accuracy": 0.390625, "confidence_cutoff": 0.39601410025379213 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 25, "accuracy": 0.125, "full_probability_metrics": { "n": 200, "nll": 2.5917944313715453, "acc": 0.125, "ece": 0.25616445737555654, "brier": 1.0204430675216853, "mean_conf": 0.3811644573755565, "confident_error_rate": 0.025, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8694262499280748, "error_rate_at_0_9": 1.0, "confidence_bias": 0.2561644573755565, "top_bins": { "0.9": { "n": 5, "errors": 5, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.17, "confidence_cutoff": 0.3029599144613756 }, "0.8": { "coverage": 0.8, "accuracy": 0.1375, "confidence_cutoff": 0.18382079830832482 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.5917944313715453, "acc": 0.125, "ece": 0.25616445737555654, "brier": 1.0204430675216853, "mean_conf": 0.3811644573755565, "confident_error_rate": 0.025, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8694262499280748, "error_rate_at_0_9": 1.0, "confidence_bias": 0.2561644573755565, "top_bins": { "0.9": { "n": 5, "errors": 5, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.17, "confidence_cutoff": 0.3029599144613756 }, "0.8": { "coverage": 0.8, "accuracy": 0.1375, "confidence_cutoff": 0.18382079830832482 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 38, "accuracy": 0.475, "full_probability_metrics": { "n": 80, "nll": 0.7090254071044348, "acc": 0.475, "ece": 0.1666798379272222, "brier": 0.5166084866955849, "mean_conf": 0.6398283705115319, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3926720126883591, "error_rate_at_0_9": 0.0, "confidence_bias": 0.16482837051153187, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.6297575533390045 }, "0.8": { "coverage": 0.8, "accuracy": 0.515625, "confidence_cutoff": 0.5585934221744537 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.7090254071044348, "acc": 0.475, "ece": 0.1666798379272222, "brier": 0.5166084866955849, "mean_conf": 0.6398283705115319, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3926720126883591, "error_rate_at_0_9": 0.0, "confidence_bias": 0.16482837051153187, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.6297575533390045 }, "0.8": { "coverage": 0.8, "accuracy": 0.515625, "confidence_cutoff": 0.5585934221744537 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 57, "accuracy": 0.7125, "full_probability_metrics": { "n": 80, "nll": 0.7029168919881199, "acc": 0.7125, "ece": 0.1819537915289402, "brier": 0.44881665172376445, "mean_conf": 0.8604280255734921, "confident_error_rate": 0.1125, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.22011525782827887, "error_rate_at_0_9": 0.25, "confidence_bias": 0.14792802557349205, "top_bins": { "0.9": { "n": 36, "errors": 9, "error_rate": 0.25 }, "0.95": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.8962836265563965 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.7553151249885559 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.7029168919881199, "acc": 0.7125, "ece": 0.1819537915289402, "brier": 0.44881665172376445, "mean_conf": 0.8604280255734921, "confident_error_rate": 0.1125, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.22011525782827887, "error_rate_at_0_9": 0.25, "confidence_bias": 0.14792802557349205, "top_bins": { "0.9": { "n": 36, "errors": 9, "error_rate": 0.25 }, "0.95": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.8962836265563965 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.7553151249885559 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 72, "accuracy": 0.9, "full_probability_metrics": { "n": 80, "nll": 0.28774349774806274, "acc": 0.9, "ece": 0.03287756702694229, "brier": 0.15326008074439718, "mean_conf": 0.8900365305265652, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.6375, "accuracy_at_0_9": 0.9803921568627451, "coverage_at_5pct_error": 0.8375, "coverage_at_1pct_error": 0.5875, "aurc": 0.018813080914695256, "error_rate_at_0_9": 0.0196078431372549, "confidence_bias": -0.00996346947343485, "top_bins": { "0.9": { "n": 51, "errors": 1, "error_rate": 0.0196078431372549 }, "0.95": { "n": 43, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 25, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9633495856408731 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.8168928892105871 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.28774349774806274, "acc": 0.9, "ece": 0.03287756702694229, "brier": 0.15326008074439718, "mean_conf": 0.8900365305265652, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.6375, "accuracy_at_0_9": 0.9803921568627451, "coverage_at_5pct_error": 0.8375, "coverage_at_1pct_error": 0.5875, "aurc": 0.018813080914695256, "error_rate_at_0_9": 0.0196078431372549, "confidence_bias": -0.00996346947343485, "top_bins": { "0.9": { "n": 51, "errors": 1, "error_rate": 0.0196078431372549 }, "0.95": { "n": 43, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 25, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9633495856408731 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.8168928892105871 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 63, "accuracy": 0.7875, "full_probability_metrics": { "n": 80, "nll": 0.38772103274242486, "acc": 0.7875, "ece": 0.07751428142655643, "brier": 0.25593492533075707, "mean_conf": 0.8444197329459712, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.975609756097561, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.4875, "aurc": 0.05995296937845444, "error_rate_at_0_9": 0.024390243902439025, "confidence_bias": 0.05691973294597119, "top_bins": { "0.9": { "n": 41, "errors": 1, "error_rate": 0.024390243902439025 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9004859104752541 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7327178418636322 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.38772103274242486, "acc": 0.7875, "ece": 0.07751428142655643, "brier": 0.25593492533075707, "mean_conf": 0.8444197329459712, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.975609756097561, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.4875, "aurc": 0.05995296937845444, "error_rate_at_0_9": 0.024390243902439025, "confidence_bias": 0.05691973294597119, "top_bins": { "0.9": { "n": 41, "errors": 1, "error_rate": 0.024390243902439025 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9004859104752541 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7327178418636322 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.6191405658106865, "acc": 0.5, "ece": 0.45958920121192937, "brier": 0.9210477728721707, "mean_conf": 0.9595892012119294, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4106349206349207, "error_rate_at_0_9": 0.5, "confidence_bias": 0.45958920121192937, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9605617523193359 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9569776654243469 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.6191405658106865, "acc": 0.5, "ece": 0.45958920121192937, "brier": 0.9210477728721707, "mean_conf": 0.9595892012119294, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4106349206349207, "error_rate_at_0_9": 0.5, "confidence_bias": 0.45958920121192937, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9605617523193359 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9569776654243469 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.4433423409086612, "acc": 0.5, "ece": 0.4431973397731781, "brier": 0.8877473385116084, "mean_conf": 0.9431973397731781, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.31007936507936507, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4431973397731781, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9448668360710144 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9334832429885864 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.4433423409086612, "acc": 0.5, "ece": 0.4431973397731781, "brier": 0.8877473385116084, "mean_conf": 0.9431973397731781, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.31007936507936507, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4431973397731781, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9448668360710144 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9334832429885864 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.0939339000796315, "acc": 0.5, "ece": 0.16012432878157545, "brier": 0.6635327214197148, "mean_conf": 0.33987567121842455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5417460317460316, "error_rate_at_0_9": null, "confidence_bias": -0.16012432878157545, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.34033403752243296 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33860839375207463 } }, "score_mae": 0.6999460694543735, "ranked_probability_score": 0.22628900665691365 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0939339000796315, "acc": 0.5, "ece": 0.16012432878157545, "brier": 0.6635327214197148, "mean_conf": 0.33987567121842455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5417460317460316, "error_rate_at_0_9": null, "confidence_bias": -0.16012432878157545, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.34033403752243296 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33860839375207463 } }, "score_mae": 0.6999460694543735, "ranked_probability_score": 0.22628900665691365 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.0998773717268528, "acc": 0.3, "ece": 0.040559427915460855, "brier": 0.667516799962047, "mean_conf": 0.34055942791546084, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7634920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.040559427915460855, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.34070939775053277 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3402503031417637 } }, "score_mae": 0.604843196236742, "ranked_probability_score": 0.21132520467597854 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0998773717268528, "acc": 0.3, "ece": 0.040559427915460855, "brier": 0.667516799962047, "mean_conf": 0.34055942791546084, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7634920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.040559427915460855, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.34070939775053277 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3402503031417637 } }, "score_mae": 0.604843196236742, "ranked_probability_score": 0.21132520467597854 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 2, "accuracy": 0.2, "full_probability_metrics": { "n": 10, "nll": 1.3488958869964383, "acc": 0.2, "ece": 0.36076402581608485, "brier": 0.8420843440836799, "mean_conf": 0.5607640258160849, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7725396825396824, "error_rate_at_0_9": null, "confidence_bias": 0.36076402581608485, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5632423246389974 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5573770403862 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3488958869964383, "acc": 0.2, "ece": 0.36076402581608485, "brier": 0.8420843440836799, "mean_conf": 0.5607640258160849, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7725396825396824, "error_rate_at_0_9": null, "confidence_bias": 0.36076402581608485, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5632423246389974 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5573770403862 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.8455590159265375, "acc": 0.5, "ece": 0.25936471819877627, "brier": 0.6308352614839883, "mean_conf": 0.7593647181987763, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5249206349206349, "error_rate_at_0_9": null, "confidence_bias": 0.25936471819877627, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7657169103622437 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7524310350418091 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.8455590159265375, "acc": 0.5, "ece": 0.25936471819877627, "brier": 0.6308352614839883, "mean_conf": 0.7593647181987763, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5249206349206349, "error_rate_at_0_9": null, "confidence_bias": 0.25936471819877627, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7657169103622437 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7524310350418091 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 2, "accuracy": 0.2, "full_probability_metrics": { "n": 10, "nll": 1.1046858639299395, "acc": 0.2, "ece": 0.14331720862452646, "brier": 0.6707438425130093, "mean_conf": 0.34331720862452647, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6592063492063491, "error_rate_at_0_9": null, "confidence_bias": 0.14331720862452646, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3432188034057617 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.34267568588256836 } }, "score_mae": 0.7090615362866717, "ranked_probability_score": 0.22997851394300825 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.1046858639299395, "acc": 0.2, "ece": 0.14331720862452646, "brier": 0.6707438425130093, "mean_conf": 0.34331720862452647, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6592063492063491, "error_rate_at_0_9": null, "confidence_bias": 0.14331720862452646, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3432188034057617 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.34267568588256836 } }, "score_mae": 0.7090615362866717, "ranked_probability_score": 0.22997851394300825 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.0987144215013314, "acc": 0.3, "ece": 0.03676625683862805, "brier": 0.6667406922057552, "mean_conf": 0.33676625683862804, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6759920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.03676625683862805, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3369380034841297 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.3366921942440329 } }, "score_mae": 0.5049051225214941, "ranked_probability_score": 0.1947439920178244 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0987144215013314, "acc": 0.3, "ece": 0.03676625683862805, "brier": 0.6667406922057552, "mean_conf": 0.33676625683862804, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6759920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.03676625683862805, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3369380034841297 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.3366921942440329 } }, "score_mae": 0.5049051225214941, "ranked_probability_score": 0.1947439920178244 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.7539871718217989, "acc": 0.5, "ece": 0.05788718583661284, "brier": 0.5150498126641045, "mean_conf": 0.5419843572093712, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.543968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.041984357209371215, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5388615412416368 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5307940205127446 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.7539871718217989, "acc": 0.5, "ece": 0.05788718583661284, "brier": 0.5150498126641045, "mean_conf": 0.5419843572093712, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.543968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.041984357209371215, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5388615412416368 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5307940205127446 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.0978271810968157, "acc": 0.4, "ece": 0.06276650428716868, "brier": 0.6661392699562907, "mean_conf": 0.33723349571283134, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5396428571428571, "error_rate_at_0_9": null, "confidence_bias": -0.06276650428716868, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.33723676204681396 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3366008400917053 } }, "score_mae": 0.8012357295066987, "ranked_probability_score": 0.24436618036188062 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0978271810968157, "acc": 0.4, "ece": 0.06276650428716868, "brier": 0.6661392699562907, "mean_conf": 0.33723349571283134, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5396428571428571, "error_rate_at_0_9": null, "confidence_bias": -0.06276650428716868, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.33723676204681396 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3366008400917053 } }, "score_mae": 0.8012357295066987, "ranked_probability_score": 0.24436618036188062 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 2, "accuracy": 0.2, "full_probability_metrics": { "n": 10, "nll": 1.100952041539422, "acc": 0.2, "ece": 0.13708551726927687, "brier": 0.6682286858387834, "mean_conf": 0.3370855172692769, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7425396825396826, "error_rate_at_0_9": null, "confidence_bias": 0.13708551726927687, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3369665343473394 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3365091185721492 } }, "score_mae": 0.5030123859135118, "ranked_probability_score": 0.19485693574117507 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.100952041539422, "acc": 0.2, "ece": 0.13708551726927687, "brier": 0.6682286858387834, "mean_conf": 0.3370855172692769, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7425396825396826, "error_rate_at_0_9": null, "confidence_bias": 0.13708551726927687, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3369665343473394 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3365091185721492 } }, "score_mae": 0.5030123859135118, "ranked_probability_score": 0.19485693574117507 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.4752314814814814, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.4444444444444444, "flip_rate": 0.1388888888888889, "covered_only_flip_rate": 0.1388888888888889, "half_l1": 0.11467214772666708 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.5210661476391094, "share_at_0_9": 0.18181818181818182, "normalized_entropy": 0.8263149981933025 }, "covered_only_confidence_metrics": { "mean_max_p": 0.5210661476391094, "share_at_0_9": 0.18181818181818182, "normalized_entropy": 0.8263149981933025 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 41, "accuracy": 0.37272727272727274, "full_probability_metrics": { "n": 110, "nll": 1.1460832510307377, "acc": 0.37272727272727274, "ece": 0.1545215654353171, "brier": 0.7090605946828319, "mean_conf": 0.5272488381625897, "confident_error_rate": 0.09090909090909091, "coverage_at_0_9": 0.18181818181818182, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.00909090909090909, "coverage_at_1pct_error": 0.00909090909090909, "aurc": 0.5519609281522065, "error_rate_at_0_9": 0.5, "confidence_bias": 0.15452156543531698, "top_bins": { "0.9": { "n": 20, "errors": 10, "error_rate": 0.5 }, "0.95": { "n": 13, "errors": 6, "error_rate": 0.46153846153846156 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.41818181818181815, "confidence_cutoff": 0.3432188034057617 }, "0.8": { "coverage": 0.8, "accuracy": 0.4090909090909091, "confidence_cutoff": 0.33757469058036804 } }, "score_mae": 0.6371673399865823, "ranked_probability_score": 0.21692663889946345 }, "covered_only_probability_metrics": { "n": 110, "nll": 1.1460832510307377, "acc": 0.37272727272727274, "ece": 0.1545215654353171, "brier": 0.7090605946828319, "mean_conf": 0.5272488381625897, "confident_error_rate": 0.09090909090909091, "coverage_at_0_9": 0.18181818181818182, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.00909090909090909, "coverage_at_1pct_error": 0.00909090909090909, "aurc": 0.5519609281522065, "error_rate_at_0_9": 0.5, "confidence_bias": 0.15452156543531698, "top_bins": { "0.9": { "n": 20, "errors": 10, "error_rate": 0.5 }, "0.95": { "n": 13, "errors": 6, "error_rate": 0.46153846153846156 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.41818181818181815, "confidence_cutoff": 0.3432188034057617 }, "0.8": { "coverage": 0.8, "accuracy": 0.4090909090909091, "confidence_cutoff": 0.33757469058036804 } }, "score_mae": 0.6371673399865823, "ranked_probability_score": 0.21692663889946345 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": 0.006182690523480224 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 0 }, "complete_unchanged_upstream_report": { "objective": -1.069855899754869, "paired_flip": { "pairs": 64, "flip_rate": 0.015625, "both_correct_rate": 0.015625, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.625 }, "unknowable": { "n": 110, "mean_max_p": 0.5210661476391094, "share_at_0_9": 0.18181818181818182, "control_mean_max_p": 0.5272488381625897, "control_share_at_0_9": 0.18181818181818182, "control_acc": 0.37272727272727274, "paired_confidence_drop": 0.006182690523480224, "share_less_confident_than_control": 0.39090909090909093 }, "clean": { "n": 1046, "nll": 1.2391125114356256, "acc": 0.4837476099426386, "ece": 0.14362213022361828, "brier": 0.6391659534279467, "mean_conf": 0.6246297488272995, "confident_error_rate": 0.052581261950286805, "coverage_at_0_9": 0.2074569789674952, "accuracy_at_0_9": 0.7465437788018433, "coverage_at_5pct_error": 0.040152963671128104, "coverage_at_1pct_error": 0.0248565965583174, "aurc": 0.33448141017109756, "error_rate_at_0_9": 0.2534562211981567, "confidence_bias": 0.1408821388846609, "top_bins": { "0.9": { "n": 217, "errors": 55, "error_rate": 0.2534562211981567 }, "0.95": { "n": 125, "errors": 15, "error_rate": 0.12 }, "0.99": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6462715105162524, "confidence_cutoff": 0.626142047277567 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5507765830346476, "confidence_cutoff": 0.34034422991608204 } }, "score_mae": 0.6822981273891354, "ranked_probability_score": 0.22411109211700433 }, "tasks": { "buried_emotion": { "n": 20, "nll": 1.6537518068546924, "acc": 0.45, "ece": 0.16356178033417812, "brier": 0.7658320409843673, "mean_conf": 0.4202073978171811, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.540369532857923, "error_rate_at_0_9": null, "confidence_bias": -0.029792602182818928, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.4285053818321599 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.3242095757158038 } } }, "buried_paws": { "n": 20, "nll": 0.7827316896839417, "acc": 0.35, "ece": 0.24277215003967284, "brier": 0.5858853695879119, "mean_conf": 0.5927721500396729, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6953620351956266, "error_rate_at_0_9": null, "confidence_bias": 0.2427721500396729, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5672407746315002 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5477952659130096 } } }, "buried_qnli": { "n": 20, "nll": 0.8555022618540915, "acc": 0.55, "ece": 0.2654646812006831, "brier": 0.6094519995364962, "mean_conf": 0.7867073891684413, "confident_error_rate": 0.05, "coverage_at_0_9": 0.25, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.34032932603172544, "error_rate_at_0_9": 0.2, "confidence_bias": 0.23670738916844125, "top_bins": { "0.9": { "n": 5, "errors": 1, "error_rate": 0.2 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8149255514144897 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.6623975336551666 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.5035467531682788, "acc": 0.75, "ece": 0.19359442964196205, "brier": 0.32708305202234894, "mean_conf": 0.6864339232444763, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.10817796152935473, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06356607675552373, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.6365915834903717 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5576783418655396 } } }, "composition_final_combination": { "n": 32, "nll": 0.7071989721918829, "acc": 0.625, "ece": 0.1490274060111474, "brier": 0.5070841555253245, "mean_conf": 0.6285915295793278, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.35915273066606157, "error_rate_at_0_9": null, "confidence_bias": 0.0035915295793278457, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.6476195068586167 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.5155036449432373 } } }, "composition_final_exception": { "n": 32, "nll": 0.7698616075669997, "acc": 0.53125, "ece": 0.23369681178224483, "brier": 0.5720272792530656, "mean_conf": 0.6067463536620291, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5686590575565279, "error_rate_at_0_9": null, "confidence_bias": 0.07549635366202911, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.375, "confidence_cutoff": 0.6199085301133552 }, "0.8": { "coverage": 0.84375, "accuracy": 0.48148148148148145, "confidence_cutoff": 0.5331743359565735 } } }, "composition_final_negation": { "n": 32, "nll": 0.925746127186436, "acc": 0.5625, "ece": 0.23582328519674503, "brier": 0.6420966270281695, "mean_conf": 0.798323285196745, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.03125, "coverage_at_1pct_error": 0.03125, "aurc": 0.5213700566016615, "error_rate_at_0_9": 0.75, "confidence_bias": 0.235823285196745, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.4117647058823529, "confidence_cutoff": 0.7928446171535437 }, "0.8": { "coverage": 0.84375, "accuracy": 0.5185185185185185, "confidence_cutoff": 0.7655237088941422 } } }, "contrastive_authorization": { "n": 40, "nll": 1.4121071983456814, "acc": 0.5, "ece": 0.4394341051578522, "brier": 0.879494291145727, "mean_conf": 0.9394341051578522, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.125, "coverage_at_1pct_error": 0.125, "aurc": 0.3325839665024844, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4394341051578522, "top_bins": { "0.9": { "n": 40, "errors": 20, "error_rate": 0.5 }, "0.95": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.55, "confidence_cutoff": 0.9448155164718628 }, "0.8": { "coverage": 0.8, "accuracy": 0.53125, "confidence_cutoff": 0.930177628993988 } } }, "contrastive_deadline": { "n": 40, "nll": 1.0939331104726837, "acc": 0.5, "ece": 0.16049458460030175, "brier": 0.6635325488255226, "mean_conf": 0.33950541539969825, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.4541642624276797, "error_rate_at_0_9": null, "confidence_bias": -0.16049458460030175, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.339652419090271 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33875223033143465 } }, "score_mae": 0.7499943084929652, "ranked_probability_score": 0.23488777194331573 }, "emotion": { "n": 80, "nll": 1.3203244603528614, "acc": 0.5375, "ece": 0.0954288831844616, "brier": 0.6231048759158845, "mean_conf": 0.6141983500053696, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.075, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.0375, "coverage_at_1pct_error": 0.0375, "aurc": 0.3275746152390444, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.07669835000536962, "top_bins": { "0.9": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.6244700705076331 }, "0.8": { "coverage": 0.8, "accuracy": 0.578125, "confidence_cutoff": 0.42069205720106523 } } }, "mmlu": { "n": 80, "nll": 1.5752882834112132, "acc": 0.375, "ece": 0.19078021772010062, "brier": 0.8012440739258787, "mean_conf": 0.5657802177201006, "confident_error_rate": 0.05, "coverage_at_0_9": 0.0875, "accuracy_at_0_9": 0.42857142857142855, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6181087704306802, "error_rate_at_0_9": 0.5714285714285714, "confidence_bias": 0.19078021772010056, "top_bins": { "0.9": { "n": 7, "errors": 4, "error_rate": 0.5714285714285714 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5258800905066715 }, "0.8": { "coverage": 0.8, "accuracy": 0.390625, "confidence_cutoff": 0.39601410025379213 } } }, "mmlu_pro": { "n": 200, "nll": 2.5917944313715453, "acc": 0.125, "ece": 0.25616445737555654, "brier": 1.0204430675216853, "mean_conf": 0.3811644573755565, "confident_error_rate": 0.025, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8694262499280748, "error_rate_at_0_9": 1.0, "confidence_bias": 0.2561644573755565, "top_bins": { "0.9": { "n": 5, "errors": 5, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.17, "confidence_cutoff": 0.3029599144613756 }, "0.8": { "coverage": 0.8, "accuracy": 0.1375, "confidence_cutoff": 0.18382079830832482 } } }, "paws": { "n": 80, "nll": 0.7090254071044348, "acc": 0.475, "ece": 0.1666798379272222, "brier": 0.5166084866955849, "mean_conf": 0.6398283705115319, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.3926720126883591, "error_rate_at_0_9": 0.0, "confidence_bias": 0.16482837051153187, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.6297575533390045 }, "0.8": { "coverage": 0.8, "accuracy": 0.515625, "confidence_cutoff": 0.5585934221744537 } } }, "qnli": { "n": 80, "nll": 0.7029168919881199, "acc": 0.7125, "ece": 0.1819537915289402, "brier": 0.44881665172376445, "mean_conf": 0.8604280255734921, "confident_error_rate": 0.1125, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "aurc": 0.22011525782827887, "error_rate_at_0_9": 0.25, "confidence_bias": 0.14792802557349205, "top_bins": { "0.9": { "n": 36, "errors": 9, "error_rate": 0.25 }, "0.95": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.8962836265563965 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.7553151249885559 } } }, "sciq": { "n": 80, "nll": 0.28774349774806274, "acc": 0.9, "ece": 0.03287756702694229, "brier": 0.15326008074439718, "mean_conf": 0.8900365305265652, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.6375, "accuracy_at_0_9": 0.9803921568627451, "coverage_at_5pct_error": 0.8375, "coverage_at_1pct_error": 0.5875, "aurc": 0.018813080914695256, "error_rate_at_0_9": 0.0196078431372549, "confidence_bias": -0.00996346947343485, "top_bins": { "0.9": { "n": 51, "errors": 1, "error_rate": 0.0196078431372549 }, "0.95": { "n": 43, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 25, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9633495856408731 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.8168928892105871 } } }, "tweet_offensive": { "n": 80, "nll": 0.38772103274242486, "acc": 0.7875, "ece": 0.07751428142655643, "brier": 0.25593492533075707, "mean_conf": 0.8444197329459712, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.5125, "accuracy_at_0_9": 0.975609756097561, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.4875, "aurc": 0.05995296937845444, "error_rate_at_0_9": 0.024390243902439025, "confidence_bias": 0.05691973294597119, "top_bins": { "0.9": { "n": 41, "errors": 1, "error_rate": 0.024390243902439025 }, "0.95": { "n": 29, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9004859104752541 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.7327178418636322 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 1.642865228921649, "acc": 0.5, "ece": 0.46098687648773196, "brier": 0.9250307537209096, "mean_conf": 0.960986876487732, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.46098687648773196, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.9612478613853455 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9606207013130188 } } }, "unknowable_authorization": { "n": 10, "nll": 1.5255140954789224, "acc": 0.5, "ece": 0.4480596125125885, "brier": 0.9047932053537331, "mean_conf": 0.9480596125125885, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5818650793650793, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4480596125125885, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9487297534942627 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.9429174065589905 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 1.6191405658106865, "acc": 0.5, "ece": 0.45958920121192937, "brier": 0.9210477728721707, "mean_conf": 0.9595892012119294, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.4106349206349207, "error_rate_at_0_9": 0.5, "confidence_bias": 0.45958920121192937, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9605617523193359 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9569776654243469 } } }, "unknowable_control_authorization": { "n": 10, "nll": 1.4433423409086612, "acc": 0.5, "ece": 0.4431973397731781, "brier": 0.8877473385116084, "mean_conf": 0.9431973397731781, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.31007936507936507, "error_rate_at_0_9": 0.5, "confidence_bias": 0.4431973397731781, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9448668360710144 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9334832429885864 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.0939339000796315, "acc": 0.5, "ece": 0.16012432878157545, "brier": 0.6635327214197148, "mean_conf": 0.33987567121842455, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5417460317460316, "error_rate_at_0_9": null, "confidence_bias": -0.16012432878157545, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.34033403752243296 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.33860839375207463 } }, "score_mae": 0.6999460694543735, "ranked_probability_score": 0.22628900665691365 }, "unknowable_control_late_fee": { "n": 10, "nll": 1.0998773717268528, "acc": 0.3, "ece": 0.040559427915460855, "brier": 0.667516799962047, "mean_conf": 0.34055942791546084, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7634920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.040559427915460855, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.34070939775053277 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3402503031417637 } }, "score_mae": 0.604843196236742, "ranked_probability_score": 0.21132520467597854 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 1.3488958869964383, "acc": 0.2, "ece": 0.36076402581608485, "brier": 0.8420843440836799, "mean_conf": 0.5607640258160849, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7725396825396824, "error_rate_at_0_9": null, "confidence_bias": 0.36076402581608485, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5632423246389974 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5573770403862 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.8455590159265375, "acc": 0.5, "ece": 0.25936471819877627, "brier": 0.6308352614839883, "mean_conf": 0.7593647181987763, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5249206349206349, "error_rate_at_0_9": null, "confidence_bias": 0.25936471819877627, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.7657169103622437 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7524310350418091 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 1.1046858639299395, "acc": 0.2, "ece": 0.14331720862452646, "brier": 0.6707438425130093, "mean_conf": 0.34331720862452647, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6592063492063491, "error_rate_at_0_9": null, "confidence_bias": 0.14331720862452646, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3432188034057617 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.34267568588256836 } }, "score_mae": 0.7090615362866717, "ranked_probability_score": 0.22997851394300825 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.0987144215013314, "acc": 0.3, "ece": 0.03676625683862805, "brier": 0.6667406922057552, "mean_conf": 0.33676625683862804, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.6759920634920634, "error_rate_at_0_9": null, "confidence_bias": 0.03676625683862805, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3369380034841297 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.3366921942440329 } }, "score_mae": 0.5049051225214941, "ranked_probability_score": 0.1947439920178244 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.7539871718217989, "acc": 0.5, "ece": 0.05788718583661284, "brier": 0.5150498126641045, "mean_conf": 0.5419843572093712, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.543968253968254, "error_rate_at_0_9": null, "confidence_bias": 0.041984357209371215, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5388615412416368 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5307940205127446 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 1.0978271810968157, "acc": 0.4, "ece": 0.06276650428716868, "brier": 0.6661392699562907, "mean_conf": 0.33723349571283134, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5396428571428571, "error_rate_at_0_9": null, "confidence_bias": -0.06276650428716868, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.33723676204681396 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3366008400917053 } }, "score_mae": 0.8012357295066987, "ranked_probability_score": 0.24436618036188062 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 1.100952041539422, "acc": 0.2, "ece": 0.13708551726927687, "brier": 0.6682286858387834, "mean_conf": 0.3370855172692769, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7425396825396826, "error_rate_at_0_9": null, "confidence_bias": 0.13708551726927687, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.3369665343473394 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.3365091185721492 } }, "score_mae": 0.5030123859135118, "ranked_probability_score": 0.19485693574117507 }, "unknowable_deadline": { "n": 10, "nll": 1.0932729745942718, "acc": 0.5, "ece": 0.1596766939176677, "brier": 0.66308934703813, "mean_conf": 0.3403233060823323, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.33646825396825397, "error_rate_at_0_9": null, "confidence_bias": -0.1596766939176677, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.34032610058784485 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.3401913148221765 } }, "score_mae": 0.6997847883000105, "ranked_probability_score": 0.22606377824233923 }, "unknowable_late_fee": { "n": 10, "nll": 1.0998564930186718, "acc": 0.3, "ece": 0.04059652482758408, "brier": 0.6675052741157881, "mean_conf": 0.34059652482758407, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8566666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.04059652482758408, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.34084375676978357 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.34024547516568077 } }, "score_mae": 0.6052634239196777, "ranked_probability_score": 0.211342772788189 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.411919027968128, "acc": 0.2, "ece": 0.3920694577384348, "brier": 0.8858062838782198, "mean_conf": 0.5920694577384348, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8516666666666668, "error_rate_at_0_9": null, "confidence_bias": 0.39206945773843477, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.5985613376116553 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.5803585138970392 } } }, "unknowable_return_window": { "n": 10, "nll": 0.712707396335963, "acc": 0.5, "ece": 0.10319202542304992, "brier": 0.5191555614314189, "mean_conf": 0.60319202542305, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.451468253968254, "error_rate_at_0_9": null, "confidence_bias": 0.10319202542304995, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5948264002799988 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5866559147834778 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.1056280632292723, "acc": 0.2, "ece": 0.14401089725546173, "brier": 0.6713693029888372, "mean_conf": 0.34401089725546175, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.5142063492063491, "error_rate_at_0_9": null, "confidence_bias": 0.14401089725546173, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.34401907491627115 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.34380265045754166 } }, "score_mae": 0.7095349667106752, "ranked_probability_score": 0.23019839200180905 }, "unknowable_sla_response": { "n": 10, "nll": 1.098810293942781, "acc": 0.2, "ece": 0.13668654760232696, "brier": 0.6668048716502509, "mean_conf": 0.33668654760232697, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7516666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.13668654760232696, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.33705062645371997 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.33609532332988523 } }, "score_mae": 0.5049429415949115, "ranked_probability_score": 0.19478188122385562 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.7527647650902792, "acc": 0.5, "ece": 0.09028240692950466, "brier": 0.524918506595234, "mean_conf": 0.5902824069295046, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.09028240692950462, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.5850138046405695 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5829904588611355 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.0978988829239023, "acc": 0.4, "ece": 0.06257395543212535, "brier": 0.6661878070124657, "mean_conf": 0.3374260445678747, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6283333333333333, "error_rate_at_0_9": null, "confidence_bias": -0.06257395543212535, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.3373629946302475 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.3370372157405977 } }, "score_mae": 0.801189136540566, "ranked_probability_score": 0.2444071333460754 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.1018525227820546, "acc": 0.2, "ece": 0.13809392460331582, "brier": 0.6688339029189675, "mean_conf": 0.33809392460331583, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.9421031746031747, "error_rate_at_0_9": null, "confidence_bias": 0.13809392460331582, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.3381013672978093 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.33792778849601746 } }, "score_mae": 0.5043253809994545, "ranked_probability_score": 0.1950938413111672 } }, "variants": { "clean": { "n": 1156, "nll": 1.2305731699001068, "acc": 0.47231833910034604, "ece": 0.14245674179382123, "brier": 0.6455039234019653, "mean_conf": 0.6147750808941672, "confident_error_rate": 0.056228373702422146, "coverage_at_0_9": 0.2050173010380623, "accuracy_at_0_9": 0.7257383966244726, "coverage_at_5pct_error": 0.03633217993079585, "coverage_at_1pct_error": 0.02249134948096886, "aurc": 0.35124298519705427, "error_rate_at_0_9": 0.2742616033755274, "confidence_bias": 0.14245674179382117, "top_bins": { "0.9": { "n": 237, "errors": 65, "error_rate": 0.2742616033755274 }, "0.95": { "n": 139, "errors": 22, "error_rate": 0.15827338129496402 }, "0.99": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6245674740484429, "confidence_cutoff": 0.6070134043693542 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.5362162162162162, "confidence_cutoff": 0.3400193452835083 } }, "score_mae": 0.6655013694972904, "ranked_probability_score": 0.22143742000521746 }, "none_absent": { "n": 36, "nll": 1.8908013051700792, "acc": 0.25, "ece": 0.3700346774850222, "brier": 0.9861632251788706, "mean_conf": 0.5800092289492815, "confident_error_rate": 0.1111111111111111, "coverage_at_0_9": 0.1111111111111111, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8326778776123417, "error_rate_at_0_9": 1.0, "confidence_bias": 0.33000922894928153, "top_bins": { "0.9": { "n": 4, "errors": 4, "error_rate": 1.0 }, "0.95": { "n": 3, "errors": 3, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.531673341896758 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.2413793103448276, "confidence_cutoff": 0.4096834415386056 } } }, "none_present": { "n": 36, "nll": 1.681773723669734, "acc": 0.4722222222222222, "ece": 0.20219841092488827, "brier": 0.7177957137473535, "mean_conf": 0.6006037815462434, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.2222222222222222, "accuracy_at_0_9": 0.625, "coverage_at_5pct_error": 0.027777777777777776, "coverage_at_1pct_error": 0.027777777777777776, "aurc": 0.3942601945073396, "error_rate_at_0_9": 0.375, "confidence_bias": 0.12838155932402118, "top_bins": { "0.9": { "n": 8, "errors": 3, "error_rate": 0.375 }, "0.95": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6111111111111112, "confidence_cutoff": 0.5441543317992973 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5517241379310345, "confidence_cutoff": 0.3820274213350393 } } }, "permuted": { "n": 36, "nll": 1.4596139189319304, "acc": 0.4444444444444444, "ece": 0.2505226180720368, "brier": 0.712078793324645, "mean_conf": 0.6860688523914482, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.2777777777777778, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.05555555555555555, "coverage_at_1pct_error": 0.05555555555555555, "aurc": 0.37403462185037384, "error_rate_at_0_9": 0.2, "confidence_bias": 0.2416244079470038, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 8, "errors": 2, "error_rate": 0.25 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6111111111111112, "confidence_cutoff": 0.6979859911355527 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5172413793103449, "confidence_cutoff": 0.461902532442678 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.10844161248110817, "flip_rate": 0.1388888888888889 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 1.2391125114356256, "acc": 0.4837476099426386, "ece": 0.14362213022361828, "brier": 0.6391659534279467, "mean_conf": 0.6246297488272995, "confident_error_rate": 0.052581261950286805, "coverage_at_0_9": 0.2074569789674952, "accuracy_at_0_9": 0.7465437788018433, "coverage_at_5pct_error": 0.040152963671128104, "coverage_at_1pct_error": 0.0248565965583174, "aurc": 0.33448141017109756, "error_rate_at_0_9": 0.2534562211981567, "confidence_bias": 0.1408821388846609, "top_bins": { "0.9": { "n": 217, "errors": 55, "error_rate": 0.2534562211981567 }, "0.95": { "n": 125, "errors": 15, "error_rate": 0.12 }, "0.99": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6462715105162524, "confidence_cutoff": 0.626142047277567 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5507765830346476, "confidence_cutoff": 0.34034422991608204 } }, "score_mae": 0.6822981273891354, "ranked_probability_score": 0.22411109211700433 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 0 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Laya-base": { "panels": { "old_core": { "point": 0.5654389880952381, "exact_fraction": "15199/26880", "ci95": [ 0.5267113095238096, 0.6052836681547619 ] }, "v3_core": { "point": 0.35333333333333333, "exact_fraction": "53/150", "ci95": [ 0.3304166666666667, 0.3754166666666667 ] }, "v4": { "point": 0.5140625, "exact_fraction": "329/640", "ci95": [ 0.4665676771744132, 0.5609375 ] }, "v5": { "point": 0.6375, "exact_fraction": "51/80", "ci95": [ 0.6032948178983685, 0.6716809713450388 ] }, "transfer_v9_test": { "point": 0.5305927342256214, "exact_fraction": "555/1046", "ci95": [ 0.49951779324714507, 0.5612903225806452 ] }, "original_four_panel_mean": { "point": 0.5175837053571428, "exact_fraction": "92751/179200", "ci95": [ 0.49926016480112173, 0.5358944563265255 ] }, "weighted_mean": { "point": 0.510288314895748, "exact_fraction": "17934389/35145600", "ci95": [ 0.4942623090587989, 0.5268384804713624 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 117, "requested": 128 }, "boolean_constraints": { "correct": 28, "requested": 64 }, "dbpedia_14": { "correct": 94, "requested": 112 }, "natural_intents": { "correct": 52, "requested": 64 }, "option_carrier": { "correct": 92, "requested": 96 }, "ordinal_rubric": { "correct": 12, "requested": 64 }, "relational_composition": { "correct": 24, "requested": 96 }, "scoped_evidence": { "correct": 36, "requested": 96 }, "state_tracking": { "correct": 23, "requested": 96 }, "unknown_rejection": { "correct": 41, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 38, "requested": 80 }, "capacitated_assignment": { "correct": 51, "requested": 80 }, "constraint_assignment": { "correct": 18, "requested": 80 }, "massive_en": { "correct": 89, "requested": 120 }, "massive_zh": { "correct": 53, "requested": 120 }, "multiset_reconciliation": { "correct": 21, "requested": 80 }, "ordinal_service_loss": { "correct": 16, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 15, "requested": 80 }, "temporal_exclusion": { "correct": 10, "requested": 80 }, "transaction_recovery": { "correct": 19, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 111, "requested": 160 }, "belebele_en": { "correct": 61, "requested": 160 }, "belebele_zh": { "correct": 46, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 36, "requested": 120 }, "squad2_answerability": { "correct": 69, "requested": 120 }, "snli": { "correct": 87, "requested": 120 }, "qasc": { "correct": 114, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 555, "accuracy": 0.5305927342256214, "full_probability_metrics": { "n": 1046, "nll": 1.2274823338319296, "acc": 0.5305927342256214, "ece": 0.1290347987840952, "brier": 0.5832038000970096, "mean_conf": 0.6596275330097165, "confident_error_rate": 0.030592734225621414, "coverage_at_0_9": 0.1701720841300191, "accuracy_at_0_9": 0.8202247191011236, "coverage_at_5pct_error": 0.004780114722753346, "coverage_at_1pct_error": 0.004780114722753346, "aurc": 0.2779831915729428, "error_rate_at_0_9": 0.1797752808988764, "confidence_bias": 0.12903479878409518, "top_bins": { "0.9": { "n": 178, "errors": 32, "error_rate": 0.1797752808988764 }, "0.95": { "n": 99, "errors": 18, "error_rate": 0.18181818181818182 }, "0.99": { "n": 46, "errors": 9, "error_rate": 0.1956521739130435 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7476099426386233, "confidence_cutoff": 0.6987 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6272401433691757, "confidence_cutoff": 0.4267 } }, "score_mae": 0.6827758263793483, "ranked_probability_score": 0.22948632636102131 }, "covered_only_probability_metrics": { "n": 1046, "nll": 1.2274823338319296, "acc": 0.5305927342256214, "ece": 0.1290347987840952, "brier": 0.5832038000970096, "mean_conf": 0.6596275330097165, "confident_error_rate": 0.030592734225621414, "coverage_at_0_9": 0.1701720841300191, "accuracy_at_0_9": 0.8202247191011236, "coverage_at_5pct_error": 0.004780114722753346, "coverage_at_1pct_error": 0.004780114722753346, "aurc": 0.2779831915729428, "error_rate_at_0_9": 0.1797752808988764, "confidence_bias": 0.12903479878409518, "top_bins": { "0.9": { "n": 178, "errors": 32, "error_rate": 0.1797752808988764 }, "0.95": { "n": 99, "errors": 18, "error_rate": 0.18181818181818182 }, "0.99": { "n": 46, "errors": 9, "error_rate": 0.1956521739130435 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7476099426386233, "confidence_cutoff": 0.6987 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6272401433691757, "confidence_cutoff": 0.4267 } }, "score_mae": 0.6827758263793483, "ranked_probability_score": 0.22948632636102131 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 12, "accuracy": 0.6, "full_probability_metrics": { "n": 20, "nll": 2.1679106403070927, "acc": 0.6, "ece": 0.35507536438894877, "brier": 0.7600905626662446, "mean_conf": 0.9094453643889489, "confident_error_rate": 0.3, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.37800006323226143, "error_rate_at_0_9": 0.4, "confidence_bias": 0.3094453643889489, "top_bins": { "0.9": { "n": 15, "errors": 6, "error_rate": 0.4 }, "0.95": { "n": 12, "errors": 4, "error_rate": 0.3333333333333333 }, "0.99": { "n": 4, "errors": 1, "error_rate": 0.25 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9705 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8999 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 2.1679106403070927, "acc": 0.6, "ece": 0.35507536438894877, "brier": 0.7600905626662446, "mean_conf": 0.9094453643889489, "confident_error_rate": 0.3, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.37800006323226143, "error_rate_at_0_9": 0.4, "confidence_bias": 0.3094453643889489, "top_bins": { "0.9": { "n": 15, "errors": 6, "error_rate": 0.4 }, "0.95": { "n": 12, "errors": 4, "error_rate": 0.3333333333333333 }, "0.99": { "n": 4, "errors": 1, "error_rate": 0.25 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9705 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8999 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 11, "accuracy": 0.55, "full_probability_metrics": { "n": 20, "nll": 1.116889670298588, "acc": 0.55, "ece": 0.32998000000000005, "brier": 0.71078022, "mean_conf": 0.8082499999999999, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5361713312487305, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.25824999999999987, "top_bins": { "0.9": { "n": 7, "errors": 3, "error_rate": 0.42857142857142855 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8319 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7362 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 1.116889670298588, "acc": 0.55, "ece": 0.32998000000000005, "brier": 0.71078022, "mean_conf": 0.8082499999999999, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5361713312487305, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.25824999999999987, "top_bins": { "0.9": { "n": 7, "errors": 3, "error_rate": 0.42857142857142855 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8319 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7362 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 12, "accuracy": 0.6, "full_probability_metrics": { "n": 20, "nll": 0.6449111287341289, "acc": 0.6, "ece": 0.22739999999999994, "brier": 0.44230673, "mean_conf": 0.7885800000000001, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.2165638452964304, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.18858000000000008, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7999 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.6454 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.6449111287341289, "acc": 0.6, "ece": 0.22739999999999994, "brier": 0.44230673, "mean_conf": 0.7885800000000001, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.2165638452964304, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.18858000000000008, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7999 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.6454 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 14, "accuracy": 0.7, "full_probability_metrics": { "n": 20, "nll": 0.5006911706085153, "acc": 0.7, "ece": 0.09355500000000004, "brier": 0.328020027, "mean_conf": 0.702645, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.10157037686255951, "error_rate_at_0_9": null, "confidence_bias": 0.0026450000000000085, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.7182 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5751999999999999 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.5006911706085153, "acc": 0.7, "ece": 0.09355500000000004, "brier": 0.328020027, "mean_conf": 0.702645, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.10157037686255951, "error_rate_at_0_9": null, "confidence_bias": 0.0026450000000000085, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.7182 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5751999999999999 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 20, "accuracy": 0.625, "full_probability_metrics": { "n": 32, "nll": 0.7815065853012877, "acc": 0.625, "ece": 0.23545624999999998, "brier": 0.544465325, "mean_conf": 0.72265625, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5476544483724437, "error_rate_at_0_9": null, "confidence_bias": 0.09765625, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5625, "confidence_cutoff": 0.7594 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6538461538461539, "confidence_cutoff": 0.5368 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.7815065853012877, "acc": 0.625, "ece": 0.23545624999999998, "brier": 0.544465325, "mean_conf": 0.72265625, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5476544483724437, "error_rate_at_0_9": null, "confidence_bias": 0.09765625, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5625, "confidence_cutoff": 0.7594 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6538461538461539, "confidence_cutoff": 0.5368 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 14, "accuracy": 0.4375, "full_probability_metrics": { "n": 32, "nll": 0.9210365756459901, "acc": 0.4375, "ece": 0.2895312500000001, "brier": 0.6930435024999999, "mean_conf": 0.72113125, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.4868870894891721, "error_rate_at_0_9": 0.0, "confidence_bias": 0.28363125, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4375, "confidence_cutoff": 0.7383 }, "0.8": { "coverage": 0.8125, "accuracy": 0.4230769230769231, "confidence_cutoff": 0.6007 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.9210365756459901, "acc": 0.4375, "ece": 0.2895312500000001, "brier": 0.6930435024999999, "mean_conf": 0.72113125, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.4868870894891721, "error_rate_at_0_9": 0.0, "confidence_bias": 0.28363125, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4375, "confidence_cutoff": 0.7383 }, "0.8": { "coverage": 0.8125, "accuracy": 0.4230769230769231, "confidence_cutoff": 0.6007 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 17, "accuracy": 0.53125, "full_probability_metrics": { "n": 32, "nll": 0.9706960277772426, "acc": 0.53125, "ece": 0.27430312500000004, "brier": 0.6578405431250001, "mean_conf": 0.7362906250000001, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5800520110903649, "error_rate_at_0_9": 0.75, "confidence_bias": 0.2050406250000001, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7745 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.5735 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.9706960277772426, "acc": 0.53125, "ece": 0.27430312500000004, "brier": 0.6578405431250001, "mean_conf": 0.7362906250000001, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5800520110903649, "error_rate_at_0_9": 0.75, "confidence_bias": 0.2050406250000001, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7745 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.5735 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 27, "accuracy": 0.675, "full_probability_metrics": { "n": 40, "nll": 0.5455872965191679, "acc": 0.675, "ece": 0.1403649999999999, "brier": 0.376447768, "mean_conf": 0.80847, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.575, "aurc": 0.07823763467343041, "error_rate_at_0_9": null, "confidence_bias": 0.13346999999999998, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8557 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.7831 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 0.5455872965191679, "acc": 0.675, "ece": 0.1403649999999999, "brier": 0.376447768, "mean_conf": 0.80847, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.575, "aurc": 0.07823763467343041, "error_rate_at_0_9": null, "confidence_bias": 0.13346999999999998, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8557 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.7831 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 14, "accuracy": 0.35, "full_probability_metrics": { "n": 40, "nll": 1.1563305693979475, "acc": 0.35, "ece": 0.1855862491980375, "brier": 0.7183618101913636, "mean_conf": 0.5202867366005424, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7051511051725096, "error_rate_at_0_9": null, "confidence_bias": 0.1702867366005424, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.5162 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.4551 } }, "score_mae": 0.7228034603678346, "ranked_probability_score": 0.23046055531514115 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.1563305693979475, "acc": 0.35, "ece": 0.1855862491980375, "brier": 0.7183618101913636, "mean_conf": 0.5202867366005424, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7051511051725096, "error_rate_at_0_9": null, "confidence_bias": 0.1702867366005424, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.5162 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.4551 } }, "score_mae": 0.7228034603678346, "ranked_probability_score": 0.23046055531514115 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 50, "accuracy": 0.625, "full_probability_metrics": { "n": 80, "nll": 1.9539633393631164, "acc": 0.625, "ece": 0.2589496108924017, "brier": 0.6106731427871583, "mean_conf": 0.8839496108924019, "confident_error_rate": 0.15, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 0.7818181818181819, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.2472313323243478, "error_rate_at_0_9": 0.21818181818181817, "confidence_bias": 0.25894961089240187, "top_bins": { "0.9": { "n": 55, "errors": 12, "error_rate": 0.21818181818181817 }, "0.95": { "n": 49, "errors": 9, "error_rate": 0.1836734693877551 }, "0.99": { "n": 35, "errors": 8, "error_rate": 0.22857142857142856 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9865 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.7808 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.9539633393631164, "acc": 0.625, "ece": 0.2589496108924017, "brier": 0.6106731427871583, "mean_conf": 0.8839496108924019, "confident_error_rate": 0.15, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 0.7818181818181819, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.2472313323243478, "error_rate_at_0_9": 0.21818181818181817, "confidence_bias": 0.25894961089240187, "top_bins": { "0.9": { "n": 55, "errors": 12, "error_rate": 0.21818181818181817 }, "0.95": { "n": 49, "errors": 9, "error_rate": 0.1836734693877551 }, "0.99": { "n": 35, "errors": 8, "error_rate": 0.22857142857142856 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9865 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.7808 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 18, "accuracy": 0.225, "full_probability_metrics": { "n": 80, "nll": 1.5401243227571941, "acc": 0.225, "ece": 0.24747234397808598, "brier": 0.8082065105463727, "mean_conf": 0.45740301554524265, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.6335801473003778, "error_rate_at_0_9": 0.0, "confidence_bias": 0.23240301554524265, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.275, "confidence_cutoff": 0.419 }, "0.8": { "coverage": 0.8, "accuracy": 0.265625, "confidence_cutoff": 0.347 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.5401243227571941, "acc": 0.225, "ece": 0.24747234397808598, "brier": 0.8082065105463727, "mean_conf": 0.45740301554524265, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.6335801473003778, "error_rate_at_0_9": 0.0, "confidence_bias": 0.23240301554524265, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.275, "confidence_cutoff": 0.419 }, "0.8": { "coverage": 0.8, "accuracy": 0.265625, "confidence_cutoff": 0.347 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 22, "accuracy": 0.11, "full_probability_metrics": { "n": 200, "nll": 2.625007900734808, "acc": 0.11, "ece": 0.25029851592705105, "brier": 1.0134749687447435, "mean_conf": 0.360298515927051, "confident_error_rate": 0.015, "coverage_at_0_9": 0.015, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8879816989545773, "error_rate_at_0_9": 1.0, "confidence_bias": 0.250298515927051, "top_bins": { "0.9": { "n": 3, "errors": 3, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.12, "confidence_cutoff": 0.2986298629862986 }, "0.8": { "coverage": 0.8, "accuracy": 0.13125, "confidence_cutoff": 0.2053 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.625007900734808, "acc": 0.11, "ece": 0.25029851592705105, "brier": 1.0134749687447435, "mean_conf": 0.360298515927051, "confident_error_rate": 0.015, "coverage_at_0_9": 0.015, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8879816989545773, "error_rate_at_0_9": 1.0, "confidence_bias": 0.250298515927051, "top_bins": { "0.9": { "n": 3, "errors": 3, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.12, "confidence_cutoff": 0.2986298629862986 }, "0.8": { "coverage": 0.8, "accuracy": 0.13125, "confidence_cutoff": 0.2053 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 69, "accuracy": 0.8625, "full_probability_metrics": { "n": 80, "nll": 0.4109330235284993, "acc": 0.8625, "ece": 0.08666000000000004, "brier": 0.2518360155, "mean_conf": 0.817865, "confident_error_rate": 0.025, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.9130434782608695, "coverage_at_5pct_error": 0.1625, "coverage_at_1pct_error": 0.1625, "aurc": 0.0912037829160317, "error_rate_at_0_9": 0.08695652173913043, "confidence_bias": -0.04463500000000009, "top_bins": { "0.9": { "n": 23, "errors": 2, "error_rate": 0.08695652173913043 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.8452 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.7158 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4109330235284993, "acc": 0.8625, "ece": 0.08666000000000004, "brier": 0.2518360155, "mean_conf": 0.817865, "confident_error_rate": 0.025, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.9130434782608695, "coverage_at_5pct_error": 0.1625, "coverage_at_1pct_error": 0.1625, "aurc": 0.0912037829160317, "error_rate_at_0_9": 0.08695652173913043, "confidence_bias": -0.04463500000000009, "top_bins": { "0.9": { "n": 23, "errors": 2, "error_rate": 0.08695652173913043 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.8452 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.7158 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 64, "accuracy": 0.8, "full_probability_metrics": { "n": 80, "nll": 0.4410695105405936, "acc": 0.8, "ece": 0.02513125000000002, "brier": 0.28695005924999994, "mean_conf": 0.79861625, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.225, "accuracy_at_0_9": 0.9444444444444444, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.1875, "aurc": 0.091529616160848, "error_rate_at_0_9": 0.05555555555555555, "confidence_bias": -0.001383750000000017, "top_bins": { "0.9": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.8259 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.6784 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4410695105405936, "acc": 0.8, "ece": 0.02513125000000002, "brier": 0.28695005924999994, "mean_conf": 0.79861625, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.225, "accuracy_at_0_9": 0.9444444444444444, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.1875, "aurc": 0.091529616160848, "error_rate_at_0_9": 0.05555555555555555, "confidence_bias": -0.001383750000000017, "top_bins": { "0.9": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.8259 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.6784 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 72, "accuracy": 0.9, "full_probability_metrics": { "n": 80, "nll": 0.38696245586895806, "acc": 0.9, "ece": 0.1526194004867314, "brier": 0.18052963398172583, "mean_conf": 0.7548080995132684, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.575, "aurc": 0.019154830834346076, "error_rate_at_0_9": 0.0, "confidence_bias": -0.14519190048673158, "top_bins": { "0.9": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8173817381738174 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.5863 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.38696245586895806, "acc": 0.9, "ece": 0.1526194004867314, "brier": 0.18052963398172583, "mean_conf": 0.7548080995132684, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.575, "aurc": 0.019154830834346076, "error_rate_at_0_9": 0.0, "confidence_bias": -0.14519190048673158, "top_bins": { "0.9": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8173817381738174 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.5863 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 65, "accuracy": 0.8125, "full_probability_metrics": { "n": 80, "nll": 0.38971269291666005, "acc": 0.8125, "ece": 0.09374749999999998, "brier": 0.24490397799999997, "mean_conf": 0.814805, "confident_error_rate": 0.0, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6375, "coverage_at_1pct_error": 0.3125, "aurc": 0.05859934221337241, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0023050000000000015, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.8368 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.7424 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.38971269291666005, "acc": 0.8125, "ece": 0.09374749999999998, "brier": 0.24490397799999997, "mean_conf": 0.814805, "confident_error_rate": 0.0, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6375, "coverage_at_1pct_error": 0.3125, "aurc": 0.05859934221337241, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0023050000000000015, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.8368 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.7424 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 9, "accuracy": 0.9, "full_probability_metrics": { "n": 10, "nll": 0.3087863578303291, "acc": 0.9, "ece": 0.25366, "brier": 0.16031426800000004, "mean_conf": 0.75274, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.14726000000000006, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7759 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.6869 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.3087863578303291, "acc": 0.9, "ece": 0.25366, "brier": 0.16031426800000004, "mean_conf": 0.75274, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.14726000000000006, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7759 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.6869 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 7, "accuracy": 0.7, "full_probability_metrics": { "n": 10, "nll": 0.5003682990113598, "acc": 0.7, "ece": 0.32675000000000004, "brier": 0.34025959400000005, "mean_conf": 0.7882499999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.09150793650793651, "error_rate_at_0_9": null, "confidence_bias": 0.08824999999999994, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8452 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7342 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.5003682990113598, "acc": 0.7, "ece": 0.32675000000000004, "brier": 0.34025959400000005, "mean_conf": 0.7882499999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.09150793650793651, "error_rate_at_0_9": null, "confidence_bias": 0.08824999999999994, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8452 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7342 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.0640636729305286, "acc": 0.3, "ece": 0.308550494089105, "brier": 0.6688734558322902, "mean_conf": 0.5298704940891048, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5496428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.22987049408910482, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5468000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5085 } }, "score_mae": 0.6655700487869005, "ranked_probability_score": 0.21164108259840736 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0640636729305286, "acc": 0.3, "ece": 0.308550494089105, "brier": 0.6688734558322902, "mean_conf": 0.5298704940891048, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5496428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.22987049408910482, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5468000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5085 } }, "score_mae": 0.6655700487869005, "ranked_probability_score": 0.21164108259840736 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.3039416262137309, "acc": 0.4, "ece": 0.36599085798530856, "brier": 0.8409417947276431, "mean_conf": 0.7659908579853086, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6979761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.3659908579853086, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7754 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7359 } }, "score_mae": 0.5934694769318948, "ranked_probability_score": 0.23073304286701618 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3039416262137309, "acc": 0.4, "ece": 0.36599085798530856, "brier": 0.8409417947276431, "mean_conf": 0.7659908579853086, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6979761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.3659908579853086, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7754 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7359 } }, "score_mae": 0.5934694769318948, "ranked_probability_score": 0.23073304286701618 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 1.0763716721583632, "acc": 0.6, "ece": 0.14274403054604032, "brier": 0.6375434544101031, "mean_conf": 0.45725596945395974, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.33218253968253963, "error_rate_at_0_9": null, "confidence_bias": -0.14274403054604023, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.4448444844484448 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.41474147414741475 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0763716721583632, "acc": 0.6, "ece": 0.14274403054604032, "brier": 0.6375434544101031, "mean_conf": 0.45725596945395974, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.33218253968253963, "error_rate_at_0_9": null, "confidence_bias": -0.14274403054604023, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.4448444844484448 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.41474147414741475 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 6, "accuracy": 0.6, "full_probability_metrics": { "n": 10, "nll": 0.6754330719907694, "acc": 0.6, "ece": 0.02111999999999996, "brier": 0.48173350400000003, "mean_conf": 0.6083, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.47146825396825404, "error_rate_at_0_9": null, "confidence_bias": 0.008299999999999974, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6495 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5583 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.6754330719907694, "acc": 0.6, "ece": 0.02111999999999996, "brier": 0.48173350400000003, "mean_conf": 0.6083, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.47146825396825404, "error_rate_at_0_9": null, "confidence_bias": 0.008299999999999974, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6495 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5583 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.3175089868474639, "acc": 0.3, "ece": 0.2305454094590541, "brier": 0.775904734560073, "mean_conf": 0.530545409459054, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5129761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.23054540945905405, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5456 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.488 } }, "score_mae": 0.804096690330967, "ranked_probability_score": 0.271071591388441 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3175089868474639, "acc": 0.3, "ece": 0.2305454094590541, "brier": 0.775904734560073, "mean_conf": 0.530545409459054, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5129761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.23054540945905405, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5456 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.488 } }, "score_mae": 0.804096690330967, "ranked_probability_score": 0.271071591388441 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.0847826778426977, "acc": 0.5, "ece": 0.28567238351002383, "brier": 0.6796589087585527, "mean_conf": 0.6792484271143844, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.456031746031746, "error_rate_at_0_9": null, "confidence_bias": 0.17924842711438438, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7272000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5819 } }, "score_mae": 0.5022111220782113, "ranked_probability_score": 0.18802835782711252 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0847826778426977, "acc": 0.5, "ece": 0.28567238351002383, "brier": 0.6796589087585527, "mean_conf": 0.6792484271143844, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.456031746031746, "error_rate_at_0_9": null, "confidence_bias": 0.17924842711438438, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7272000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5819 } }, "score_mae": 0.5022111220782113, "ranked_probability_score": 0.18802835782711252 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.844674013055144, "acc": 0.5, "ece": 0.18678372137213722, "brier": 0.550189334993938, "mean_conf": 0.5734562786278629, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.448531746031746, "error_rate_at_0_9": null, "confidence_bias": 0.07345627862786286, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5563 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5364 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.844674013055144, "acc": 0.5, "ece": 0.18678372137213722, "brier": 0.550189334993938, "mean_conf": 0.5734562786278629, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.448531746031746, "error_rate_at_0_9": null, "confidence_bias": 0.07345627862786286, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5563 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5364 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 2, "accuracy": 0.2, "full_probability_metrics": { "n": 10, "nll": 1.4356097418369373, "acc": 0.2, "ece": 0.5537333752922339, "brier": 0.9508376461251846, "mean_conf": 0.6735333752922338, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8359920634920635, "error_rate_at_0_9": null, "confidence_bias": 0.47353337529223377, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6935 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.6234 } }, "score_mae": 0.7633107298296073, "ranked_probability_score": 0.2746908143419903 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.4356097418369373, "acc": 0.2, "ece": 0.5537333752922339, "brier": 0.9508376461251846, "mean_conf": 0.6735333752922338, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8359920634920635, "error_rate_at_0_9": null, "confidence_bias": 0.47353337529223377, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6935 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.6234 } }, "score_mae": 0.7633107298296073, "ranked_probability_score": 0.2746908143419903 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.0779891371226018, "acc": 0.4, "ece": 0.2564253394660534, "brier": 0.6558153212245438, "mean_conf": 0.5031853394660534, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4594841269841269, "error_rate_at_0_9": null, "confidence_bias": 0.10318533946605335, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5123 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4308 } }, "score_mae": 0.6078863543645636, "ranked_probability_score": 0.19685615332668166 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.0779891371226018, "acc": 0.4, "ece": 0.2564253394660534, "brier": 0.6558153212245438, "mean_conf": 0.5031853394660534, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4594841269841269, "error_rate_at_0_9": null, "confidence_bias": 0.10318533946605335, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5123 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4308 } }, "score_mae": 0.6078863543645636, "ranked_probability_score": 0.19685615332668166 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.548287037037037, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.4444444444444444, "flip_rate": 0.19444444444444445, "covered_only_flip_rate": 0.19444444444444445, "half_l1": 0.09281195829606463 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.6796175646695757, "share_at_0_9": 0.0, "normalized_entropy": 0.7497419801552645 }, "covered_only_confidence_metrics": { "mean_max_p": 0.6796175646695757, "share_at_0_9": 0.0, "normalized_entropy": 0.7497419801552645 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 54, "accuracy": 0.4909090909090909, "full_probability_metrics": { "n": 110, "nll": 0.9717753869854477, "acc": 0.4909090909090909, "ece": 0.1386487410443602, "brier": 0.6129156378756662, "mean_conf": 0.6238523774079964, "confident_error_rate": 0.0, "coverage_at_0_9": 0.00909090909090909, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.08181818181818182, "coverage_at_1pct_error": 0.08181818181818182, "aurc": 0.3807500990034818, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13294328649890547, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6071 }, "0.8": { "coverage": 0.8, "accuracy": 0.5227272727272727, "confidence_cutoff": 0.5074 } }, "score_mae": 0.6560907370536906, "ranked_probability_score": 0.22883684039160815 }, "covered_only_probability_metrics": { "n": 110, "nll": 0.9717753869854477, "acc": 0.4909090909090909, "ece": 0.1386487410443602, "brier": 0.6129156378756662, "mean_conf": 0.6238523774079964, "confident_error_rate": 0.0, "coverage_at_0_9": 0.00909090909090909, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.08181818181818182, "coverage_at_1pct_error": 0.08181818181818182, "aurc": 0.3807500990034818, "error_rate_at_0_9": 0.0, "confidence_bias": 0.13294328649890547, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6071 }, "0.8": { "coverage": 0.8, "accuracy": 0.5227272727272727, "confidence_cutoff": 0.5074 } }, "score_mae": 0.6560907370536906, "ranked_probability_score": 0.22883684039160815 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": -0.05576518726157914 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 34 }, "complete_unchanged_upstream_report": { "objective": -1.0089948950792489, "paired_flip": { "pairs": 64, "flip_rate": 0.25, "both_correct_rate": 0.171875, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.5416666666666666 }, "unknowable": { "n": 110, "mean_max_p": 0.6796175646695757, "share_at_0_9": 0.0, "control_mean_max_p": 0.6238523774079964, "control_share_at_0_9": 0.00909090909090909, "control_acc": 0.4909090909090909, "paired_confidence_drop": -0.05576518726157914, "share_less_confident_than_control": 0.3090909090909091 }, "clean": { "n": 1046, "nll": 1.2274823338319296, "acc": 0.5305927342256214, "ece": 0.1290347987840952, "brier": 0.5832038000970096, "mean_conf": 0.6596275330097165, "confident_error_rate": 0.030592734225621414, "coverage_at_0_9": 0.1701720841300191, "accuracy_at_0_9": 0.8202247191011236, "coverage_at_5pct_error": 0.004780114722753346, "coverage_at_1pct_error": 0.004780114722753346, "aurc": 0.2779831915729428, "error_rate_at_0_9": 0.1797752808988764, "confidence_bias": 0.12903479878409518, "top_bins": { "0.9": { "n": 178, "errors": 32, "error_rate": 0.1797752808988764 }, "0.95": { "n": 99, "errors": 18, "error_rate": 0.18181818181818182 }, "0.99": { "n": 46, "errors": 9, "error_rate": 0.1956521739130435 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7476099426386233, "confidence_cutoff": 0.6987 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6272401433691757, "confidence_cutoff": 0.4267 } }, "score_mae": 0.6827758263793483, "ranked_probability_score": 0.22948632636102131 }, "tasks": { "buried_emotion": { "n": 20, "nll": 2.1679106403070927, "acc": 0.6, "ece": 0.35507536438894877, "brier": 0.7600905626662446, "mean_conf": 0.9094453643889489, "confident_error_rate": 0.3, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.6, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.37800006323226143, "error_rate_at_0_9": 0.4, "confidence_bias": 0.3094453643889489, "top_bins": { "0.9": { "n": 15, "errors": 6, "error_rate": 0.4 }, "0.95": { "n": 12, "errors": 4, "error_rate": 0.3333333333333333 }, "0.99": { "n": 4, "errors": 1, "error_rate": 0.25 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.9705 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8999 } } }, "buried_paws": { "n": 20, "nll": 1.116889670298588, "acc": 0.55, "ece": 0.32998000000000005, "brier": 0.71078022, "mean_conf": 0.8082499999999999, "confident_error_rate": 0.15, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 0.5714285714285714, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5361713312487305, "error_rate_at_0_9": 0.42857142857142855, "confidence_bias": 0.25824999999999987, "top_bins": { "0.9": { "n": 7, "errors": 3, "error_rate": 0.42857142857142855 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.8319 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7362 } } }, "buried_qnli": { "n": 20, "nll": 0.6449111287341289, "acc": 0.6, "ece": 0.22739999999999994, "brier": 0.44230673, "mean_conf": 0.7885800000000001, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.2165638452964304, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.18858000000000008, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7999 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.6454 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.5006911706085153, "acc": 0.7, "ece": 0.09355500000000004, "brier": 0.328020027, "mean_conf": 0.702645, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.10157037686255951, "error_rate_at_0_9": null, "confidence_bias": 0.0026450000000000085, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.7182 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.5751999999999999 } } }, "composition_final_combination": { "n": 32, "nll": 0.7815065853012877, "acc": 0.625, "ece": 0.23545624999999998, "brier": 0.544465325, "mean_conf": 0.72265625, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5476544483724437, "error_rate_at_0_9": null, "confidence_bias": 0.09765625, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5625, "confidence_cutoff": 0.7594 }, "0.8": { "coverage": 0.8125, "accuracy": 0.6538461538461539, "confidence_cutoff": 0.5368 } } }, "composition_final_exception": { "n": 32, "nll": 0.9210365756459901, "acc": 0.4375, "ece": 0.2895312500000001, "brier": 0.6930435024999999, "mean_conf": 0.72113125, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.09375, "coverage_at_1pct_error": 0.09375, "aurc": 0.4868870894891721, "error_rate_at_0_9": 0.0, "confidence_bias": 0.28363125, "top_bins": { "0.9": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4375, "confidence_cutoff": 0.7383 }, "0.8": { "coverage": 0.8125, "accuracy": 0.4230769230769231, "confidence_cutoff": 0.6007 } } }, "composition_final_negation": { "n": 32, "nll": 0.9706960277772426, "acc": 0.53125, "ece": 0.27430312500000004, "brier": 0.6578405431250001, "mean_conf": 0.7362906250000001, "confident_error_rate": 0.09375, "coverage_at_0_9": 0.125, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5800520110903649, "error_rate_at_0_9": 0.75, "confidence_bias": 0.2050406250000001, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5, "confidence_cutoff": 0.7745 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.5735 } } }, "contrastive_authorization": { "n": 40, "nll": 0.5455872965191679, "acc": 0.675, "ece": 0.1403649999999999, "brier": 0.376447768, "mean_conf": 0.80847, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.575, "aurc": 0.07823763467343041, "error_rate_at_0_9": null, "confidence_bias": 0.13346999999999998, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8557 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.7831 } } }, "contrastive_deadline": { "n": 40, "nll": 1.1563305693979475, "acc": 0.35, "ece": 0.1855862491980375, "brier": 0.7183618101913636, "mean_conf": 0.5202867366005424, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7051511051725096, "error_rate_at_0_9": null, "confidence_bias": 0.1702867366005424, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.3, "confidence_cutoff": 0.5162 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.4551 } }, "score_mae": 0.7228034603678346, "ranked_probability_score": 0.23046055531514115 }, "emotion": { "n": 80, "nll": 1.9539633393631164, "acc": 0.625, "ece": 0.2589496108924017, "brier": 0.6106731427871583, "mean_conf": 0.8839496108924019, "confident_error_rate": 0.15, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 0.7818181818181819, "coverage_at_5pct_error": 0.025, "coverage_at_1pct_error": 0.025, "aurc": 0.2472313323243478, "error_rate_at_0_9": 0.21818181818181817, "confidence_bias": 0.25894961089240187, "top_bins": { "0.9": { "n": 55, "errors": 12, "error_rate": 0.21818181818181817 }, "0.95": { "n": 49, "errors": 9, "error_rate": 0.1836734693877551 }, "0.99": { "n": 35, "errors": 8, "error_rate": 0.22857142857142856 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9865 }, "0.8": { "coverage": 0.8, "accuracy": 0.703125, "confidence_cutoff": 0.7808 } } }, "mmlu": { "n": 80, "nll": 1.5401243227571941, "acc": 0.225, "ece": 0.24747234397808598, "brier": 0.8082065105463727, "mean_conf": 0.45740301554524265, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0125, "coverage_at_1pct_error": 0.0125, "aurc": 0.6335801473003778, "error_rate_at_0_9": 0.0, "confidence_bias": 0.23240301554524265, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.275, "confidence_cutoff": 0.419 }, "0.8": { "coverage": 0.8, "accuracy": 0.265625, "confidence_cutoff": 0.347 } } }, "mmlu_pro": { "n": 200, "nll": 2.625007900734808, "acc": 0.11, "ece": 0.25029851592705105, "brier": 1.0134749687447435, "mean_conf": 0.360298515927051, "confident_error_rate": 0.015, "coverage_at_0_9": 0.015, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8879816989545773, "error_rate_at_0_9": 1.0, "confidence_bias": 0.250298515927051, "top_bins": { "0.9": { "n": 3, "errors": 3, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.12, "confidence_cutoff": 0.2986298629862986 }, "0.8": { "coverage": 0.8, "accuracy": 0.13125, "confidence_cutoff": 0.2053 } } }, "paws": { "n": 80, "nll": 0.4109330235284993, "acc": 0.8625, "ece": 0.08666000000000004, "brier": 0.2518360155, "mean_conf": 0.817865, "confident_error_rate": 0.025, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.9130434782608695, "coverage_at_5pct_error": 0.1625, "coverage_at_1pct_error": 0.1625, "aurc": 0.0912037829160317, "error_rate_at_0_9": 0.08695652173913043, "confidence_bias": -0.04463500000000009, "top_bins": { "0.9": { "n": 23, "errors": 2, "error_rate": 0.08695652173913043 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.8452 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.7158 } } }, "qnli": { "n": 80, "nll": 0.4410695105405936, "acc": 0.8, "ece": 0.02513125000000002, "brier": 0.28695005924999994, "mean_conf": 0.79861625, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.225, "accuracy_at_0_9": 0.9444444444444444, "coverage_at_5pct_error": 0.35, "coverage_at_1pct_error": 0.1875, "aurc": 0.091529616160848, "error_rate_at_0_9": 0.05555555555555555, "confidence_bias": -0.001383750000000017, "top_bins": { "0.9": { "n": 18, "errors": 1, "error_rate": 0.05555555555555555 }, "0.95": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.8259 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.6784 } } }, "sciq": { "n": 80, "nll": 0.38696245586895806, "acc": 0.9, "ece": 0.1526194004867314, "brier": 0.18052963398172583, "mean_conf": 0.7548080995132684, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.575, "aurc": 0.019154830834346076, "error_rate_at_0_9": 0.0, "confidence_bias": -0.14519190048673158, "top_bins": { "0.9": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8173817381738174 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.5863 } } }, "tweet_offensive": { "n": 80, "nll": 0.38971269291666005, "acc": 0.8125, "ece": 0.09374749999999998, "brier": 0.24490397799999997, "mean_conf": 0.814805, "confident_error_rate": 0.0, "coverage_at_0_9": 0.275, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6375, "coverage_at_1pct_error": 0.3125, "aurc": 0.05859934221337241, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0023050000000000015, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.8368 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.7424 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.9405167670745997, "acc": 0.5, "ece": 0.28193999999999997, "brier": 0.680664748, "mean_conf": 0.78194, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.28193999999999997, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.7957 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7727999999999999 } } }, "unknowable_authorization": { "n": 10, "nll": 0.9237313594257446, "acc": 0.5, "ece": 0.30334, "brier": 0.684434568, "mean_conf": 0.8033399999999998, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5249206349206349, "error_rate_at_0_9": null, "confidence_bias": 0.30333999999999983, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.8 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7962 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.3087863578303291, "acc": 0.9, "ece": 0.25366, "brier": 0.16031426800000004, "mean_conf": 0.75274, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9, "coverage_at_1pct_error": 0.9, "aurc": 0.01, "error_rate_at_0_9": 0.0, "confidence_bias": -0.14726000000000006, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.7759 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.6869 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.5003682990113598, "acc": 0.7, "ece": 0.32675000000000004, "brier": 0.34025959400000005, "mean_conf": 0.7882499999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "aurc": 0.09150793650793651, "error_rate_at_0_9": null, "confidence_bias": 0.08824999999999994, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8452 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.7342 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.0640636729305286, "acc": 0.3, "ece": 0.308550494089105, "brier": 0.6688734558322902, "mean_conf": 0.5298704940891048, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5496428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.22987049408910482, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5468000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5085 } }, "score_mae": 0.6655700487869005, "ranked_probability_score": 0.21164108259840736 }, "unknowable_control_late_fee": { "n": 10, "nll": 1.3039416262137309, "acc": 0.4, "ece": 0.36599085798530856, "brier": 0.8409417947276431, "mean_conf": 0.7659908579853086, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6979761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.3659908579853086, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7754 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7359 } }, "score_mae": 0.5934694769318948, "ranked_probability_score": 0.23073304286701618 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 1.0763716721583632, "acc": 0.6, "ece": 0.14274403054604032, "brier": 0.6375434544101031, "mean_conf": 0.45725596945395974, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.33218253968253963, "error_rate_at_0_9": null, "confidence_bias": -0.14274403054604023, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.4448444844484448 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.41474147414741475 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.6754330719907694, "acc": 0.6, "ece": 0.02111999999999996, "brier": 0.48173350400000003, "mean_conf": 0.6083, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.47146825396825404, "error_rate_at_0_9": null, "confidence_bias": 0.008299999999999974, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6495 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5583 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 1.3175089868474639, "acc": 0.3, "ece": 0.2305454094590541, "brier": 0.775904734560073, "mean_conf": 0.530545409459054, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.5129761904761905, "error_rate_at_0_9": null, "confidence_bias": 0.23054540945905405, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5456 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.488 } }, "score_mae": 0.804096690330967, "ranked_probability_score": 0.271071591388441 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.0847826778426977, "acc": 0.5, "ece": 0.28567238351002383, "brier": 0.6796589087585527, "mean_conf": 0.6792484271143844, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.456031746031746, "error_rate_at_0_9": null, "confidence_bias": 0.17924842711438438, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.7272000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5819 } }, "score_mae": 0.5022111220782113, "ranked_probability_score": 0.18802835782711252 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.844674013055144, "acc": 0.5, "ece": 0.18678372137213722, "brier": 0.550189334993938, "mean_conf": 0.5734562786278629, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.448531746031746, "error_rate_at_0_9": null, "confidence_bias": 0.07345627862786286, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.5563 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5364 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 1.4356097418369373, "acc": 0.2, "ece": 0.5537333752922339, "brier": 0.9508376461251846, "mean_conf": 0.6735333752922338, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8359920634920635, "error_rate_at_0_9": null, "confidence_bias": 0.47353337529223377, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6935 }, "0.8": { "coverage": 0.8, "accuracy": 0.125, "confidence_cutoff": 0.6234 } }, "score_mae": 0.7633107298296073, "ranked_probability_score": 0.2746908143419903 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 1.0779891371226018, "acc": 0.4, "ece": 0.2564253394660534, "brier": 0.6558153212245438, "mean_conf": 0.5031853394660534, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4594841269841269, "error_rate_at_0_9": null, "confidence_bias": 0.10318533946605335, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5123 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4308 } }, "score_mae": 0.6078863543645636, "ranked_probability_score": 0.19685615332668166 }, "unknowable_deadline": { "n": 10, "nll": 1.2770891852633914, "acc": 0.3, "ece": 0.3261042349352423, "brier": 0.7993617443749649, "mean_conf": 0.6261042349352424, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7589285714285715, "error_rate_at_0_9": null, "confidence_bias": 0.3261042349352424, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.6145 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5874 } }, "score_mae": 0.6985622898050229, "ranked_probability_score": 0.23995565046765405 }, "unknowable_late_fee": { "n": 10, "nll": 1.4358541172444246, "acc": 0.4, "ece": 0.3659356895689569, "brier": 0.8712905060454645, "mean_conf": 0.765935689568957, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5783333333333334, "error_rate_at_0_9": null, "confidence_bias": 0.36593568956895695, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.7844784478447845 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.7405 } }, "score_mae": 0.6333400000000001, "ranked_probability_score": 0.24595097550943595 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.199569405134402, "acc": 0.4, "ece": 0.0471183438343834, "brier": 0.7188038963980162, "mean_conf": 0.4471183438343834, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7283333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.0471183438343834, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.4415 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4306430643064306 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7198852900715804, "acc": 0.5, "ece": 0.12231000000000002, "brier": 0.5242700619999999, "mean_conf": 0.62231, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5106349206349206, "error_rate_at_0_9": null, "confidence_bias": 0.12231000000000003, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.629 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5863 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.9869800907640685, "acc": 0.3, "ece": 0.4413640285971403, "brier": 1.075564639138196, "mean_conf": 0.7153640285971402, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8807142857142857, "error_rate_at_0_9": null, "confidence_bias": 0.4153640285971402, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.0, "confidence_cutoff": 0.7407 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.6239 } }, "score_mae": 0.8406173452654736, "ranked_probability_score": 0.3255902580125976 }, "unknowable_sla_response": { "n": 10, "nll": 1.2533470804324627, "acc": 0.5, "ece": 0.27301594159415943, "brier": 0.7492403171068907, "mean_conf": 0.7730159415941594, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.27301594159415943, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.7970797079707971 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.7967000000000001 } }, "score_mae": 0.51928, "ranked_probability_score": 0.21089106326464396 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.7887180713418591, "acc": 0.5, "ece": 0.07446387638763877, "brier": 0.5310002064502596, "mean_conf": 0.5744638763876387, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.07446387638763874, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.5797579757975797 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5523000000000001 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.7432689764991256, "acc": 0.2, "ece": 0.5406000000000001, "brier": 1.095487968, "mean_conf": 0.7405999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8766666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.5406, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.7145 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7115 } }, "score_mae": 0.8143799999999999, "ranked_probability_score": 0.315112492 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.206572446329321, "acc": 0.5, "ece": 0.18955956023799567, "brier": 0.7131215419428969, "mean_conf": 0.6256010964478109, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6518650793650794, "error_rate_at_0_9": null, "confidence_bias": 0.12560109644781092, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.655034496550345 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5908590859085909 } }, "score_mae": 0.6102188062865881, "ranked_probability_score": 0.21379398962551882 } }, "variants": { "clean": { "n": 1156, "nll": 1.2272507345017372, "acc": 0.5198961937716263, "ece": 0.14163350488046444, "brier": 0.6007470388201028, "mean_conf": 0.6615296986520908, "confident_error_rate": 0.02768166089965398, "coverage_at_0_9": 0.15397923875432526, "accuracy_at_0_9": 0.8202247191011236, "coverage_at_5pct_error": 0.004325259515570935, "coverage_at_1pct_error": 0.004325259515570935, "aurc": 0.2950672123661356, "error_rate_at_0_9": 0.1797752808988764, "confidence_bias": 0.14163350488046444, "top_bins": { "0.9": { "n": 178, "errors": 32, "error_rate": 0.1797752808988764 }, "0.95": { "n": 99, "errors": 18, "error_rate": 0.18181818181818182 }, "0.99": { "n": 46, "errors": 9, "error_rate": 0.1956521739130435 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7145328719723183, "confidence_cutoff": 0.6949 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.6064864864864865, "confidence_cutoff": 0.4448444844484448 } }, "score_mae": 0.6840097940719104, "ranked_probability_score": 0.24038485578062901 }, "none_absent": { "n": 36, "nll": 0.9117502447148043, "acc": 0.6388888888888888, "ece": 0.15667153635202094, "brier": 0.47774654754138673, "mean_conf": 0.6894889719210286, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.3055555555555556, "accuracy_at_0_9": 0.8181818181818182, "coverage_at_5pct_error": 0.1388888888888889, "coverage_at_1pct_error": 0.1388888888888889, "aurc": 0.17887075589814766, "error_rate_at_0_9": 0.18181818181818182, "confidence_bias": 0.05060008303213981, "top_bins": { "0.9": { "n": 11, "errors": 2, "error_rate": 0.18181818181818182 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.6445 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7586206896551724, "confidence_cutoff": 0.4462 } } }, "none_present": { "n": 36, "nll": 1.989547486767386, "acc": 0.3611111111111111, "ece": 0.3283672872040025, "brier": 0.8473168929980956, "mean_conf": 0.6385869524295399, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.2222222222222222, "accuracy_at_0_9": 0.625, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5729351659867584, "error_rate_at_0_9": 0.375, "confidence_bias": 0.2774758413184288, "top_bins": { "0.9": { "n": 8, "errors": 3, "error_rate": 0.375 }, "0.95": { "n": 4, "errors": 2, "error_rate": 0.5 }, "0.99": { "n": 3, "errors": 2, "error_rate": 0.6666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4444444444444444, "confidence_cutoff": 0.6784 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.3448275862068966, "confidence_cutoff": 0.4015 } } }, "permuted": { "n": 36, "nll": 1.399980743528154, "acc": 0.5277777777777778, "ece": 0.17422823416919958, "brier": 0.5981863166336433, "mean_conf": 0.6543337897247552, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.3055555555555556, "accuracy_at_0_9": 0.8181818181818182, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.34407451656512256, "error_rate_at_0_9": 0.18181818181818182, "confidence_bias": 0.1265560119469774, "top_bins": { "0.9": { "n": 11, "errors": 2, "error_rate": 0.18181818181818182 }, "0.95": { "n": 8, "errors": 1, "error_rate": 0.125 }, "0.99": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7222222222222222, "confidence_cutoff": 0.6510651065106511 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5517241379310345, "confidence_cutoff": 0.3916 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.08834388366563391, "flip_rate": 0.19444444444444445 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 1.2274823338319296, "acc": 0.5305927342256214, "ece": 0.1290347987840952, "brier": 0.5832038000970096, "mean_conf": 0.6596275330097165, "confident_error_rate": 0.030592734225621414, "coverage_at_0_9": 0.1701720841300191, "accuracy_at_0_9": 0.8202247191011236, "coverage_at_5pct_error": 0.004780114722753346, "coverage_at_1pct_error": 0.004780114722753346, "aurc": 0.2779831915729428, "error_rate_at_0_9": 0.1797752808988764, "confidence_bias": 0.12903479878409518, "top_bins": { "0.9": { "n": 178, "errors": 32, "error_rate": 0.1797752808988764 }, "0.95": { "n": 99, "errors": 18, "error_rate": 0.18181818181818182 }, "0.99": { "n": 46, "errors": 9, "error_rate": 0.1956521739130435 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7476099426386233, "confidence_cutoff": 0.6987 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.6272401433691757, "confidence_cutoff": 0.4267 } }, "score_mae": 0.6827758263793483, "ranked_probability_score": 0.22948632636102131 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 51 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } }, "Laya-multilingual": { "panels": { "old_core": { "point": 0.4725074404761905, "exact_fraction": "12701/26880", "ci95": [ 0.4348214285714286, 0.5106770833333333 ] }, "v3_core": { "point": 0.38916666666666666, "exact_fraction": "467/1200", "ci95": [ 0.36750000000000005, 0.4108333333333333 ] }, "v4": { "point": 0.5078125, "exact_fraction": "65/128", "ci95": [ 0.46041588697604796, 0.5537583428022258 ] }, "v5": { "point": 0.5729166666666666, "exact_fraction": "55/96", "ci95": [ 0.534468477839384, 0.6107271796729912 ] }, "transfer_v9_test": { "point": 0.4713193116634799, "exact_fraction": "493/1046", "ci95": [ 0.4413982125383613, 0.5019654155862125 ] }, "original_four_panel_mean": { "point": 0.48560081845238096, "exact_fraction": "261059/537600", "ci95": [ 0.4670842171902512, 0.5042618801237501 ] }, "weighted_mean": { "point": 0.4718511705590458, "exact_fraction": "6633397/14058240", "ci95": [ 0.45579692367519703, 0.48818287369166813 ] } }, "original_details": { "old_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "ag_news": { "correct": 115, "requested": 128 }, "boolean_constraints": { "correct": 25, "requested": 64 }, "dbpedia_14": { "correct": 64, "requested": 112 }, "natural_intents": { "correct": 52, "requested": 64 }, "option_carrier": { "correct": 52, "requested": 96 }, "ordinal_rubric": { "correct": 8, "requested": 64 }, "relational_composition": { "correct": 34, "requested": 96 }, "scoped_evidence": { "correct": 35, "requested": 96 }, "state_tracking": { "correct": 28, "requested": 96 }, "unknown_rejection": { "correct": 24, "requested": 64 } } }, "v3_core": { "requested": 880, "valid_predictions": 880, "valid_probability_rows": 880, "families": { "canonical_record_identity": { "correct": 40, "requested": 80 }, "capacitated_assignment": { "correct": 40, "requested": 80 }, "constraint_assignment": { "correct": 22, "requested": 80 }, "massive_en": { "correct": 85, "requested": 120 }, "massive_zh": { "correct": 88, "requested": 120 }, "multiset_reconciliation": { "correct": 26, "requested": 80 }, "ordinal_service_loss": { "correct": 14, "requested": 80 }, "paraconsistent_rule_closure": { "correct": 16, "requested": 80 }, "temporal_exclusion": { "correct": 23, "requested": 80 }, "transaction_recovery": { "correct": 15, "requested": 80 } } }, "v4": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "boolq": { "correct": 111, "requested": 160 }, "belebele_en": { "correct": 58, "requested": 160 }, "belebele_zh": { "correct": 45, "requested": 160 } } }, "v5": { "requested": 480, "valid_predictions": 480, "valid_probability_rows": 480, "families": { "cosmos_qa": { "correct": 24, "requested": 120 }, "squad2_answerability": { "correct": 66, "requested": 120 }, "snli": { "correct": 87, "requested": 120 }, "qasc": { "correct": 98, "requested": 120 } } } }, "transfer_diagnostics": { "clean": { "requested": 1046, "valid_probability_rows": 1046, "correct": 493, "accuracy": 0.4713193116634799, "full_probability_metrics": { "n": 1046, "nll": 1.4653173168330305, "acc": 0.4713193116634799, "ece": 0.21360890228222623, "brier": 0.687331148328108, "mean_conf": 0.6849282139457061, "confident_error_rate": 0.09177820267686425, "coverage_at_0_9": 0.30879541108986613, "accuracy_at_0_9": 0.7027863777089783, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.34595085121108154, "error_rate_at_0_9": 0.29721362229102166, "confidence_bias": 0.21360890228222623, "top_bins": { "0.9": { "n": 323, "errors": 96, "error_rate": 0.29721362229102166 }, "0.95": { "n": 226, "errors": 58, "error_rate": 0.25663716814159293 }, "0.99": { "n": 94, "errors": 14, "error_rate": 0.14893617021276595 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6520076481835564, "confidence_cutoff": 0.7214 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5531660692951016, "confidence_cutoff": 0.42064206420642064 } }, "score_mae": 0.783383694623847, "ranked_probability_score": 0.2908790076547602 }, "covered_only_probability_metrics": { "n": 1046, "nll": 1.4653173168330305, "acc": 0.4713193116634799, "ece": 0.21360890228222623, "brier": 0.687331148328108, "mean_conf": 0.6849282139457061, "confident_error_rate": 0.09177820267686425, "coverage_at_0_9": 0.30879541108986613, "accuracy_at_0_9": 0.7027863777089783, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.34595085121108154, "error_rate_at_0_9": 0.29721362229102166, "confidence_bias": 0.21360890228222623, "top_bins": { "0.9": { "n": 323, "errors": 96, "error_rate": 0.29721362229102166 }, "0.95": { "n": 226, "errors": 58, "error_rate": 0.25663716814159293 }, "0.99": { "n": 94, "errors": 14, "error_rate": 0.14893617021276595 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6520076481835564, "confidence_cutoff": 0.7214 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5531660692951016, "confidence_cutoff": 0.42064206420642064 } }, "score_mae": 0.783383694623847, "ranked_probability_score": 0.2908790076547602 }, "coverage": 1.0 }, "tasks": { "buried_emotion": { "requested": 20, "valid_probability_rows": 20, "correct": 7, "accuracy": 0.35, "full_probability_metrics": { "n": 20, "nll": 3.234717489970181, "acc": 0.35, "ece": 0.5112127584320277, "brier": 1.0791496632216997, "mean_conf": 0.7914065107339652, "confident_error_rate": 0.4, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7413647976980019, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.4414065107339652, "top_bins": { "0.9": { "n": 12, "errors": 8, "error_rate": 0.6666666666666666 }, "0.95": { "n": 10, "errors": 6, "error_rate": 0.6 }, "0.99": { "n": 3, "errors": 3, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.961003899610039 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5038 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 3.234717489970181, "acc": 0.35, "ece": 0.5112127584320277, "brier": 1.0791496632216997, "mean_conf": 0.7914065107339652, "confident_error_rate": 0.4, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7413647976980019, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.4414065107339652, "top_bins": { "0.9": { "n": 12, "errors": 8, "error_rate": 0.6666666666666666 }, "0.95": { "n": 10, "errors": 6, "error_rate": 0.6 }, "0.99": { "n": 3, "errors": 3, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.961003899610039 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5038 } } }, "coverage": 1.0 }, "buried_paws": { "requested": 20, "valid_probability_rows": 20, "correct": 14, "accuracy": 0.7, "full_probability_metrics": { "n": 20, "nll": 0.8037398020119598, "acc": 0.7, "ece": 0.14450999999999997, "brier": 0.500355646, "mean_conf": 0.79871, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3877225038892995, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.09871000000000008, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.7899 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.7138 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.8037398020119598, "acc": 0.7, "ece": 0.14450999999999997, "brier": 0.500355646, "mean_conf": 0.79871, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3877225038892995, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.09871000000000008, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.7899 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.7138 } } }, "coverage": 1.0 }, "buried_qnli": { "requested": 20, "valid_probability_rows": 20, "correct": 11, "accuracy": 0.55, "full_probability_metrics": { "n": 20, "nll": 0.8184011585913623, "acc": 0.55, "ece": 0.297045, "brier": 0.554452911, "mean_conf": 0.8470449999999999, "confident_error_rate": 0.1, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.7777777777777778, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.20907979640758279, "error_rate_at_0_9": 0.2222222222222222, "confidence_bias": 0.2970449999999999, "top_bins": { "0.9": { "n": 9, "errors": 2, "error_rate": 0.2222222222222222 }, "0.95": { "n": 8, "errors": 1, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.897 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.7372000000000001 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.8184011585913623, "acc": 0.55, "ece": 0.297045, "brier": 0.554452911, "mean_conf": 0.8470449999999999, "confident_error_rate": 0.1, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.7777777777777778, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.20907979640758279, "error_rate_at_0_9": 0.2222222222222222, "confidence_bias": 0.2970449999999999, "top_bins": { "0.9": { "n": 9, "errors": 2, "error_rate": 0.2222222222222222 }, "0.95": { "n": 8, "errors": 1, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.897 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.7372000000000001 } } }, "coverage": 1.0 }, "buried_tweet_offensive": { "requested": 20, "valid_probability_rows": 20, "correct": 16, "accuracy": 0.8, "full_probability_metrics": { "n": 20, "nll": 0.4389838105923823, "acc": 0.8, "ece": 0.07347499999999996, "brier": 0.28466956499999996, "mean_conf": 0.7490950000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.07395177105316424, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05090499999999998, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.744 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.6411 } } }, "covered_only_probability_metrics": { "n": 20, "nll": 0.4389838105923823, "acc": 0.8, "ece": 0.07347499999999996, "brier": 0.28466956499999996, "mean_conf": 0.7490950000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.07395177105316424, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05090499999999998, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.744 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.6411 } } }, "coverage": 1.0 }, "composition_final_combination": { "requested": 32, "valid_probability_rows": 32, "correct": 14, "accuracy": 0.4375, "full_probability_metrics": { "n": 32, "nll": 0.8671013711684612, "acc": 0.4375, "ece": 0.28689375, "brier": 0.6236914687499999, "mean_conf": 0.72439375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5470499052468711, "error_rate_at_0_9": null, "confidence_bias": 0.28689374999999995, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5625, "confidence_cutoff": 0.7567 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5384615384615384, "confidence_cutoff": 0.6305 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 0.8671013711684612, "acc": 0.4375, "ece": 0.28689375, "brier": 0.6236914687499999, "mean_conf": 0.72439375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5470499052468711, "error_rate_at_0_9": null, "confidence_bias": 0.28689374999999995, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5625, "confidence_cutoff": 0.7567 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5384615384615384, "confidence_cutoff": 0.6305 } } }, "coverage": 1.0 }, "composition_final_exception": { "requested": 32, "valid_probability_rows": 32, "correct": 12, "accuracy": 0.375, "full_probability_metrics": { "n": 32, "nll": 1.2777479519024777, "acc": 0.375, "ece": 0.43409687500000005, "brier": 0.863296135625, "mean_conf": 0.809096875, "confident_error_rate": 0.25, "coverage_at_0_9": 0.34375, "accuracy_at_0_9": 0.2727272727272727, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6464269931623269, "error_rate_at_0_9": 0.7272727272727273, "confidence_bias": 0.43409687500000005, "top_bins": { "0.9": { "n": 11, "errors": 8, "error_rate": 0.7272727272727273 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5294117647058824, "confidence_cutoff": 0.8611 }, "0.8": { "coverage": 0.8125, "accuracy": 0.34615384615384615, "confidence_cutoff": 0.6491 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 1.2777479519024777, "acc": 0.375, "ece": 0.43409687500000005, "brier": 0.863296135625, "mean_conf": 0.809096875, "confident_error_rate": 0.25, "coverage_at_0_9": 0.34375, "accuracy_at_0_9": 0.2727272727272727, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6464269931623269, "error_rate_at_0_9": 0.7272727272727273, "confidence_bias": 0.43409687500000005, "top_bins": { "0.9": { "n": 11, "errors": 8, "error_rate": 0.7272727272727273 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5294117647058824, "confidence_cutoff": 0.8611 }, "0.8": { "coverage": 0.8125, "accuracy": 0.34615384615384615, "confidence_cutoff": 0.6491 } } }, "coverage": 1.0 }, "composition_final_negation": { "requested": 32, "valid_probability_rows": 32, "correct": 16, "accuracy": 0.5, "full_probability_metrics": { "n": 32, "nll": 1.236344507640615, "acc": 0.5, "ece": 0.38400625, "brier": 0.79811755875, "mean_conf": 0.79000625, "confident_error_rate": 0.25, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6914995270338682, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.29000625, "top_bins": { "0.9": { "n": 12, "errors": 8, "error_rate": 0.6666666666666666 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4375, "confidence_cutoff": 0.8563 }, "0.8": { "coverage": 0.8125, "accuracy": 0.46153846153846156, "confidence_cutoff": 0.5613 } } }, "covered_only_probability_metrics": { "n": 32, "nll": 1.236344507640615, "acc": 0.5, "ece": 0.38400625, "brier": 0.79811755875, "mean_conf": 0.79000625, "confident_error_rate": 0.25, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6914995270338682, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.29000625, "top_bins": { "0.9": { "n": 12, "errors": 8, "error_rate": 0.6666666666666666 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4375, "confidence_cutoff": 0.8563 }, "0.8": { "coverage": 0.8125, "accuracy": 0.46153846153846156, "confidence_cutoff": 0.5613 } } }, "coverage": 1.0 }, "contrastive_authorization": { "requested": 40, "valid_probability_rows": 40, "correct": 20, "accuracy": 0.5, "full_probability_metrics": { "n": 40, "nll": 1.5501653515681346, "acc": 0.5, "ece": 0.4587175000000001, "brier": 0.8650006975, "mean_conf": 0.9587175, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.475, "aurc": 0.1663037721188986, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.4587175, "top_bins": { "0.9": { "n": 36, "errors": 16, "error_rate": 0.4444444444444444 }, "0.95": { "n": 32, "errors": 12, "error_rate": 0.375 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9885 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.957 } } }, "covered_only_probability_metrics": { "n": 40, "nll": 1.5501653515681346, "acc": 0.5, "ece": 0.4587175000000001, "brier": 0.8650006975, "mean_conf": 0.9587175, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.475, "aurc": 0.1663037721188986, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.4587175, "top_bins": { "0.9": { "n": 36, "errors": 16, "error_rate": 0.4444444444444444 }, "0.95": { "n": 32, "errors": 12, "error_rate": 0.375 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9885 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.957 } } }, "coverage": 1.0 }, "contrastive_deadline": { "requested": 40, "valid_probability_rows": 40, "correct": 9, "accuracy": 0.225, "full_probability_metrics": { "n": 40, "nll": 1.526156646217014, "acc": 0.225, "ece": 0.3226398774693988, "brier": 0.8790970758757233, "mean_conf": 0.5476398774693988, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8349630919159832, "error_rate_at_0_9": null, "confidence_bias": 0.3226398774693988, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.25, "confidence_cutoff": 0.561043895610439 }, "0.8": { "coverage": 0.8, "accuracy": 0.21875, "confidence_cutoff": 0.4513451345134513 } }, "score_mae": 0.8199080506958556, "ranked_probability_score": 0.2859539972143025 }, "covered_only_probability_metrics": { "n": 40, "nll": 1.526156646217014, "acc": 0.225, "ece": 0.3226398774693988, "brier": 0.8790970758757233, "mean_conf": 0.5476398774693988, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8349630919159832, "error_rate_at_0_9": null, "confidence_bias": 0.3226398774693988, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.25, "confidence_cutoff": 0.561043895610439 }, "0.8": { "coverage": 0.8, "accuracy": 0.21875, "confidence_cutoff": 0.4513451345134513 } }, "score_mae": 0.8199080506958556, "ranked_probability_score": 0.2859539972143025 }, "coverage": 1.0 }, "emotion": { "requested": 80, "valid_probability_rows": 80, "correct": 45, "accuracy": 0.5625, "full_probability_metrics": { "n": 80, "nll": 2.038456168904628, "acc": 0.5625, "ece": 0.2962758193492206, "brier": 0.6734534039937483, "mean_conf": 0.8587758193492208, "confident_error_rate": 0.2125, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.6730769230769231, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3081425858155885, "error_rate_at_0_9": 0.3269230769230769, "confidence_bias": 0.29627581934922076, "top_bins": { "0.9": { "n": 52, "errors": 17, "error_rate": 0.3269230769230769 }, "0.95": { "n": 43, "errors": 11, "error_rate": 0.2558139534883721 }, "0.99": { "n": 24, "errors": 6, "error_rate": 0.25 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.9637963796379638 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.6743 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 2.038456168904628, "acc": 0.5625, "ece": 0.2962758193492206, "brier": 0.6734534039937483, "mean_conf": 0.8587758193492208, "confident_error_rate": 0.2125, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.6730769230769231, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3081425858155885, "error_rate_at_0_9": 0.3269230769230769, "confidence_bias": 0.29627581934922076, "top_bins": { "0.9": { "n": 52, "errors": 17, "error_rate": 0.3269230769230769 }, "0.95": { "n": 43, "errors": 11, "error_rate": 0.2558139534883721 }, "0.99": { "n": 24, "errors": 6, "error_rate": 0.25 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.9637963796379638 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.6743 } } }, "coverage": 1.0 }, "mmlu": { "requested": 80, "valid_probability_rows": 80, "correct": 22, "accuracy": 0.275, "full_probability_metrics": { "n": 80, "nll": 1.5136150303015745, "acc": 0.275, "ece": 0.2287108836303588, "brier": 0.7925400048936556, "mean_conf": 0.4928619685218697, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6235168831530026, "error_rate_at_0_9": 0.5, "confidence_bias": 0.21786196852186968, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.4557 }, "0.8": { "coverage": 0.8, "accuracy": 0.296875, "confidence_cutoff": 0.3507 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 1.5136150303015745, "acc": 0.275, "ece": 0.2287108836303588, "brier": 0.7925400048936556, "mean_conf": 0.4928619685218697, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6235168831530026, "error_rate_at_0_9": 0.5, "confidence_bias": 0.21786196852186968, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.4557 }, "0.8": { "coverage": 0.8, "accuracy": 0.296875, "confidence_cutoff": 0.3507 } } }, "coverage": 1.0 }, "mmlu_pro": { "requested": 200, "valid_probability_rows": 200, "correct": 23, "accuracy": 0.115, "full_probability_metrics": { "n": 200, "nll": 2.5668209627148872, "acc": 0.115, "ece": 0.22578658993648842, "brier": 0.9788974704906136, "mean_conf": 0.3407865899364884, "confident_error_rate": 0.01, "coverage_at_0_9": 0.01, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8614676106841176, "error_rate_at_0_9": 1.0, "confidence_bias": 0.22578658993648842, "top_bins": { "0.9": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.15, "confidence_cutoff": 0.2895289528952895 }, "0.8": { "coverage": 0.8, "accuracy": 0.13125, "confidence_cutoff": 0.2018 } } }, "covered_only_probability_metrics": { "n": 200, "nll": 2.5668209627148872, "acc": 0.115, "ece": 0.22578658993648842, "brier": 0.9788974704906136, "mean_conf": 0.3407865899364884, "confident_error_rate": 0.01, "coverage_at_0_9": 0.01, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8614676106841176, "error_rate_at_0_9": 1.0, "confidence_bias": 0.22578658993648842, "top_bins": { "0.9": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.15, "confidence_cutoff": 0.2895289528952895 }, "0.8": { "coverage": 0.8, "accuracy": 0.13125, "confidence_cutoff": 0.2018 } } }, "coverage": 1.0 }, "paws": { "requested": 80, "valid_probability_rows": 80, "correct": 60, "accuracy": 0.75, "full_probability_metrics": { "n": 80, "nll": 0.657053017135968, "acc": 0.75, "ece": 0.11198749999999993, "brier": 0.39931878249999997, "mean_conf": 0.8415600000000001, "confident_error_rate": 0.075, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.23704764262605565, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.09156000000000009, "top_bins": { "0.9": { "n": 36, "errors": 6, "error_rate": 0.16666666666666666 }, "0.95": { "n": 21, "errors": 4, "error_rate": 0.19047619047619047 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8887 }, "0.8": { "coverage": 0.8, "accuracy": 0.765625, "confidence_cutoff": 0.6941999999999999 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.657053017135968, "acc": 0.75, "ece": 0.11198749999999993, "brier": 0.39931878249999997, "mean_conf": 0.8415600000000001, "confident_error_rate": 0.075, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.23704764262605565, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.09156000000000009, "top_bins": { "0.9": { "n": 36, "errors": 6, "error_rate": 0.16666666666666666 }, "0.95": { "n": 21, "errors": 4, "error_rate": 0.19047619047619047 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8887 }, "0.8": { "coverage": 0.8, "accuracy": 0.765625, "confidence_cutoff": 0.6941999999999999 } } }, "coverage": 1.0 }, "qnli": { "requested": 80, "valid_probability_rows": 80, "correct": 59, "accuracy": 0.7375, "full_probability_metrics": { "n": 80, "nll": 0.6658760189188047, "acc": 0.7375, "ece": 0.11610874999999994, "brier": 0.38656590174999994, "mean_conf": 0.85271625, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.5375, "accuracy_at_0_9": 0.8372093023255814, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18121491972080836, "error_rate_at_0_9": 0.16279069767441862, "confidence_bias": 0.11521624999999991, "top_bins": { "0.9": { "n": 43, "errors": 7, "error_rate": 0.16279069767441862 }, "0.95": { "n": 28, "errors": 5, "error_rate": 0.17857142857142858 }, "0.99": { "n": 12, "errors": 2, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.9176 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7132 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.6658760189188047, "acc": 0.7375, "ece": 0.11610874999999994, "brier": 0.38656590174999994, "mean_conf": 0.85271625, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.5375, "accuracy_at_0_9": 0.8372093023255814, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18121491972080836, "error_rate_at_0_9": 0.16279069767441862, "confidence_bias": 0.11521624999999991, "top_bins": { "0.9": { "n": 43, "errors": 7, "error_rate": 0.16279069767441862 }, "0.95": { "n": 28, "errors": 5, "error_rate": 0.17857142857142858 }, "0.99": { "n": 12, "errors": 2, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.9176 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7132 } } }, "coverage": 1.0 }, "sciq": { "requested": 80, "valid_probability_rows": 80, "correct": 58, "accuracy": 0.725, "full_probability_metrics": { "n": 80, "nll": 0.7249295668223826, "acc": 0.725, "ece": 0.14402921845692968, "brier": 0.38948680277908526, "mean_conf": 0.64625175821333, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.3625, "aurc": 0.11505783929543811, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07874824178666995, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 13, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.6073000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.4469 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.7249295668223826, "acc": 0.725, "ece": 0.14402921845692968, "brier": 0.38948680277908526, "mean_conf": 0.64625175821333, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.3625, "aurc": 0.11505783929543811, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07874824178666995, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 13, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.6073000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.4469 } } }, "coverage": 1.0 }, "tweet_offensive": { "requested": 80, "valid_probability_rows": 80, "correct": 66, "accuracy": 0.825, "full_probability_metrics": { "n": 80, "nll": 0.4165720387614019, "acc": 0.825, "ece": 0.09130374999999999, "brier": 0.27136572175, "mean_conf": 0.8980612499999999, "confident_error_rate": 0.05, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.92, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.4625, "aurc": 0.057603822836134744, "error_rate_at_0_9": 0.08, "confidence_bias": 0.07306124999999997, "top_bins": { "0.9": { "n": 50, "errors": 4, "error_rate": 0.08 }, "0.95": { "n": 37, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9463 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8317 } } }, "covered_only_probability_metrics": { "n": 80, "nll": 0.4165720387614019, "acc": 0.825, "ece": 0.09130374999999999, "brier": 0.27136572175, "mean_conf": 0.8980612499999999, "confident_error_rate": 0.05, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.92, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.4625, "aurc": 0.057603822836134744, "error_rate_at_0_9": 0.08, "confidence_bias": 0.07306124999999997, "top_bins": { "0.9": { "n": 50, "errors": 4, "error_rate": 0.08 }, "0.95": { "n": 37, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9463 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8317 } } }, "coverage": 1.0 }, "unknowable_control_age_eligibility": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 0.9039582646692856, "acc": 0.5, "ece": 0.3813, "brier": 0.600219408, "mean_conf": 0.71264, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.36007936507936505, "error_rate_at_0_9": 0.5, "confidence_bias": 0.21264000000000005, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7082 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6385 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 0.9039582646692856, "acc": 0.5, "ece": 0.3813, "brier": 0.600219408, "mean_conf": 0.71264, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.36007936507936505, "error_rate_at_0_9": 0.5, "confidence_bias": 0.21264000000000005, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7082 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6385 } } }, "coverage": 1.0 }, "unknowable_control_authorization": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 1.7946999977508278, "acc": 0.5, "ece": 0.4760899999999999, "brier": 0.917096954, "mean_conf": 0.9760899999999999, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.4760899999999999, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9951 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.9832 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.7946999977508278, "acc": 0.5, "ece": 0.4760899999999999, "brier": 0.917096954, "mean_conf": 0.9760899999999999, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.4760899999999999, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9951 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.9832 } } }, "coverage": 1.0 }, "unknowable_control_deadline": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.5261665564384048, "acc": 0.3, "ece": 0.22261451467274518, "brier": 0.8678551352949488, "mean_conf": 0.5226145146727451, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7847619047619048, "error_rate_at_0_9": null, "confidence_bias": 0.22261451467274512, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5367 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.4415 } }, "score_mae": 0.8218755666354557, "ranked_probability_score": 0.29207077892919847 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.5261665564384048, "acc": 0.3, "ece": 0.22261451467274518, "brier": 0.8678551352949488, "mean_conf": 0.5226145146727451, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7847619047619048, "error_rate_at_0_9": null, "confidence_bias": 0.22261451467274512, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5367 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.4415 } }, "score_mae": 0.8218755666354557, "ranked_probability_score": 0.29207077892919847 }, "coverage": 1.0 }, "unknowable_control_late_fee": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 2.1455872887258716, "acc": 0.4, "ece": 0.34763971002899713, "brier": 0.8860525708384708, "mean_conf": 0.7476397100289971, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4326984126984126, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.3476397100289971, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.8084 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5302469753024698 } }, "score_mae": 0.6914803759624037, "ranked_probability_score": 0.2650603338498591 }, "covered_only_probability_metrics": { "n": 10, "nll": 2.1455872887258716, "acc": 0.4, "ece": 0.34763971002899713, "brier": 0.8860525708384708, "mean_conf": 0.7476397100289971, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4326984126984126, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.3476397100289971, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.8084 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5302469753024698 } }, "score_mae": 0.6914803759624037, "ranked_probability_score": 0.2650603338498591 }, "coverage": 1.0 }, "unknowable_control_quantity_limit": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.3099659215887423, "acc": 0.3, "ece": 0.30288439643964393, "brier": 0.7398723617076403, "mean_conf": 0.5179556035603561, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5546428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.2179556035603561, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.504 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4721 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.3099659215887423, "acc": 0.3, "ece": 0.30288439643964393, "brier": 0.7398723617076403, "mean_conf": 0.5179556035603561, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5546428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.2179556035603561, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.504 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4721 } } }, "coverage": 1.0 }, "unknowable_control_return_window": { "requested": 10, "valid_probability_rows": 10, "correct": 4, "accuracy": 0.4, "full_probability_metrics": { "n": 10, "nll": 1.9043568129358028, "acc": 0.4, "ece": 0.5349, "brier": 1.037261264, "mean_conf": 0.8747, "confident_error_rate": 0.5, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.16666666666666666, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7989285714285714, "error_rate_at_0_9": 0.8333333333333334, "confidence_bias": 0.4747, "top_bins": { "0.9": { "n": 6, "errors": 5, "error_rate": 0.8333333333333334 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.9446 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.8329 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 1.9043568129358028, "acc": 0.4, "ece": 0.5349, "brier": 1.037261264, "mean_conf": 0.8747, "confident_error_rate": 0.5, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.16666666666666666, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7989285714285714, "error_rate_at_0_9": 0.8333333333333334, "confidence_bias": 0.4747, "top_bins": { "0.9": { "n": 6, "errors": 5, "error_rate": 0.8333333333333334 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.9446 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.8329 } } }, "coverage": 1.0 }, "unknowable_control_shipping_delay": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.1806438160049517, "acc": 0.3, "ece": 0.2715613848615138, "brier": 0.7178877768111797, "mean_conf": 0.5715613848615139, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7589285714285715, "error_rate_at_0_9": null, "confidence_bias": 0.2715613848615139, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.5747 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5038496150384961 } }, "score_mae": 0.633525612438756, "ranked_probability_score": 0.20970614225048964 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.1806438160049517, "acc": 0.3, "ece": 0.2715613848615138, "brier": 0.7178877768111797, "mean_conf": 0.5715613848615139, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7589285714285715, "error_rate_at_0_9": null, "confidence_bias": 0.2715613848615139, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.5747 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5038496150384961 } }, "score_mae": 0.633525612438756, "ranked_probability_score": 0.20970614225048964 }, "coverage": 1.0 }, "unknowable_control_sla_response": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.705826792755279, "acc": 0.3, "ece": 0.49495109138481086, "brier": 1.0059048667706372, "mean_conf": 0.7949510913848108, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7880952380952382, "error_rate_at_0_9": null, "confidence_bias": 0.4949510913848108, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.8055 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7489 } }, "score_mae": 0.8110433892255339, "ranked_probability_score": 0.32487051848653087 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.705826792755279, "acc": 0.3, "ece": 0.49495109138481086, "brier": 1.0059048667706372, "mean_conf": 0.7949510913848108, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7880952380952382, "error_rate_at_0_9": null, "confidence_bias": 0.4949510913848108, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.8055 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7489 } }, "score_mae": 0.8110433892255339, "ranked_probability_score": 0.32487051848653087 }, "coverage": 1.0 }, "unknowable_control_spend_threshold": { "requested": 10, "valid_probability_rows": 10, "correct": 5, "accuracy": 0.5, "full_probability_metrics": { "n": 10, "nll": 3.3173792867266365, "acc": 0.5, "ece": 0.4331086588658865, "brier": 0.9120519964572784, "mean_conf": 0.9331086588658865, "confident_error_rate": 0.1, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.255515873015873, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.43310865886588645, "top_bins": { "0.9": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.95": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9375 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8920892089208922 } } }, "covered_only_probability_metrics": { "n": 10, "nll": 3.3173792867266365, "acc": 0.5, "ece": 0.4331086588658865, "brier": 0.9120519964572784, "mean_conf": 0.9331086588658865, "confident_error_rate": 0.1, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.255515873015873, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.43310865886588645, "top_bins": { "0.9": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.95": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9375 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8920892089208922 } } }, "coverage": 1.0 }, "unknowable_control_volume_discount": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 1.910510899590001, "acc": 0.3, "ece": 0.5917195165958951, "brier": 1.092333952149472, "mean_conf": 0.7953795165958952, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.5042063492063492, "error_rate_at_0_9": 0.0, "confidence_bias": 0.49537951659589524, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.822117788221178 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7737773777377738 } }, "score_mae": 0.7818597991928978, "ranked_probability_score": 0.3129796351743147 }, "covered_only_probability_metrics": { "n": 10, "nll": 1.910510899590001, "acc": 0.3, "ece": 0.5917195165958951, "brier": 1.092333952149472, "mean_conf": 0.7953795165958952, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.5042063492063492, "error_rate_at_0_9": 0.0, "confidence_bias": 0.49537951659589524, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.822117788221178 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7737773777377738 } }, "score_mae": 0.7818597991928978, "ranked_probability_score": 0.3129796351743147 }, "coverage": 1.0 }, "unknowable_control_warranty_claim": { "requested": 10, "valid_probability_rows": 10, "correct": 3, "accuracy": 0.3, "full_probability_metrics": { "n": 10, "nll": 2.387868950744031, "acc": 0.3, "ece": 0.52617, "brier": 1.112524292, "mean_conf": 0.82617, "confident_error_rate": 0.3, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7880952380952382, "error_rate_at_0_9": 0.75, "confidence_bias": 0.52617, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.8804 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7053 } }, "score_mae": 0.8144199999999999, "ranked_probability_score": 0.360286679 }, "covered_only_probability_metrics": { "n": 10, "nll": 2.387868950744031, "acc": 0.3, "ece": 0.52617, "brier": 1.112524292, "mean_conf": 0.82617, "confident_error_rate": 0.3, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7880952380952382, "error_rate_at_0_9": 0.75, "confidence_bias": 0.52617, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.8804 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7053 } }, "score_mae": 0.8144199999999999, "ranked_probability_score": 0.360286679 }, "coverage": 1.0 } }, "clean_task_macro_accuracy": 0.46398148148148155, "permutation": { "requested_pairs": 36, "valid_pairs": 36, "both_correct_requested": 0.5, "flip_rate": 0.2222222222222222, "covered_only_flip_rate": 0.2222222222222222, "half_l1": 0.14911431834839317 }, "unknowable": { "requested": 110, "valid": 110, "accuracy": null, "full_confidence_metrics": { "mean_max_p": 0.7570207792119349, "share_at_0_9": 0.2909090909090909, "normalized_entropy": 0.5307491622513651 }, "covered_only_confidence_metrics": { "mean_max_p": 0.7570207792119349, "share_at_0_9": 0.2909090909090909, "normalized_entropy": 0.5307491622513651 }, "controls": { "requested": 110, "valid_probability_rows": 110, "correct": 41, "accuracy": 0.37272727272727274, "full_probability_metrics": { "n": 110, "nll": 1.8260876898118035, "acc": 0.37272727272727274, "ece": 0.37934640727001867, "brier": 0.8990055070936025, "mean_conf": 0.7520736799972914, "confident_error_rate": 0.13636363636363635, "coverage_at_0_9": 0.2818181818181818, "accuracy_at_0_9": 0.5161290322580645, "coverage_at_5pct_error": 0.06363636363636363, "coverage_at_1pct_error": 0.06363636363636363, "aurc": 0.5185657418184665, "error_rate_at_0_9": 0.4838709677419355, "confidence_bias": 0.3793464072700186, "top_bins": { "0.9": { "n": 31, "errors": 15, "error_rate": 0.4838709677419355 }, "0.95": { "n": 24, "errors": 12, "error_rate": 0.5 }, "0.99": { "n": 8, "errors": 1, "error_rate": 0.125 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.38181818181818183, "confidence_cutoff": 0.7912 }, "0.8": { "coverage": 0.8, "accuracy": 0.3977272727272727, "confidence_cutoff": 0.5367 } }, "score_mae": 0.7590341239091746, "ranked_probability_score": 0.29416234794839874 }, "covered_only_probability_metrics": { "n": 110, "nll": 1.8260876898118035, "acc": 0.37272727272727274, "ece": 0.37934640727001867, "brier": 0.8990055070936025, "mean_conf": 0.7520736799972914, "confident_error_rate": 0.13636363636363635, "coverage_at_0_9": 0.2818181818181818, "accuracy_at_0_9": 0.5161290322580645, "coverage_at_5pct_error": 0.06363636363636363, "coverage_at_1pct_error": 0.06363636363636363, "aurc": 0.5185657418184665, "error_rate_at_0_9": 0.4838709677419355, "confidence_bias": 0.3793464072700186, "top_bins": { "0.9": { "n": 31, "errors": 15, "error_rate": 0.4838709677419355 }, "0.95": { "n": 24, "errors": 12, "error_rate": 0.5 }, "0.99": { "n": 8, "errors": 1, "error_rate": 0.125 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.38181818181818183, "confidence_cutoff": 0.7912 }, "0.8": { "coverage": 0.8, "accuracy": 0.3977272727272727, "confidence_cutoff": 0.5367 } }, "score_mae": 0.7590341239091746, "ranked_probability_score": 0.29416234794839874 }, "coverage": 1.0 }, "requested_pairs": 110, "valid_pairs": 110, "paired_confidence_drop": -0.004947099214643722 }, "coverage": { "questions": 1264, "valid": 1264, "failures_by_status": {}, "truncated_questions": 14 }, "complete_unchanged_upstream_report": { "objective": -1.4971720548574838, "paired_flip": { "pairs": 64, "flip_rate": 0.203125, "both_correct_rate": 0.0, "invariant_pairs": 24, "invariance_rate": 0.9583333333333334, "invariant_both_correct_rate": 0.4583333333333333 }, "unknowable": { "n": 110, "mean_max_p": 0.7570207792119349, "share_at_0_9": 0.2909090909090909, "control_mean_max_p": 0.7520736799972914, "control_share_at_0_9": 0.2818181818181818, "control_acc": 0.37272727272727274, "paired_confidence_drop": -0.004947099214643722, "share_less_confident_than_control": 0.4818181818181818 }, "clean": { "n": 1046, "nll": 1.4653173168330305, "acc": 0.4713193116634799, "ece": 0.21360890228222623, "brier": 0.687331148328108, "mean_conf": 0.6849282139457061, "confident_error_rate": 0.09177820267686425, "coverage_at_0_9": 0.30879541108986613, "accuracy_at_0_9": 0.7027863777089783, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.34595085121108154, "error_rate_at_0_9": 0.29721362229102166, "confidence_bias": 0.21360890228222623, "top_bins": { "0.9": { "n": 323, "errors": 96, "error_rate": 0.29721362229102166 }, "0.95": { "n": 226, "errors": 58, "error_rate": 0.25663716814159293 }, "0.99": { "n": 94, "errors": 14, "error_rate": 0.14893617021276595 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6520076481835564, "confidence_cutoff": 0.7214 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5531660692951016, "confidence_cutoff": 0.42064206420642064 } }, "score_mae": 0.783383694623847, "ranked_probability_score": 0.2908790076547602 }, "tasks": { "buried_emotion": { "n": 20, "nll": 3.234717489970181, "acc": 0.35, "ece": 0.5112127584320277, "brier": 1.0791496632216997, "mean_conf": 0.7914065107339652, "confident_error_rate": 0.4, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7413647976980019, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.4414065107339652, "top_bins": { "0.9": { "n": 12, "errors": 8, "error_rate": 0.6666666666666666 }, "0.95": { "n": 10, "errors": 6, "error_rate": 0.6 }, "0.99": { "n": 3, "errors": 3, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.961003899610039 }, "0.8": { "coverage": 0.8, "accuracy": 0.3125, "confidence_cutoff": 0.5038 } } }, "buried_paws": { "n": 20, "nll": 0.8037398020119598, "acc": 0.7, "ece": 0.14450999999999997, "brier": 0.500355646, "mean_conf": 0.79871, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3877225038892995, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.09871000000000008, "top_bins": { "0.9": { "n": 6, "errors": 2, "error_rate": 0.3333333333333333 }, "0.95": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.7899 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.7138 } } }, "buried_qnli": { "n": 20, "nll": 0.8184011585913623, "acc": 0.55, "ece": 0.297045, "brier": 0.554452911, "mean_conf": 0.8470449999999999, "confident_error_rate": 0.1, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.7777777777777778, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.3, "aurc": 0.20907979640758279, "error_rate_at_0_9": 0.2222222222222222, "confidence_bias": 0.2970449999999999, "top_bins": { "0.9": { "n": 9, "errors": 2, "error_rate": 0.2222222222222222 }, "0.95": { "n": 8, "errors": 1, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.897 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.7372000000000001 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.4389838105923823, "acc": 0.8, "ece": 0.07347499999999996, "brier": 0.28466956499999996, "mean_conf": 0.7490950000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "aurc": 0.07395177105316424, "error_rate_at_0_9": 0.0, "confidence_bias": -0.05090499999999998, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.744 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.6411 } } }, "composition_final_combination": { "n": 32, "nll": 0.8671013711684612, "acc": 0.4375, "ece": 0.28689375, "brier": 0.6236914687499999, "mean_conf": 0.72439375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5470499052468711, "error_rate_at_0_9": null, "confidence_bias": 0.28689374999999995, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.5625, "confidence_cutoff": 0.7567 }, "0.8": { "coverage": 0.8125, "accuracy": 0.5384615384615384, "confidence_cutoff": 0.6305 } } }, "composition_final_exception": { "n": 32, "nll": 1.2777479519024777, "acc": 0.375, "ece": 0.43409687500000005, "brier": 0.863296135625, "mean_conf": 0.809096875, "confident_error_rate": 0.25, "coverage_at_0_9": 0.34375, "accuracy_at_0_9": 0.2727272727272727, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6464269931623269, "error_rate_at_0_9": 0.7272727272727273, "confidence_bias": 0.43409687500000005, "top_bins": { "0.9": { "n": 11, "errors": 8, "error_rate": 0.7272727272727273 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 0.5294117647058824, "confidence_cutoff": 0.8611 }, "0.8": { "coverage": 0.8125, "accuracy": 0.34615384615384615, "confidence_cutoff": 0.6491 } } }, "composition_final_negation": { "n": 32, "nll": 1.236344507640615, "acc": 0.5, "ece": 0.38400625, "brier": 0.79811755875, "mean_conf": 0.79000625, "confident_error_rate": 0.25, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.3333333333333333, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6914995270338682, "error_rate_at_0_9": 0.6666666666666666, "confidence_bias": 0.29000625, "top_bins": { "0.9": { "n": 12, "errors": 8, "error_rate": 0.6666666666666666 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4375, "confidence_cutoff": 0.8563 }, "0.8": { "coverage": 0.8125, "accuracy": 0.46153846153846156, "confidence_cutoff": 0.5613 } } }, "contrastive_authorization": { "n": 40, "nll": 1.5501653515681346, "acc": 0.5, "ece": 0.4587175000000001, "brier": 0.8650006975, "mean_conf": 0.9587175, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.475, "aurc": 0.1663037721188986, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.4587175, "top_bins": { "0.9": { "n": 36, "errors": 16, "error_rate": 0.4444444444444444 }, "0.95": { "n": 32, "errors": 12, "error_rate": 0.375 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9885 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.957 } } }, "contrastive_deadline": { "n": 40, "nll": 1.526156646217014, "acc": 0.225, "ece": 0.3226398774693988, "brier": 0.8790970758757233, "mean_conf": 0.5476398774693988, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8349630919159832, "error_rate_at_0_9": null, "confidence_bias": 0.3226398774693988, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.25, "confidence_cutoff": 0.561043895610439 }, "0.8": { "coverage": 0.8, "accuracy": 0.21875, "confidence_cutoff": 0.4513451345134513 } }, "score_mae": 0.8199080506958556, "ranked_probability_score": 0.2859539972143025 }, "emotion": { "n": 80, "nll": 2.038456168904628, "acc": 0.5625, "ece": 0.2962758193492206, "brier": 0.6734534039937483, "mean_conf": 0.8587758193492208, "confident_error_rate": 0.2125, "coverage_at_0_9": 0.65, "accuracy_at_0_9": 0.6730769230769231, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.3081425858155885, "error_rate_at_0_9": 0.3269230769230769, "confidence_bias": 0.29627581934922076, "top_bins": { "0.9": { "n": 52, "errors": 17, "error_rate": 0.3269230769230769 }, "0.95": { "n": 43, "errors": 11, "error_rate": 0.2558139534883721 }, "0.99": { "n": 24, "errors": 6, "error_rate": 0.25 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.9637963796379638 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.6743 } } }, "mmlu": { "n": 80, "nll": 1.5136150303015745, "acc": 0.275, "ece": 0.2287108836303588, "brier": 0.7925400048936556, "mean_conf": 0.4928619685218697, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.025, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6235168831530026, "error_rate_at_0_9": 0.5, "confidence_bias": 0.21786196852186968, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 1, "errors": 1, "error_rate": 1.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.4557 }, "0.8": { "coverage": 0.8, "accuracy": 0.296875, "confidence_cutoff": 0.3507 } } }, "mmlu_pro": { "n": 200, "nll": 2.5668209627148872, "acc": 0.115, "ece": 0.22578658993648842, "brier": 0.9788974704906136, "mean_conf": 0.3407865899364884, "confident_error_rate": 0.01, "coverage_at_0_9": 0.01, "accuracy_at_0_9": 0.0, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8614676106841176, "error_rate_at_0_9": 1.0, "confidence_bias": 0.22578658993648842, "top_bins": { "0.9": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.95": { "n": 2, "errors": 2, "error_rate": 1.0 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.15, "confidence_cutoff": 0.2895289528952895 }, "0.8": { "coverage": 0.8, "accuracy": 0.13125, "confidence_cutoff": 0.2018 } } }, "paws": { "n": 80, "nll": 0.657053017135968, "acc": 0.75, "ece": 0.11198749999999993, "brier": 0.39931878249999997, "mean_conf": 0.8415600000000001, "confident_error_rate": 0.075, "coverage_at_0_9": 0.45, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.23704764262605565, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.09156000000000009, "top_bins": { "0.9": { "n": 36, "errors": 6, "error_rate": 0.16666666666666666 }, "0.95": { "n": 21, "errors": 4, "error_rate": 0.19047619047619047 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8887 }, "0.8": { "coverage": 0.8, "accuracy": 0.765625, "confidence_cutoff": 0.6941999999999999 } } }, "qnli": { "n": 80, "nll": 0.6658760189188047, "acc": 0.7375, "ece": 0.11610874999999994, "brier": 0.38656590174999994, "mean_conf": 0.85271625, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.5375, "accuracy_at_0_9": 0.8372093023255814, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18121491972080836, "error_rate_at_0_9": 0.16279069767441862, "confidence_bias": 0.11521624999999991, "top_bins": { "0.9": { "n": 43, "errors": 7, "error_rate": 0.16279069767441862 }, "0.95": { "n": 28, "errors": 5, "error_rate": 0.17857142857142858 }, "0.99": { "n": 12, "errors": 2, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.9176 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.7132 } } }, "sciq": { "n": 80, "nll": 0.7249295668223826, "acc": 0.725, "ece": 0.14402921845692968, "brier": 0.38948680277908526, "mean_conf": 0.64625175821333, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.3625, "aurc": 0.11505783929543811, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07874824178666995, "top_bins": { "0.9": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 13, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.825, "confidence_cutoff": 0.6073000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.4469 } } }, "tweet_offensive": { "n": 80, "nll": 0.4165720387614019, "acc": 0.825, "ece": 0.09130374999999999, "brier": 0.27136572175, "mean_conf": 0.8980612499999999, "confident_error_rate": 0.05, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.92, "coverage_at_5pct_error": 0.525, "coverage_at_1pct_error": 0.4625, "aurc": 0.057603822836134744, "error_rate_at_0_9": 0.08, "confidence_bias": 0.07306124999999997, "top_bins": { "0.9": { "n": 50, "errors": 4, "error_rate": 0.08 }, "0.95": { "n": 37, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 16, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9463 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8317 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.8251544458461442, "acc": 0.5, "ece": 0.20848000000000005, "brier": 0.602306568, "mean_conf": 0.70848, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": null, "confidence_bias": 0.20848, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.6531 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6496 } } }, "unknowable_authorization": { "n": 10, "nll": 1.7165022005595714, "acc": 0.5, "ece": 0.47055999999999987, "brier": 0.9335291399999999, "mean_conf": 0.9705599999999999, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.27218253968253964, "error_rate_at_0_9": 0.5, "confidence_bias": 0.47055999999999987, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9722 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.9636 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.9039582646692856, "acc": 0.5, "ece": 0.3813, "brier": 0.600219408, "mean_conf": 0.71264, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.36007936507936505, "error_rate_at_0_9": 0.5, "confidence_bias": 0.21264000000000005, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.7082 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6385 } } }, "unknowable_control_authorization": { "n": 10, "nll": 1.7946999977508278, "acc": 0.5, "ece": 0.4760899999999999, "brier": 0.917096954, "mean_conf": 0.9760899999999999, "confident_error_rate": 0.4, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.5555555555555556, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.5, "aurc": 0.17718253968253966, "error_rate_at_0_9": 0.4444444444444444, "confidence_bias": 0.4760899999999999, "top_bins": { "0.9": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.95": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9951 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.9832 } } }, "unknowable_control_deadline": { "n": 10, "nll": 1.5261665564384048, "acc": 0.3, "ece": 0.22261451467274518, "brier": 0.8678551352949488, "mean_conf": 0.5226145146727451, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7847619047619048, "error_rate_at_0_9": null, "confidence_bias": 0.22261451467274512, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5367 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.4415 } }, "score_mae": 0.8218755666354557, "ranked_probability_score": 0.29207077892919847 }, "unknowable_control_late_fee": { "n": 10, "nll": 2.1455872887258716, "acc": 0.4, "ece": 0.34763971002899713, "brier": 0.8860525708384708, "mean_conf": 0.7476397100289971, "confident_error_rate": 0.1, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.6666666666666666, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.4326984126984126, "error_rate_at_0_9": 0.3333333333333333, "confidence_bias": 0.3476397100289971, "top_bins": { "0.9": { "n": 3, "errors": 1, "error_rate": 0.3333333333333333 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.8084 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5302469753024698 } }, "score_mae": 0.6914803759624037, "ranked_probability_score": 0.2650603338498591 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 1.3099659215887423, "acc": 0.3, "ece": 0.30288439643964393, "brier": 0.7398723617076403, "mean_conf": 0.5179556035603561, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5546428571428572, "error_rate_at_0_9": null, "confidence_bias": 0.2179556035603561, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.504 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4721 } } }, "unknowable_control_return_window": { "n": 10, "nll": 1.9043568129358028, "acc": 0.4, "ece": 0.5349, "brier": 1.037261264, "mean_conf": 0.8747, "confident_error_rate": 0.5, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.16666666666666666, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7989285714285714, "error_rate_at_0_9": 0.8333333333333334, "confidence_bias": 0.4747, "top_bins": { "0.9": { "n": 6, "errors": 5, "error_rate": 0.8333333333333334 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 1, "errors": 1, "error_rate": 1.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.9446 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.8329 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 1.1806438160049517, "acc": 0.3, "ece": 0.2715613848615138, "brier": 0.7178877768111797, "mean_conf": 0.5715613848615139, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7589285714285715, "error_rate_at_0_9": null, "confidence_bias": 0.2715613848615139, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.5747 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.5038496150384961 } }, "score_mae": 0.633525612438756, "ranked_probability_score": 0.20970614225048964 }, "unknowable_control_sla_response": { "n": 10, "nll": 1.705826792755279, "acc": 0.3, "ece": 0.49495109138481086, "brier": 1.0059048667706372, "mean_conf": 0.7949510913848108, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7880952380952382, "error_rate_at_0_9": null, "confidence_bias": 0.4949510913848108, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.8055 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7489 } }, "score_mae": 0.8110433892255339, "ranked_probability_score": 0.32487051848653087 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 3.3173792867266365, "acc": 0.5, "ece": 0.4331086588658865, "brier": 0.9120519964572784, "mean_conf": 0.9331086588658865, "confident_error_rate": 0.1, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.255515873015873, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.43310865886588645, "top_bins": { "0.9": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 }, "0.95": { "n": 4, "errors": 1, "error_rate": 0.25 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.9375 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.8920892089208922 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 1.910510899590001, "acc": 0.3, "ece": 0.5917195165958951, "brier": 1.092333952149472, "mean_conf": 0.7953795165958952, "confident_error_rate": 0.0, "coverage_at_0_9": 0.1, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.5042063492063492, "error_rate_at_0_9": 0.0, "confidence_bias": 0.49537951659589524, "top_bins": { "0.9": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.822117788221178 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7737773777377738 } }, "score_mae": 0.7818597991928978, "ranked_probability_score": 0.3129796351743147 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 2.387868950744031, "acc": 0.3, "ece": 0.52617, "brier": 1.112524292, "mean_conf": 0.82617, "confident_error_rate": 0.3, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.25, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.7880952380952382, "error_rate_at_0_9": 0.75, "confidence_bias": 0.52617, "top_bins": { "0.9": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.95": { "n": 4, "errors": 3, "error_rate": 0.75 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.8804 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7053 } }, "score_mae": 0.8144199999999999, "ranked_probability_score": 0.360286679 }, "unknowable_deadline": { "n": 10, "nll": 1.334259591149368, "acc": 0.3, "ece": 0.2199703099327032, "brier": 0.7872956943662375, "mean_conf": 0.5199703099327031, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "aurc": 0.48059523809523813, "error_rate_at_0_9": null, "confidence_bias": 0.2199703099327031, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.5276472352764724 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4773 } }, "score_mae": 0.793016204191362, "ranked_probability_score": 0.2729261275647014 }, "unknowable_late_fee": { "n": 10, "nll": 2.285495001001151, "acc": 0.4, "ece": 0.33252000000000004, "brier": 0.8315564, "mean_conf": 0.7325199999999998, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.52, "error_rate_at_0_9": null, "confidence_bias": 0.3325199999999998, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.7307 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.6956 } }, "score_mae": 0.68514, "ranked_probability_score": 0.25358242 }, "unknowable_quantity_limit": { "n": 10, "nll": 1.2347494839296274, "acc": 0.4, "ece": 0.19786942905709431, "brier": 0.7445845887370736, "mean_conf": 0.5398505709429056, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.6283333333333333, "error_rate_at_0_9": null, "confidence_bias": 0.13985057094290554, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.5055 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4714528547145285 } } }, "unknowable_return_window": { "n": 10, "nll": 2.691275657916257, "acc": 0.5, "ece": 0.49523000000000006, "brier": 0.9890620819999999, "mean_conf": 0.9952300000000001, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.38563492063492066, "error_rate_at_0_9": 0.5, "confidence_bias": 0.49523000000000006, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.9969 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.993 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.5116382983777021, "acc": 0.3, "ece": 0.4327095811927958, "brier": 0.9301067400222174, "mean_conf": 0.7327095811927958, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8047619047619048, "error_rate_at_0_9": null, "confidence_bias": 0.43270958119279584, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.743 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.6635 } }, "score_mae": 0.6778679223231792, "ranked_probability_score": 0.2513329398210322 }, "unknowable_sla_response": { "n": 10, "nll": 1.8124945596959114, "acc": 0.3, "ece": 0.4939749334933493, "brier": 1.008245661100702, "mean_conf": 0.7939749334933494, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8066666666666666, "error_rate_at_0_9": null, "confidence_bias": 0.49397493349334937, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.7729 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.7653 } }, "score_mae": 0.82356, "ranked_probability_score": 0.3310777483419607 }, "unknowable_spend_threshold": { "n": 10, "nll": 3.8693100731229757, "acc": 0.5, "ece": 0.48841999999999997, "brier": 0.98817938, "mean_conf": 0.98842, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.5, "error_rate_at_0_9": 0.5, "confidence_bias": 0.48841999999999997, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 4, "errors": 2, "error_rate": 0.5 } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.9881000000000001 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.9841 } } }, "unknowable_volume_discount": { "n": 10, "nll": 1.9301346006414597, "acc": 0.2, "ece": 0.47985329467053306, "brier": 1.048263746847218, "mean_conf": 0.679853294670533, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8766666666666667, "error_rate_at_0_9": null, "confidence_bias": 0.479853294670533, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.16666666666666666, "confidence_cutoff": 0.6976 }, "0.8": { "coverage": 0.8, "accuracy": 0.25, "confidence_cutoff": 0.5937 } }, "score_mae": 0.8464600000000001, "ranked_probability_score": 0.31775871915850284 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.3723890891818318, "acc": 0.4, "ece": 0.2864598810989988, "brier": 0.7507683986317131, "mean_conf": 0.6656598810989988, "confident_error_rate": 0.1, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "aurc": 0.41507936507936505, "error_rate_at_0_9": 0.5, "confidence_bias": 0.26565988109899874, "top_bins": { "0.9": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.95": { "n": 2, "errors": 1, "error_rate": 0.5 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.6034 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.5568 } }, "score_mae": 0.6284019502581195, "ranked_probability_score": 0.23042169135406954 } }, "variants": { "clean": { "n": 1156, "nll": 1.5039411275273096, "acc": 0.46366782006920415, "ece": 0.22812041306273484, "brier": 0.7050928764258242, "mean_conf": 0.691788233131939, "confident_error_rate": 0.09688581314878893, "coverage_at_0_9": 0.3070934256055363, "accuracy_at_0_9": 0.6845070422535211, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.36554259244582366, "error_rate_at_0_9": 0.3154929577464789, "confidence_bias": 0.2281204130627349, "top_bins": { "0.9": { "n": 355, "errors": 112, "error_rate": 0.3154929577464789 }, "0.95": { "n": 258, "errors": 74, "error_rate": 0.2868217054263566 }, "0.99": { "n": 108, "errors": 20, "error_rate": 0.18518518518518517 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6280276816608996, "confidence_cutoff": 0.7241275872412758 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.5427027027027027, "confidence_cutoff": 0.4494 } }, "score_mae": 0.7680176889381956, "ranked_probability_score": 0.28536810767424187 }, "none_absent": { "n": 36, "nll": 0.45479486787045825, "acc": 0.8333333333333334, "ece": 0.16405435987237685, "brier": 0.263069912676635, "mean_conf": 0.80831188830623, "confident_error_rate": 0.0, "coverage_at_0_9": 0.5277777777777778, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6388888888888888, "coverage_at_1pct_error": 0.5833333333333334, "aurc": 0.049144816240759004, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02502144502710335, "top_bins": { "0.9": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9321067893210679 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8620689655172413, "confidence_cutoff": 0.5141 } } }, "none_present": { "n": 36, "nll": 5.711102265809461, "acc": 0.25, "ece": 0.5984661229757735, "brier": 1.3003859615972342, "mean_conf": 0.8199894209181732, "confident_error_rate": 0.4166666666666667, "coverage_at_0_9": 0.4444444444444444, "accuracy_at_0_9": 0.0625, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.8692322404794948, "error_rate_at_0_9": 0.9375, "confidence_bias": 0.5699894209181732, "top_bins": { "0.9": { "n": 16, "errors": 15, "error_rate": 0.9375 }, "0.95": { "n": 14, "errors": 13, "error_rate": 0.9285714285714286 }, "0.99": { "n": 12, "errors": 11, "error_rate": 0.9166666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.05555555555555555, "confidence_cutoff": 0.8885 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.27586206896551724, "confidence_cutoff": 0.7247 } } }, "permuted": { "n": 36, "nll": 1.3391318070555556, "acc": 0.5555555555555556, "ece": 0.23174670959082816, "brier": 0.6248788236326477, "mean_conf": 0.6144596442500687, "confident_error_rate": 0.05555555555555555, "coverage_at_0_9": 0.2777777777777778, "accuracy_at_0_9": 0.8, "coverage_at_5pct_error": 0.19444444444444445, "coverage_at_1pct_error": 0.19444444444444445, "aurc": 0.26244561922399146, "error_rate_at_0_9": 0.2, "confidence_bias": 0.058904088694513135, "top_bins": { "0.9": { "n": 10, "errors": 2, "error_rate": 0.2 }, "0.95": { "n": 8, "errors": 1, "error_rate": 0.125 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7777777777777778, "confidence_cutoff": 0.5737 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.5517241379310345, "confidence_cutoff": 0.3751375137513751 } } } }, "heldout_tasks": {}, "permutation": { "n": 36, "mean_max_delta": 0.14085231171571755, "flip_rate": 0.2222222222222222 }, "temperature": 1.0, "calibrated_clean": { "n": 1046, "nll": 1.4653173168330305, "acc": 0.4713193116634799, "ece": 0.21360890228222623, "brier": 0.687331148328108, "mean_conf": 0.6849282139457061, "confident_error_rate": 0.09177820267686425, "coverage_at_0_9": 0.30879541108986613, "accuracy_at_0_9": 0.7027863777089783, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.34595085121108154, "error_rate_at_0_9": 0.29721362229102166, "confidence_bias": 0.21360890228222623, "top_bins": { "0.9": { "n": 323, "errors": 96, "error_rate": 0.29721362229102166 }, "0.95": { "n": 226, "errors": 58, "error_rate": 0.25663716814159293 }, "0.99": { "n": 94, "errors": 14, "error_rate": 0.14893617021276595 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6520076481835564, "confidence_cutoff": 0.7214 }, "0.8": { "coverage": 0.8001912045889101, "accuracy": 0.5531660692951016, "confidence_cutoff": 0.42064206420642064 } }, "score_mae": 0.783383694623847, "ranked_probability_score": 0.2908790076547602 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 74 } }, "metric_scope": "Refused/invalid predictions count incorrect in requested categorical denominator. Missing probabilities stay missing; covered-only metrics never presented as full metrics. Risk-coverage thresholds are retrospective diagnostics, not a deployed error guarantee." } } }, "method": { "weights": { "old_core": "3/10", "v3_core": "1/4", "v4": "3/20", "v5": "3/20", "transfer_v9_test": "3/20" }, "seed": 202609225801, "repeats": 10000, "transfer_components": { "count": 796, "questions": 1046, "size_histogram": { "1": 599, "2": 170, "3": 1, "4": 26 }, "partition_sha256": "0625bd9b57e1b18d4a3186b4951447a3d48b76277f468e73c49f1598fca68592" }, "original_panels": "Unchanged original native-decision scoring, family weights and original paired source-cluster draw_panel functions.", "transfer_accuracy": "Upstream ordered probability argmax, clean knowable records, micro average. Refused/missing probabilities count wrong; unknown evidence is not accuracy.", "transfer_clusters": "Union IDs/group/parent/control/pair plus exact state SHA and ordered-request IDs, built over all variants then scored only clean knowable rows. Complete components jointly resampled; variable sampled denominator. No whole-family grouping.", "pairing": "Identical sampled source components across all models; five panels sampled independently. Original four marginal CIs preserved.", "accuracy_arithmetic": "Exact rational integer correct/requested. Outcome-informed product-priority weights chosen by the user after earlier results; this display does not change the frozen scorer or constitute a training gain.", "interval_role": "Descriptive paired95% intervals; no positive-CI release requirement invented.", "diagnostics": "Full per-task/proper-score/order/unknown-evidence results remain separate; no accuracy/latency/ECE unit mixing.", "exposure": "Observed regression tests; weighting user-approved after earlier results; not a blind prospective benchmark." }, "paired_comparisons": {}, "presentation_note": "Fifteen current models. Fixed scores and original statistics are preserved; per-row provenance identifies independently closed runs.", "size_note": "Family suffixes are rounded model-size classes; Kai and Eos have 571,909,635 and 753,446,208 parameters." }