| - dataset: |
| id: LocalLLaMA/typed-decisions |
| task_id: accuracy |
| value: 0.72 |
| date: '2026-10-08' |
| source: |
| url: https://huggingface.co/datasets/olaverse/african-typed-decisions/blob/main/leaderboard/all_metrics.csv |
| name: Scored with the African Typed Decisions harness (English original rows) |
| user: olaverse |
| notes: 'General, zero-shot: never trained on the Typed Decisions train split. Test split, 400 cases, 2,000 decisions, one request per case with the state and all five questions, all answered, zero errors. Accuracy = agreement with the gold label; KL = KL(gold || prediction); Brier summed over options. One forward pass per state, no generated tokens. bf16, calibrated temperatures from purplemist_config.json.' |
| - dataset: |
| id: LocalLLaMA/typed-decisions |
| task_id: kl_from_gold |
| value: 0.1889 |
| date: '2026-10-08' |
| source: |
| url: https://huggingface.co/datasets/olaverse/african-typed-decisions/blob/main/leaderboard/all_metrics.csv |
| name: Scored with the African Typed Decisions harness (English original rows) |
| user: olaverse |
| notes: 'General, zero-shot: never trained on the Typed Decisions train split. Test split, 400 cases, 2,000 decisions, one request per case with the state and all five questions, all answered, zero errors. Accuracy = agreement with the gold label; KL = KL(gold || prediction); Brier summed over options. One forward pass per state, no generated tokens. bf16, calibrated temperatures from purplemist_config.json.' |
| - dataset: |
| id: LocalLLaMA/typed-decisions |
| task_id: brier |
| value: 0.0975 |
| date: '2026-10-08' |
| source: |
| url: https://huggingface.co/datasets/olaverse/african-typed-decisions/blob/main/leaderboard/all_metrics.csv |
| name: Scored with the African Typed Decisions harness (English original rows) |
| user: olaverse |
| notes: 'General, zero-shot: never trained on the Typed Decisions train split. Test split, 400 cases, 2,000 decisions, one request per case with the state and all five questions, all answered, zero errors. Accuracy = agreement with the gold label; KL = KL(gold || prediction); Brier summed over options. One forward pass per state, no generated tokens. bf16, calibrated temperatures from purplemist_config.json.' |
| - dataset: |
| id: LocalLLaMA/typed-decisions |
| task_id: ece |
| value: 0.0562 |
| date: '2026-10-08' |
| source: |
| url: https://huggingface.co/datasets/olaverse/african-typed-decisions/blob/main/leaderboard/all_metrics.csv |
| name: Scored with the African Typed Decisions harness (English original rows) |
| user: olaverse |
| notes: 'General, zero-shot: never trained on the Typed Decisions train split. Test split, 400 cases, 2,000 decisions, one request per case with the state and all five questions, all answered, zero errors. Accuracy = agreement with the gold label; KL = KL(gold || prediction); Brier summed over options. One forward pass per state, no generated tokens. bf16, calibrated temperatures from purplemist_config.json.' |
|
|
