{
 "meta": {
  "base_model": "Qwen/Qwen3.5-4B-Base",
  "d_head": 256,
  "lora": false,
  "renderer": {
   "max_state_tokens": 768,
   "max_instruction_tokens": 256,
   "max_option_tokens": 64
  },
  "step": 734,
  "val_nll": 0.27321311831474304
 },
 "temperatures": {
  "choice": 1.0929638744012644,
  "score": 1.1011288758276154,
  "noul": 1.228405177254357
 },
 "test": {
  "raw": {
   "overall": {
    "n": 5070,
    "acc": 0.8964496850967407,
    "nll": 0.276126503944397,
    "brier": 0.1475353091955185,
    "ece": 0.010809999694056847,
    "mean_conf": 0.9048414826393127,
    "score_mae": 0.41506025195121765
   },
   "by_type": {
    "choice": {
     "n": 3041,
     "acc": 0.9003617167472839,
     "nll": 0.2871852517127991,
     "brier": 0.1437402367591858,
     "ece": 0.013187985173783867,
     "mean_conf": 0.9091227054595947
    },
    "score": {
     "n": 273,
     "acc": 0.6849817037582397,
     "nll": 0.7681964039802551,
     "brier": 0.4400661289691925,
     "ece": 0.047672064850177796,
     "mean_conf": 0.6837503910064697,
     "score_mae": 0.41506025195121765
    },
    "noul": {
     "n": 1756,
     "acc": 0.9225512742996216,
     "nll": 0.18047469854354858,
     "brier": 0.10862862318754196,
     "ece": 0.013375402451816704,
     "mean_conf": 0.9318000674247742
    }
   },
   "by_source": {
    "ag_news": {
     "n": 86,
     "acc": 0.930232584476471,
     "nll": 0.23873434960842133,
     "brier": 0.10834771394729614,
     "ece": 0.056332710342662184,
     "mean_conf": 0.9181896448135376
    },
    "amazon_stars": {
     "n": 86,
     "acc": 0.5930232405662537,
     "nll": 0.8701136112213135,
     "brier": 0.5058151483535767,
     "ece": 0.10280728207642115,
     "mean_conf": 0.6330943703651428,
     "score_mae": 0.4689413011074066
    },
    "anli": {
     "n": 79,
     "acc": 0.746835470199585,
     "nll": 0.6076061129570007,
     "brier": 0.3763708174228668,
     "ece": 0.15590721699211704,
     "mean_conf": 0.8249932527542114
    },
    "banking77": {
     "n": 91,
     "acc": 0.9670329689979553,
     "nll": 0.109474316239357,
     "brier": 0.05170481652021408,
     "ece": 0.04274910372363963,
     "mean_conf": 0.9463766813278198
    },
    "boolq": {
     "n": 75,
     "acc": 0.9200000166893005,
     "nll": 0.22674815356731415,
     "brier": 0.14536462724208832,
     "ece": 0.07328386494371486,
     "mean_conf": 0.908048689365387
    },
    "cifar100": {
     "n": 297,
     "acc": 0.936026930809021,
     "nll": 0.19756226241588593,
     "brier": 0.096889927983284,
     "ece": 0.02602774308840307,
     "mean_conf": 0.9325385689735413
    },
    "civil_comments": {
     "n": 154,
     "acc": 0.9545454382896423,
     "nll": 0.1635725051164627,
     "brier": 0.08106027543544769,
     "ece": 0.08850163645923792,
     "mean_conf": 0.8801278471946716
    },
    "clinc_oos": {
     "n": 85,
     "acc": 0.9529411792755127,
     "nll": 0.10286867618560791,
     "brier": 0.057649165391922,
     "ece": 0.028260532641420116,
     "mean_conf": 0.9765596389770508
    },
    "cola": {
     "n": 89,
     "acc": 0.8202247023582458,
     "nll": 0.3538511097431183,
     "brier": 0.23637528717517853,
     "ece": 0.07856890498333513,
     "mean_conf": 0.8428974747657776
    },
    "commonsense_qa": {
     "n": 79,
     "acc": 0.797468364238739,
     "nll": 0.47439879179000854,
     "brier": 0.2571961283683777,
     "ece": 0.11401724499686991,
     "mean_conf": 0.8429151177406311
    },
    "copa": {
     "n": 19,
     "acc": 0.9473684430122375,
     "nll": 0.08234653621912003,
     "brier": 0.056282687932252884,
     "ece": 0.053942021972607,
     "mean_conf": 0.9678375720977783
    },
    "cub_birds": {
     "n": 209,
     "acc": 0.9282296895980835,
     "nll": 0.205092191696167,
     "brier": 0.09457369148731232,
     "ece": 0.0337111855618914,
     "mean_conf": 0.9249444007873535
    },
    "dbpedia": {
     "n": 83,
     "acc": 0.9879518151283264,
     "nll": 0.0267021507024765,
     "brier": 0.01387640181928873,
     "ece": 0.01773379953028531,
     "mean_conf": 0.9858810305595398
    },
    "dtd": {
     "n": 193,
     "acc": 0.9326424598693848,
     "nll": 0.1894713044166565,
     "brier": 0.10953276604413986,
     "ece": 0.0462280399812367,
     "mean_conf": 0.9360920786857605
    },
    "eurosat": {
     "n": 200,
     "acc": 0.9700000286102295,
     "nll": 0.09461035579442978,
     "brier": 0.04988160729408264,
     "ece": 0.022912378822553925,
     "mean_conf": 0.9720562696456909
    },
    "go_emotions": {
     "n": 156,
     "acc": 0.8461538553237915,
     "nll": 0.4075973331928253,
     "brier": 0.2239874303340912,
     "ece": 0.052582084089755954,
     "mean_conf": 0.8886739015579224
    },
    "gtsrb": {
     "n": 150,
     "acc": 0.9266666769981384,
     "nll": 0.2583926022052765,
     "brier": 0.10076691210269928,
     "ece": 0.0349075827405604,
     "mean_conf": 0.9396733641624451
    },
    "hellaswag": {
     "n": 71,
     "acc": 0.9718309640884399,
     "nll": 0.07258392125368118,
     "brier": 0.03423016890883446,
     "ece": 0.037721984350516546,
     "mean_conf": 0.9680467844009399
    },
    "imagenet100": {
     "n": 209,
     "acc": 0.9521530866622925,
     "nll": 0.1411520093679428,
     "brier": 0.06291362643241882,
     "ece": 0.01678104336341813,
     "mean_conf": 0.9670758247375488
    },
    "jailbreak": {
     "n": 43,
     "acc": 1.0,
     "nll": 0.003266148502007127,
     "brier": 0.0003119981265626848,
     "ece": 0.003183770989621637,
     "mean_conf": 0.9968161582946777
    },
    "mnli": {
     "n": 88,
     "acc": 0.9204545617103577,
     "nll": 0.17574532330036163,
     "brier": 0.10608046501874924,
     "ece": 0.039473747201574705,
     "mean_conf": 0.9106850028038025
    },
    "ms_marco": {
     "n": 77,
     "acc": 0.5714285969734192,
     "nll": 1.1672861576080322,
     "brier": 0.5488265156745911,
     "ece": 0.1069421570546632,
     "mean_conf": 0.5573300123214722
    },
    "newsgroups": {
     "n": 72,
     "acc": 0.8194444179534912,
     "nll": 0.656489372253418,
     "brier": 0.24557192623615265,
     "ece": 0.10580966421453894,
     "mean_conf": 0.8494588732719421
    },
    "openbookqa": {
     "n": 73,
     "acc": 0.9452054500579834,
     "nll": 0.2053627371788025,
     "brier": 0.10051807016134262,
     "ece": 0.07717989225561772,
     "mean_conf": 0.8888569474220276
    },
    "paws": {
     "n": 75,
     "acc": 0.8799999952316284,
     "nll": 0.252999871969223,
     "brier": 0.16616970300674438,
     "ece": 0.07703955284745612,
     "mean_conf": 0.957039475440979
    },
    "prompt_injection": {
     "n": 19,
     "acc": 1.0,
     "nll": 0.07026468217372894,
     "brier": 0.02881462872028351,
     "ece": 0.06134651247871403,
     "mean_conf": 0.9386534690856934
    },
    "qasc": {
     "n": 78,
     "acc": 0.8974359035491943,
     "nll": 0.22243788838386536,
     "brier": 0.11997074633836746,
     "ece": 0.058650574809634426,
     "mean_conf": 0.9030769467353821
    },
    "qnli": {
     "n": 79,
     "acc": 0.9620253443717957,
     "nll": 0.12711185216903687,
     "brier": 0.06058496609330177,
     "ece": 0.02656769830988215,
     "mean_conf": 0.9400109648704529
    },
    "qqp": {
     "n": 67,
     "acc": 0.8507462739944458,
     "nll": 0.32717540860176086,
     "brier": 0.20558716356754303,
     "ece": 0.05535818888270405,
     "mean_conf": 0.8827617168426514
    },
    "race": {
     "n": 71,
     "acc": 0.8169013857841492,
     "nll": 0.5218772292137146,
     "brier": 0.26671260595321655,
     "ece": 0.08260945939768694,
     "mean_conf": 0.868141770362854
    },
    "resisc45": {
     "n": 201,
     "acc": 0.9452736377716064,
     "nll": 0.17759159207344055,
     "brier": 0.07590000331401825,
     "ece": 0.03162269839005066,
     "mean_conf": 0.9709147810935974
    },
    "rte": {
     "n": 42,
     "acc": 0.9047619104385376,
     "nll": 0.21782515943050385,
     "brier": 0.12485634535551071,
     "ece": 0.05018191372233538,
     "mean_conf": 0.9342974424362183
    },
    "rvl_cdip": {
     "n": 207,
     "acc": 0.9227052927017212,
     "nll": 0.24853183329105377,
     "brier": 0.1186482235789299,
     "ece": 0.035717121188601514,
     "mean_conf": 0.9394121170043945
    },
    "sciq": {
     "n": 70,
     "acc": 0.9571428298950195,
     "nll": 0.08418832719326019,
     "brier": 0.05511568486690521,
     "ece": 0.05332319163331367,
     "mean_conf": 0.9662564992904663
    },
    "snli": {
     "n": 78,
     "acc": 0.9615384340286255,
     "nll": 0.13747791945934296,
     "brier": 0.07794130593538284,
     "ece": 0.06147045345163815,
     "mean_conf": 0.9255906343460083
    },
    "squad_v2": {
     "n": 83,
     "acc": 0.8795180916786194,
     "nll": 0.3885398805141449,
     "brier": 0.18468858301639557,
     "ece": 0.08604057614296412,
     "mean_conf": 0.9535071849822998
    },
    "sst5": {
     "n": 101,
     "acc": 0.6732673048973083,
     "nll": 0.8455938696861267,
     "brier": 0.4966614544391632,
     "ece": 0.10307709063944676,
     "mean_conf": 0.6207720041275024,
     "score_mae": 0.45541465282440186
    },
    "stanford_cars": {
     "n": 199,
     "acc": 0.9849246144294739,
     "nll": 0.06568868458271027,
     "brier": 0.033190734684467316,
     "ece": 0.02284979613965564,
     "mean_conf": 0.9779478907585144
    },
    "sun397": {
     "n": 205,
     "acc": 0.9658536314964294,
     "nll": 0.13169661164283752,
     "brier": 0.059237346053123474,
     "ece": 0.027937838926008762,
     "mean_conf": 0.9778258800506592
    },
    "toxic_chat": {
     "n": 103,
     "acc": 0.9805825352668762,
     "nll": 0.04486699774861336,
     "brier": 0.02695567160844803,
     "ece": 0.011975512422466195,
     "mean_conf": 0.9775879383087158
    },
    "tweet_irony": {
     "n": 73,
     "acc": 0.7397260069847107,
     "nll": 0.47673916816711426,
     "brier": 0.32497861981391907,
     "ece": 0.10002183849680613,
     "mean_conf": 0.8234904408454895
    },
    "tweet_offensive": {
     "n": 95,
     "acc": 0.800000011920929,
     "nll": 0.42309972643852234,
     "brier": 0.2671454846858978,
     "ece": 0.11630791231693371,
     "mean_conf": 0.8510119915008545
    },
    "tweet_sentiment": {
     "n": 96,
     "acc": 0.7604166865348816,
     "nll": 0.6009823679924011,
     "brier": 0.3709958493709564,
     "ece": 0.08339798031360968,
     "mean_conf": 0.7406583428382874
    },
    "vitaminc": {
     "n": 84,
     "acc": 0.9285714030265808,
     "nll": 0.20010635256767273,
     "brier": 0.11288775503635406,
     "ece": 0.055092808689364614,
     "mean_conf": 0.8885759711265564
    },
    "winogrande": {
     "n": 82,
     "acc": 0.8292682766914368,
     "nll": 0.3929581940174103,
     "brier": 0.2536102533340454,
     "ece": 0.09293118971498115,
     "mean_conf": 0.9037488698959351
    },
    "yahoo_topics": {
     "n": 71,
     "acc": 0.7746478915214539,
     "nll": 0.6088139414787292,
     "brier": 0.28299495577812195,
     "ece": 0.06163965007121763,
     "mean_conf": 0.7804079055786133
    },
    "yelp": {
     "n": 127,
     "acc": 0.834645688533783,
     "nll": 0.4275256395339966,
     "brier": 0.2361738681793213,
     "ece": 0.047331056411305306,
     "mean_conf": 0.858354926109314,
     "score_mae": 0.31378617882728577
    }
   }
  },
  "calibrated": {
   "overall": {
    "n": 5070,
    "acc": 0.8964496850967407,
    "nll": 0.2756565809249878,
    "brier": 0.14737455546855927,
    "ece": 0.0079650055739778,
    "mean_conf": 0.8922349810600281,
    "score_mae": 0.42143115401268005
   },
   "by_type": {
    "choice": {
     "n": 3041,
     "acc": 0.9003617167472839,
     "nll": 0.28574398159980774,
     "brier": 0.14335514605045319,
     "ece": 0.007999743385107894,
     "mean_conf": 0.8997042179107666
    },
    "score": {
     "n": 273,
     "acc": 0.6849817037582397,
     "nll": 0.7671821713447571,
     "brier": 0.439628005027771,
     "ece": 0.046648137280912105,
     "mean_conf": 0.6634480953216553,
     "score_mae": 0.42143115401268005
    },
    "noul": {
     "n": 1756,
     "acc": 0.9225512742996216,
     "nll": 0.18177150189876556,
     "brier": 0.10889948904514313,
     "ece": 0.019957571878301106,
     "mean_conf": 0.9148688316345215
    }
   },
   "by_source": {
    "ag_news": {
     "n": 86,
     "acc": 0.930232584476471,
     "nll": 0.2372349351644516,
     "brier": 0.10982950031757355,
     "ece": 0.06062877569098146,
     "mean_conf": 0.909740686416626
    },
    "amazon_stars": {
     "n": 86,
     "acc": 0.5930232405662537,
     "nll": 0.8707610964775085,
     "brier": 0.5034118294715881,
     "ece": 0.07147133660142413,
     "mean_conf": 0.6115179657936096,
     "score_mae": 0.4738217294216156
    },
    "anli": {
     "n": 79,
     "acc": 0.746835470199585,
     "nll": 0.5904105305671692,
     "brier": 0.3706202208995819,
     "ece": 0.12736005320452623,
     "mean_conf": 0.8066393733024597
    },
    "banking77": {
     "n": 91,
     "acc": 0.9670329689979553,
     "nll": 0.11517834663391113,
     "brier": 0.05284213647246361,
     "ece": 0.04865624303995164,
     "mean_conf": 0.9390164613723755
    },
    "boolq": {
     "n": 75,
     "acc": 0.9200000166893005,
     "nll": 0.230869323015213,
     "brier": 0.14464128017425537,
     "ece": 0.05709565622368751,
     "mean_conf": 0.8859748244285583
    },
    "cifar100": {
     "n": 297,
     "acc": 0.936026930809021,
     "nll": 0.19978056848049164,
     "brier": 0.09738589078187943,
     "ece": 0.03686069778541466,
     "mean_conf": 0.9233465194702148
    },
    "civil_comments": {
     "n": 154,
     "acc": 0.9545454382896423,
     "nll": 0.1945398449897766,
     "brier": 0.09554325044155121,
     "ece": 0.11094526093531587,
     "mean_conf": 0.8504656553268433
    },
    "clinc_oos": {
     "n": 85,
     "acc": 0.9529411792755127,
     "nll": 0.10213149338960648,
     "brier": 0.05607043579220772,
     "ece": 0.03434847114305911,
     "mean_conf": 0.9722728133201599
    },
    "cola": {
     "n": 89,
     "acc": 0.8202247023582458,
     "nll": 0.3592163622379303,
     "brier": 0.2345142364501953,
     "ece": 0.07689192332149841,
     "mean_conf": 0.8134418725967407
    },
    "commonsense_qa": {
     "n": 79,
     "acc": 0.797468364238739,
     "nll": 0.47140878438949585,
     "brier": 0.25501152873039246,
     "ece": 0.08131545514081495,
     "mean_conf": 0.8276560306549072
    },
    "copa": {
     "n": 19,
     "acc": 0.9473684430122375,
     "nll": 0.08365084230899811,
     "brier": 0.05513031780719757,
     "ece": 0.05683338684464534,
     "mean_conf": 0.9632882475852966
    },
    "cub_birds": {
     "n": 209,
     "acc": 0.9282296895980835,
     "nll": 0.20787712931632996,
     "brier": 0.09552589058876038,
     "ece": 0.03924435088578615,
     "mean_conf": 0.9157379269599915
    },
    "dbpedia": {
     "n": 83,
     "acc": 0.9879518151283264,
     "nll": 0.02914765290915966,
     "brier": 0.013987519778311253,
     "ece": 0.020033360317049276,
     "mean_conf": 0.9829169511795044
    },
    "dtd": {
     "n": 193,
     "acc": 0.9326424598693848,
     "nll": 0.19226685166358948,
     "brier": 0.10932108014822006,
     "ece": 0.04905586869892781,
     "mean_conf": 0.926359236240387
    },
    "eurosat": {
     "n": 200,
     "acc": 0.9700000286102295,
     "nll": 0.09493371099233627,
     "brier": 0.04979003965854645,
     "ece": 0.019272321772114326,
     "mean_conf": 0.964755117893219
    },
    "go_emotions": {
     "n": 156,
     "acc": 0.8461538553237915,
     "nll": 0.3998943567276001,
     "brier": 0.2204635888338089,
     "ece": 0.06369368834835576,
     "mean_conf": 0.8688168525695801
    },
    "gtsrb": {
     "n": 150,
     "acc": 0.9266666769981384,
     "nll": 0.25059908628463745,
     "brier": 0.09959184378385544,
     "ece": 0.04895997661024053,
     "mean_conf": 0.9357502460479736
    },
    "hellaswag": {
     "n": 71,
     "acc": 0.9718309640884399,
     "nll": 0.07610221207141876,
     "brier": 0.034769345074892044,
     "ece": 0.0420044008320708,
     "mean_conf": 0.9625624418258667
    },
    "imagenet100": {
     "n": 209,
     "acc": 0.9521530866622925,
     "nll": 0.140834778547287,
     "brier": 0.06210394203662872,
     "ece": 0.02685701278059019,
     "mean_conf": 0.9609060883522034
    },
    "jailbreak": {
     "n": 43,
     "acc": 1.0,
     "nll": 0.007107510231435299,
     "brier": 0.0007506122929044068,
     "ece": 0.006904559023125811,
     "mean_conf": 0.9930953979492188
    },
    "mnli": {
     "n": 88,
     "acc": 0.9204545617103577,
     "nll": 0.18531149625778198,
     "brier": 0.10737426578998566,
     "ece": 0.06684734037705664,
     "mean_conf": 0.8966826796531677
    },
    "ms_marco": {
     "n": 77,
     "acc": 0.5714285969734192,
     "nll": 1.1738694906234741,
     "brier": 0.5511745810508728,
     "ece": 0.1849330931448978,
     "mean_conf": 0.5303053855895996
    },
    "newsgroups": {
     "n": 72,
     "acc": 0.8194444179534912,
     "nll": 0.6413953304290771,
     "brier": 0.2439313530921936,
     "ece": 0.0672152123840676,
     "mean_conf": 0.8347481489181519
    },
    "openbookqa": {
     "n": 73,
     "acc": 0.9452054500579834,
     "nll": 0.21645799279212952,
     "brier": 0.10464119911193848,
     "ece": 0.07754900295525902,
     "mean_conf": 0.8753600716590881
    },
    "paws": {
     "n": 75,
     "acc": 0.8799999952316284,
     "nll": 0.23008832335472107,
     "brier": 0.15555186569690704,
     "ece": 0.07505809075017267,
     "mean_conf": 0.9440792798995972
    },
    "prompt_injection": {
     "n": 19,
     "acc": 1.0,
     "nll": 0.0893387570977211,
     "brier": 0.0387185737490654,
     "ece": 0.07711471421890259,
     "mean_conf": 0.9228852987289429
    },
    "qasc": {
     "n": 78,
     "acc": 0.8974359035491943,
     "nll": 0.2294393628835678,
     "brier": 0.12008954584598541,
     "ece": 0.07760370314998621,
     "mean_conf": 0.8928042650222778
    },
    "qnli": {
     "n": 79,
     "acc": 0.9620253443717957,
     "nll": 0.13672247529029846,
     "brier": 0.0658198893070221,
     "ece": 0.04174679779133661,
     "mean_conf": 0.9221668243408203
    },
    "qqp": {
     "n": 67,
     "acc": 0.8507462739944458,
     "nll": 0.3204824924468994,
     "brier": 0.20365533232688904,
     "ece": 0.05547176195417913,
     "mean_conf": 0.8596541285514832
    },
    "race": {
     "n": 71,
     "acc": 0.8169013857841492,
     "nll": 0.508931040763855,
     "brier": 0.2651640772819519,
     "ece": 0.09953184393917758,
     "mean_conf": 0.855144202709198
    },
    "resisc45": {
     "n": 201,
     "acc": 0.9452736377716064,
     "nll": 0.1719132363796234,
     "brier": 0.07461114972829819,
     "ece": 0.03276641192363214,
     "mean_conf": 0.9654509425163269
    },
    "rte": {
     "n": 42,
     "acc": 0.9047619104385376,
     "nll": 0.2149493396282196,
     "brier": 0.12409208714962006,
     "ece": 0.09782490985591286,
     "mean_conf": 0.9137647747993469
    },
    "rvl_cdip": {
     "n": 207,
     "acc": 0.9227052927017212,
     "nll": 0.24763381481170654,
     "brier": 0.11820948868989944,
     "ece": 0.024818566785942367,
     "mean_conf": 0.9276553392410278
    },
    "sciq": {
     "n": 70,
     "acc": 0.9571428298950195,
     "nll": 0.08641297370195389,
     "brier": 0.0542028583586216,
     "ece": 0.05665828745125506,
     "mean_conf": 0.9611032605171204
    },
    "snli": {
     "n": 78,
     "acc": 0.9615384340286255,
     "nll": 0.14562663435935974,
     "brier": 0.07991129904985428,
     "ece": 0.05743888654096996,
     "mean_conf": 0.9150496125221252
    },
    "squad_v2": {
     "n": 83,
     "acc": 0.8795180916786194,
     "nll": 0.3431330621242523,
     "brier": 0.18107561767101288,
     "ece": 0.08313956865378169,
     "mean_conf": 0.9385043978691101
    },
    "sst5": {
     "n": 101,
     "acc": 0.6732673048973083,
     "nll": 0.8503657579421997,
     "brier": 0.49845677614212036,
     "ece": 0.0914942533168388,
     "mean_conf": 0.5979520082473755,
     "score_mae": 0.46103909611701965
    },
    "stanford_cars": {
     "n": 199,
     "acc": 0.9849246144294739,
     "nll": 0.0664161965250969,
     "brier": 0.03385894373059273,
     "ece": 0.02054634083136521,
     "mean_conf": 0.9744680523872375
    },
    "sun397": {
     "n": 205,
     "acc": 0.9658536314964294,
     "nll": 0.12713490426540375,
     "brier": 0.05907195806503296,
     "ece": 0.03913777098139376,
     "mean_conf": 0.973029613494873
    },
    "toxic_chat": {
     "n": 103,
     "acc": 0.9805825352668762,
     "nll": 0.04893926531076431,
     "brier": 0.027793018147349358,
     "ece": 0.0175806231729555,
     "mean_conf": 0.971018373966217
    },
    "tweet_irony": {
     "n": 73,
     "acc": 0.7397260069847107,
     "nll": 0.46547093987464905,
     "brier": 0.3147864043712616,
     "ece": 0.1386445264786409,
     "mean_conf": 0.7930445075035095
    },
    "tweet_offensive": {
     "n": 95,
     "acc": 0.800000011920929,
     "nll": 0.4134097099304199,
     "brier": 0.263822078704834,
     "ece": 0.12641781317580403,
     "mean_conf": 0.8215466737747192
    },
    "tweet_sentiment": {
     "n": 96,
     "acc": 0.7604166865348816,
     "nll": 0.6009461283683777,
     "brier": 0.37088480591773987,
     "ece": 0.09440179414827354,
     "mean_conf": 0.7233335375785828
    },
    "vitaminc": {
     "n": 84,
     "acc": 0.9285714030265808,
     "nll": 0.21254083514213562,
     "brier": 0.11658700555562973,
     "ece": 0.060456716703195634,
     "mean_conf": 0.87336665391922
    },
    "winogrande": {
     "n": 82,
     "acc": 0.8292682766914368,
     "nll": 0.3811376690864563,
     "brier": 0.24891741573810577,
     "ece": 0.07569023195409919,
     "mean_conf": 0.8934732675552368
    },
    "yahoo_topics": {
     "n": 71,
     "acc": 0.7746478915214539,
     "nll": 0.6164750456809998,
     "brier": 0.2843913733959198,
     "ece": 0.10321198388129582,
     "mean_conf": 0.7571716904640198
    },
    "yelp": {
     "n": 127,
     "acc": 0.834645688533783,
     "nll": 0.4197368919849396,
     "brier": 0.23375587165355682,
     "ece": 0.036530637687290035,
     "mean_conf": 0.8449493646621704,
     "score_mae": 0.32252421975135803
    }
   }
  },
  "reliability_raw": [
   {
    "lo": 0.133,
    "hi": 0.2,
    "conf": 0.17327187955379486,
    "acc": 0.0,
    "frac": 0.0007889546104706824
   },
   {
    "lo": 0.2,
    "hi": 0.267,
    "conf": 0.23755492269992828,
    "acc": 0.2222222238779068,
    "frac": 0.001775147975422442
   },
   {
    "lo": 0.267,
    "hi": 0.333,
    "conf": 0.3051908314228058,
    "acc": 0.2800000011920929,
    "frac": 0.004930966533720493
   },
   {
    "lo": 0.333,
    "hi": 0.4,
    "conf": 0.36935192346572876,
    "acc": 0.26923078298568726,
    "frac": 0.0051282052882015705
   },
   {
    "lo": 0.4,
    "hi": 0.467,
    "conf": 0.43677520751953125,
    "acc": 0.42307692766189575,
    "frac": 0.010256410576403141
   },
   {
    "lo": 0.467,
    "hi": 0.533,
    "conf": 0.5065982937812805,
    "acc": 0.5289855003356934,
    "frac": 0.027218934148550034
   },
   {
    "lo": 0.533,
    "hi": 0.6,
    "conf": 0.5669992566108704,
    "acc": 0.5779221057891846,
    "frac": 0.03037475422024727
   },
   {
    "lo": 0.6,
    "hi": 0.667,
    "conf": 0.63744056224823,
    "acc": 0.6171428561210632,
    "frac": 0.03451676666736603
   },
   {
    "lo": 0.667,
    "hi": 0.733,
    "conf": 0.7005413770675659,
    "acc": 0.6280487775802612,
    "frac": 0.03234713897109032
   },
   {
    "lo": 0.733,
    "hi": 0.8,
    "conf": 0.767250657081604,
    "acc": 0.7511312365531921,
    "frac": 0.043589744716882706
   },
   {
    "lo": 0.8,
    "hi": 0.867,
    "conf": 0.834570586681366,
    "acc": 0.80694979429245,
    "frac": 0.051084812730550766
   },
   {
    "lo": 0.867,
    "hi": 0.933,
    "conf": 0.9034878015518188,
    "acc": 0.9073033928871155,
    "frac": 0.07021696120500565
   },
   {
    "lo": 0.933,
    "hi": 1.0,
    "conf": 0.9904627203941345,
    "acc": 0.9853742718696594,
    "frac": 0.687771201133728
   }
  ],
  "reliability_calibrated": [
   {
    "lo": 0.133,
    "hi": 0.2,
    "conf": 0.17058970034122467,
    "acc": 0.0,
    "frac": 0.0009861933067440987
   },
   {
    "lo": 0.2,
    "hi": 0.267,
    "conf": 0.24141208827495575,
    "acc": 0.23076923191547394,
    "frac": 0.0025641026441007853
   },
   {
    "lo": 0.267,
    "hi": 0.333,
    "conf": 0.30261722207069397,
    "acc": 0.3076923191547394,
    "frac": 0.0051282052882015705
   },
   {
    "lo": 0.333,
    "hi": 0.4,
    "conf": 0.3718169331550598,
    "acc": 0.29032257199287415,
    "frac": 0.006114398594945669
   },
   {
    "lo": 0.4,
    "hi": 0.467,
    "conf": 0.4343971312046051,
    "acc": 0.5079365372657776,
    "frac": 0.012426035478711128
   },
   {
    "lo": 0.467,
    "hi": 0.533,
    "conf": 0.5043900012969971,
    "acc": 0.5302013158798218,
    "frac": 0.029388559982180595
   },
   {
    "lo": 0.533,
    "hi": 0.6,
    "conf": 0.5663641691207886,
    "acc": 0.5602409839630127,
    "frac": 0.032741617411375046
   },
   {
    "lo": 0.6,
    "hi": 0.667,
    "conf": 0.6334572434425354,
    "acc": 0.6188119053840637,
    "frac": 0.039842210710048676
   },
   {
    "lo": 0.667,
    "hi": 0.733,
    "conf": 0.7039550542831421,
    "acc": 0.6941747665405273,
    "frac": 0.040631163865327835
   },
   {
    "lo": 0.733,
    "hi": 0.8,
    "conf": 0.7678379416465759,
    "acc": 0.7991452813148499,
    "frac": 0.04615384712815285
   },
   {
    "lo": 0.8,
    "hi": 0.867,
    "conf": 0.8343891501426697,
    "acc": 0.8534798622131348,
    "frac": 0.05384615436196327
   },
   {
    "lo": 0.867,
    "hi": 0.933,
    "conf": 0.9034571051597595,
    "acc": 0.9114832282066345,
    "frac": 0.08244576305150986
   },
   {
    "lo": 0.933,
    "hi": 1.0,
    "conf": 0.9883154630661011,
    "acc": 0.9902557730674744,
    "frac": 0.6477317810058594
   }
  ]
 },
 "heldout": {
  "raw": {
   "overall": {
    "n": 9888,
    "acc": 0.7774069309234619,
    "nll": 0.6259722709655762,
    "brier": 0.3122439980506897,
    "ece": 0.055361537657564505,
    "mean_conf": 0.8327684998512268
   },
   "by_type": {
    "choice": {
     "n": 9088,
     "acc": 0.7644146084785461,
     "nll": 0.6617501974105835,
     "brier": 0.3292696177959442,
     "ece": 0.05688635497695306,
     "mean_conf": 0.8213009238243103
    },
    "noul": {
     "n": 800,
     "acc": 0.925000011920929,
     "nll": 0.21953491866588593,
     "brier": 0.11883323639631271,
     "ece": 0.042017475422659586,
     "mean_conf": 0.9630394577980042
    }
   },
   "by_source": {
    "arc_easy": {
     "n": 500,
     "acc": 0.9760000109672546,
     "nll": 0.10126849263906479,
     "brier": 0.04353426396846771,
     "ece": 0.03141405401313018,
     "mean_conf": 0.9497805237770081
    },
    "askubuntu": {
     "n": 359,
     "acc": 0.5181058645248413,
     "nll": 1.715165376663208,
     "brier": 0.6542366743087769,
     "ece": 0.16121354500097346,
     "mean_conf": 0.679319441318512
    },
    "cifar10": {
     "n": 300,
     "acc": 0.9766666889190674,
     "nll": 0.06821396946907043,
     "brier": 0.033433299511671066,
     "ece": 0.0244025032659336,
     "mean_conf": 0.9716125726699829
    },
    "emotion": {
     "n": 500,
     "acc": 0.5799999833106995,
     "nll": 1.305448293685913,
     "brier": 0.6054953932762146,
     "ece": 0.1998288362110465,
     "mean_conf": 0.7798287868499756
    },
    "food101": {
     "n": 300,
     "acc": 0.8766666650772095,
     "nll": 0.35791701078414917,
     "brier": 0.15910951793193817,
     "ece": 0.04321668811345984,
     "mean_conf": 0.8755965232849121
    },
    "imdb": {
     "n": 500,
     "acc": 0.9399999976158142,
     "nll": 0.15117090940475464,
     "brier": 0.0857759416103363,
     "ece": 0.033096101656337,
     "mean_conf": 0.9650362730026245
    },
    "massive_intent": {
     "n": 500,
     "acc": 0.7459999918937683,
     "nll": 0.9858649969100952,
     "brier": 0.3682084381580353,
     "ece": 0.09032174787909839,
     "mean_conf": 0.8352954983711243
    },
    "mnist": {
     "n": 300,
     "acc": 0.95333331823349,
     "nll": 0.15937867760658264,
     "brier": 0.06808914244174957,
     "ece": 0.03201363153018352,
     "mean_conf": 0.9333906769752502
    },
    "ni_unseen_tasks": {
     "n": 5529,
     "acc": 0.752215564250946,
     "nll": 0.6323504447937012,
     "brier": 0.34408876299858093,
     "ece": 0.05498220219487805,
     "mean_conf": 0.8071978688240051
    },
    "pets": {
     "n": 300,
     "acc": 0.8500000238418579,
     "nll": 0.5367151498794556,
     "brier": 0.22500383853912354,
     "ece": 0.06403641861143206,
     "mean_conf": 0.8197709918022156
    },
    "pope": {
     "n": 300,
     "acc": 0.8999999761581421,
     "nll": 0.33347487449645996,
     "brier": 0.17392873764038086,
     "ece": 0.07330050353340761,
     "mean_conf": 0.9597114324569702
    },
    "pubmed_qa": {
     "n": 500,
     "acc": 0.7080000042915344,
     "nll": 0.738036036491394,
     "brier": 0.4015096426010132,
     "ece": 0.08785723579766086,
     "mean_conf": 0.7890949249267578
    }
   }
  },
  "calibrated": {
   "overall": {
    "n": 9888,
    "acc": 0.7774069309234619,
    "nll": 0.6107043027877808,
    "brier": 0.30924949049949646,
    "ece": 0.040644236667579256,
    "mean_conf": 0.8180511593818665
   },
   "by_type": {
    "choice": {
     "n": 9088,
     "acc": 0.7644146084785461,
     "nll": 0.6470264792442322,
     "brier": 0.32634666562080383,
     "ece": 0.041729311528364656,
     "mean_conf": 0.8061439394950867
    },
    "noul": {
     "n": 800,
     "acc": 0.925000011920929,
     "nll": 0.1980842649936676,
     "brier": 0.11502549052238464,
     "ece": 0.03511890266276535,
     "mean_conf": 0.9533174633979797
    }
   },
   "by_source": {
    "arc_easy": {
     "n": 500,
     "acc": 0.9760000109672546,
     "nll": 0.10715165734291077,
     "brier": 0.04551118239760399,
     "ece": 0.03949616313766471,
     "mean_conf": 0.9417195320129395
    },
    "askubuntu": {
     "n": 359,
     "acc": 0.5181058645248413,
     "nll": 1.6524147987365723,
     "brier": 0.6454375982284546,
     "ece": 0.1338557621591943,
     "mean_conf": 0.651961624622345
    },
    "cifar10": {
     "n": 300,
     "acc": 0.9766666889190674,
     "nll": 0.07086114585399628,
     "brier": 0.03372351825237274,
     "ece": 0.03409448859720851,
     "mean_conf": 0.9671555757522583
    },
    "emotion": {
     "n": 500,
     "acc": 0.5799999833106995,
     "nll": 1.2474008798599243,
     "brier": 0.5944139957427979,
     "ece": 0.17934864653363677,
     "mean_conf": 0.7593486309051514
    },
    "food101": {
     "n": 300,
     "acc": 0.8766666650772095,
     "nll": 0.3655674159526825,
     "brier": 0.1599694937467575,
     "ece": 0.05726011354152369,
     "mean_conf": 0.8594070672988892
    },
    "imdb": {
     "n": 500,
     "acc": 0.9399999976158142,
     "nll": 0.1412155032157898,
     "brier": 0.08349451422691345,
     "ece": 0.025615910650256563,
     "mean_conf": 0.955990195274353
    },
    "massive_intent": {
     "n": 500,
     "acc": 0.7459999918937683,
     "nll": 0.9505149722099304,
     "brier": 0.36250603199005127,
     "ece": 0.07815012133194074,
     "mean_conf": 0.8151710033416748
    },
    "mnist": {
     "n": 300,
     "acc": 0.95333331823349,
     "nll": 0.1676100343465805,
     "brier": 0.07024406641721725,
     "ece": 0.044357041122790236,
     "mean_conf": 0.9220753908157349
    },
    "ni_unseen_tasks": {
     "n": 5529,
     "acc": 0.752215564250946,
     "nll": 0.6206715106964111,
     "brier": 0.34118887782096863,
     "ece": 0.040373205554264546,
     "mean_conf": 0.7925888299942017
    },
    "pets": {
     "n": 300,
     "acc": 0.8500000238418579,
     "nll": 0.5368061065673828,
     "brier": 0.22789603471755981,
     "ece": 0.07454185713334927,
     "mean_conf": 0.8035118579864502
    },
    "pope": {
     "n": 300,
     "acc": 0.8999999761581421,
     "nll": 0.29286548495292664,
     "brier": 0.1675771325826645,
     "ece": 0.06612932776747837,
     "mean_conf": 0.9488630890846252
    },
    "pubmed_qa": {
     "n": 500,
     "acc": 0.7080000042915344,
     "nll": 0.7209590077400208,
     "brier": 0.39785510301589966,
     "ece": 0.07103049902030684,
     "mean_conf": 0.7723875641822815
    }
   }
  },
  "reliability_raw": [
   {
    "lo": 0.133,
    "hi": 0.2,
    "conf": 0.18254680931568146,
    "acc": 0.0,
    "frac": 0.00040453075780533254
   },
   {
    "lo": 0.2,
    "hi": 0.267,
    "conf": 0.24415171146392822,
    "acc": 0.125,
    "frac": 0.004045307636260986
   },
   {
    "lo": 0.267,
    "hi": 0.333,
    "conf": 0.3000350594520569,
    "acc": 0.2711864411830902,
    "frac": 0.011933657340705395
   },
   {
    "lo": 0.333,
    "hi": 0.4,
    "conf": 0.36993393301963806,
    "acc": 0.2829268276691437,
    "frac": 0.020732199773192406
   },
   {
    "lo": 0.4,
    "hi": 0.467,
    "conf": 0.434598833322525,
    "acc": 0.356890469789505,
    "frac": 0.02862055040895939
   },
   {
    "lo": 0.467,
    "hi": 0.533,
    "conf": 0.5052576661109924,
    "acc": 0.4368070960044861,
    "frac": 0.04561084136366844
   },
   {
    "lo": 0.533,
    "hi": 0.6,
    "conf": 0.5669507384300232,
    "acc": 0.4856114983558655,
    "frac": 0.05622977390885353
   },
   {
    "lo": 0.6,
    "hi": 0.667,
    "conf": 0.6334361433982849,
    "acc": 0.5653104782104492,
    "frac": 0.047228965908288956
   },
   {
    "lo": 0.667,
    "hi": 0.733,
    "conf": 0.701598048210144,
    "acc": 0.6368821263313293,
    "frac": 0.05319579318165779
   },
   {
    "lo": 0.733,
    "hi": 0.8,
    "conf": 0.7661896347999573,
    "acc": 0.7110726833343506,
    "frac": 0.058454692363739014
   },
   {
    "lo": 0.8,
    "hi": 0.867,
    "conf": 0.8357764482498169,
    "acc": 0.7466007471084595,
    "frac": 0.08181634545326233
   },
   {
    "lo": 0.867,
    "hi": 0.933,
    "conf": 0.9009120464324951,
    "acc": 0.8152466416358948,
    "frac": 0.1127629429101944
   },
   {
    "lo": 0.933,
    "hi": 1.0,
    "conf": 0.9835764169692993,
    "acc": 0.9503800868988037,
    "frac": 0.4789643883705139
   }
  ],
  "reliability_calibrated": [
   {
    "lo": 0.133,
    "hi": 0.2,
    "conf": 0.17538046836853027,
    "acc": 0.0,
    "frac": 0.0006067961221560836
   },
   {
    "lo": 0.2,
    "hi": 0.267,
    "conf": 0.24302583932876587,
    "acc": 0.16981132328510284,
    "frac": 0.005360032431781292
   },
   {
    "lo": 0.267,
    "hi": 0.333,
    "conf": 0.30060428380966187,
    "acc": 0.25362318754196167,
    "frac": 0.013956310227513313
   },
   {
    "lo": 0.333,
    "hi": 0.4,
    "conf": 0.36858007311820984,
    "acc": 0.3234042525291443,
    "frac": 0.023766180500388145
   },
   {
    "lo": 0.4,
    "hi": 0.467,
    "conf": 0.4340149462223053,
    "acc": 0.3523809611797333,
    "frac": 0.03185679763555527
   },
   {
    "lo": 0.467,
    "hi": 0.533,
    "conf": 0.5055366158485413,
    "acc": 0.45656564831733704,
    "frac": 0.05006067827343941
   },
   {
    "lo": 0.533,
    "hi": 0.6,
    "conf": 0.5651605129241943,
    "acc": 0.5167548656463623,
    "frac": 0.05734223127365112
   },
   {
    "lo": 0.6,
    "hi": 0.667,
    "conf": 0.6337770223617554,
    "acc": 0.5630081295967102,
    "frac": 0.04975728318095207
   },
   {
    "lo": 0.667,
    "hi": 0.733,
    "conf": 0.7002703547477722,
    "acc": 0.6780238747596741,
    "frac": 0.059364885091781616
   },
   {
    "lo": 0.733,
    "hi": 0.8,
    "conf": 0.7682633399963379,
    "acc": 0.7181528806686401,
    "frac": 0.06351132690906525
   },
   {
    "lo": 0.8,
    "hi": 0.867,
    "conf": 0.8352631330490112,
    "acc": 0.7694915533065796,
    "frac": 0.08950242400169373
   },
   {
    "lo": 0.867,
    "hi": 0.933,
    "conf": 0.9024354219436646,
    "acc": 0.8416013717651367,
    "frac": 0.11620145291090012
   },
   {
    "lo": 0.933,
    "hi": 1.0,
    "conf": 0.9815592169761658,
    "acc": 0.9596588015556335,
    "frac": 0.43871358036994934
   }
  ]
 },
 "fork_invariance_max_dp": 6.145238876342773e-05,
 "latency_ms": {
  "1": {
   "fork": 108.40368270874023,
   "separate": 959.8970413208008
  },
  "4": {
   "fork": 107.42926597595215,
   "separate": 1998.2314109802246
  },
  "16": {
   "fork": 113.10410499572754,
   "separate": 7827.5275230407715
  },
  "64": {
   "fork": 256.7019462585449,
   "separate": 16031.468868255615
  }
 }
}