{"models":{"decoice":{"1":{"n":935,"overall":848,"natural_n":800,"natural":713,"confident_errors":9,"confidence_wrong":0.6667966682692579,"brier":0.14326292062762191,"nll":0.2672360843471884,"ece10":0.03666987612654181,"ece15":0.03699542128663267,"language_ms":{"n":875,"mean":72.80608609252211,"p95":599.1335019934922},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"boolq":[86,100],"winogrande":[94,100],"emotion":[79,100],"paws":[93,100],"trec":[97,100],"policy_oracle":[120,120],"mrpc":[81,100],"sst2":[95,100],"ag_news":[88,100],"ordinal_oracle":[15,15]}},"combined":{"n":2805,"overall":2539,"natural_n":2400,"natural":2134,"confident_errors":19,"confidence_wrong":0.6605297362377168,"brier":0.14333851052946703,"nll":0.2590862649084073,"ece10":0.03346622791104732,"ece15":0.034191105635707535,"language_ms":{"n":2625,"mean":52.261985840808066,"p95":47.21850994974375},"requests":2625,"failed_requests":0,"all_hard_judgments":2805,"all_natural_judgments":2400,"families":{"boolq":[261,300],"winogrande":[283,300],"emotion":[239,300],"paws":[270,300],"trec":[286,300],"policy_oracle":[360,360],"mrpc":[227,300],"sst2":[286,300],"ag_news":[282,300],"ordinal_oracle":[45,45]}},"2":{"n":935,"overall":853,"natural_n":800,"natural":718,"confident_errors":7,"confidence_wrong":0.6874811834979957,"brier":0.14141860895701838,"nll":0.25276606287728265,"ece10":0.03472629828249636,"ece15":0.03656158074568877,"language_ms":{"n":875,"mean":44.9003177910511,"p95":46.80827306583524},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"policy_oracle":[120,120],"paws":[92,100],"ag_news":[98,100],"winogrande":[94,100],"emotion":[82,100],"trec":[96,100],"sst2":[93,100],"boolq":[87,100],"ordinal_oracle":[15,15],"mrpc":[76,100]}},"3":{"n":935,"overall":838,"natural_n":800,"natural":703,"confident_errors":3,"confidence_wrong":0.6321251819893978,"brier":0.14533400200376076,"nll":0.257256647500751,"ece10":0.03745212361752029,"ece15":0.03756102649762482,"language_ms":{"n":875,"mean":39.079553638851,"p95":47.161701950244606},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"paws":[85,100],"winogrande":[95,100],"trec":[93,100],"ag_news":[96,100],"policy_oracle":[120,120],"boolq":[88,100],"ordinal_oracle":[15,15],"emotion":[78,100],"sst2":[98,100],"mrpc":[70,100]}}},"prism":{"1":{"n":935,"overall":829,"natural_n":800,"natural":694,"confident_errors":23,"confidence_wrong":0.7310855064172586,"brier":0.16438837556617902,"nll":0.3082888238509653,"ece10":0.021918651669970833,"ece15":0.031426452217786616,"language_ms":{"n":875,"mean":75.36461990193597,"p95":106.73772206064314},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"boolq":[88,100],"winogrande":[99,100],"emotion":[57,100],"paws":[91,100],"trec":[92,100],"policy_oracle":[120,120],"mrpc":[76,100],"sst2":[97,100],"ag_news":[94,100],"ordinal_oracle":[15,15]}},"combined":{"n":2805,"overall":2507,"natural_n":2400,"natural":2103,"confident_errors":66,"confidence_wrong":0.7397540621235225,"brier":0.16021232628704707,"nll":0.3075223380434366,"ece10":0.014905736262247778,"ece15":0.01964303036052214,"language_ms":{"n":2625,"mean":75.37189864225331,"p95":106.58392403274775},"requests":2625,"failed_requests":0,"all_hard_judgments":2805,"all_natural_judgments":2400,"families":{"boolq":[271,300],"winogrande":[299,300],"emotion":[186,300],"paws":[274,300],"trec":[271,300],"policy_oracle":[360,360],"mrpc":[225,300],"sst2":[289,300],"ag_news":[288,300],"ordinal_oracle":[44,45]}},"2":{"n":935,"overall":841,"natural_n":800,"natural":706,"confident_errors":20,"confidence_wrong":0.7454168275323387,"brier":0.15322387223298672,"nll":0.2928198295502985,"ece10":0.01936242272632747,"ece15":0.021026887722270054,"language_ms":{"n":875,"mean":75.34200746206832,"p95":106.5422990359366},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"policy_oracle":[120,120],"paws":[92,100],"ag_news":[98,100],"winogrande":[100,100],"emotion":[66,100],"trec":[92,100],"sst2":[95,100],"boolq":[91,100],"ordinal_oracle":[15,15],"mrpc":[72,100]}},"3":{"n":935,"overall":837,"natural_n":800,"natural":703,"confident_errors":23,"confidence_wrong":0.7436986229034739,"brier":0.16302473106197546,"nll":0.32145836072904593,"ece10":0.01285169457652081,"ece15":0.01964282460520752,"language_ms":{"n":875,"mean":75.40906856275562,"p95":106.47430701646954},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"paws":[91,100],"winogrande":[100,100],"trec":[87,100],"ag_news":[96,100],"policy_oracle":[120,120],"boolq":[92,100],"ordinal_oracle":[14,15],"emotion":[63,100],"sst2":[97,100],"mrpc":[77,100]}}},"jev114":{"1":{"n":935,"overall":807,"natural_n":800,"natural":676,"confident_errors":46,"confidence_wrong":0.791015625,"brier":0.2221597860962567,"nll":0.7912725824255779,"ece10":0.060909090909090816,"ece15":0.06090909090909085,"language_ms":{"n":875,"mean":142.06432922211076,"p95":176.97670200141147},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"boolq":[89,100],"winogrande":[98,100],"emotion":[53,100],"paws":[93,100],"trec":[89,100],"policy_oracle":[117,120],"mrpc":[71,100],"sst2":[95,100],"ag_news":[88,100],"ordinal_oracle":[14,15]}},"combined":{"n":2805,"overall":2414,"natural_n":2400,"natural":2022,"confident_errors":142,"confidence_wrong":0.7778260869565218,"brier":0.2168627450980392,"nll":0.9635567699981177,"ece10":0.05107664884135458,"ece15":0.051811051693404525,"language_ms":{"n":2625,"mean":147.11850044341935,"p95":188.40189598267898},"requests":2625,"failed_requests":0,"all_hard_judgments":2805,"all_natural_judgments":2400,"families":{"boolq":[274,300],"winogrande":[288,300],"emotion":[173,300],"paws":[257,300],"trec":[264,300],"policy_oracle":[349,360],"mrpc":[210,300],"sst2":[285,300],"ag_news":[271,300],"ordinal_oracle":[43,45]}},"2":{"n":935,"overall":798,"natural_n":800,"natural":666,"confident_errors":43,"confidence_wrong":0.7514598540145987,"brier":0.21032491978609624,"nll":1.0316104765932381,"ece10":0.05724064171122988,"ece15":0.05743315508021388,"language_ms":{"n":875,"mean":155.52935607312247,"p95":194.67835000250489},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"policy_oracle":[118,120],"paws":[82,100],"ag_news":[90,100],"winogrande":[94,100],"emotion":[60,100],"trec":[90,100],"sst2":[93,100],"boolq":[90,100],"ordinal_oracle":[14,15],"mrpc":[67,100]}},"3":{"n":935,"overall":809,"natural_n":800,"natural":680,"confident_errors":53,"confidence_wrong":0.7930952380952383,"brier":0.2181035294117647,"nll":1.0677872509755368,"ece10":0.0482780748663101,"ece15":0.05721925133689833,"language_ms":{"n":875,"mean":143.76181603502482,"p95":186.4116860087961},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"paws":[82,100],"winogrande":[96,100],"trec":[85,100],"ag_news":[93,100],"policy_oracle":[114,120],"boolq":[95,100],"ordinal_oracle":[15,15],"emotion":[60,100],"sst2":[97,100],"mrpc":[72,100]}}},"jev113":{"1":{"n":935,"overall":800,"natural_n":800,"natural":665,"confident_errors":48,"confidence_wrong":0.7745185185185185,"brier":0.21832534759358288,"nll":0.8840862624918085,"ece10":0.06306951871657744,"ece15":0.0633903743315507,"language_ms":{"n":875,"mean":143.2197958847641,"p95":182.6201730291359},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"boolq":[91,100],"winogrande":[98,100],"emotion":[51,100],"paws":[87,100],"trec":[88,100],"policy_oracle":[120,120],"mrpc":[71,100],"sst2":[92,100],"ag_news":[87,100],"ordinal_oracle":[15,15]}},"combined":{"n":2805,"overall":2424,"natural_n":2400,"natural":2025,"confident_errors":144,"confidence_wrong":0.7809973753280839,"brier":0.2108951158645276,"nll":1.0765638819983052,"ece10":0.05283422459893029,"ece15":0.052663101604277854,"language_ms":{"n":2625,"mean":143.8657254625245,"p95":185.89383602375165},"requests":2625,"failed_requests":0,"all_hard_judgments":2805,"all_natural_judgments":2400,"families":{"boolq":[273,300],"winogrande":[291,300],"emotion":[173,300],"paws":[256,300],"trec":[263,300],"policy_oracle":[355,360],"mrpc":[217,300],"sst2":[283,300],"ag_news":[269,300],"ordinal_oracle":[44,45]}},"2":{"n":935,"overall":809,"natural_n":800,"natural":677,"confident_errors":48,"confidence_wrong":0.7688888888888888,"brier":0.20463058823529412,"nll":1.09647433855261,"ece10":0.05666310160427799,"ece15":0.056213903743315474,"language_ms":{"n":875,"mean":144.48803135386802,"p95":189.69511898467317},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"policy_oracle":[118,120],"paws":[82,100],"ag_news":[90,100],"winogrande":[96,100],"emotion":[63,100],"trec":[91,100],"sst2":[94,100],"boolq":[89,100],"ordinal_oracle":[14,15],"mrpc":[72,100]}},"3":{"n":935,"overall":815,"natural_n":800,"natural":683,"confident_errors":48,"confidence_wrong":0.801,"brier":0.20972941176470586,"nll":1.2491310449504973,"ece10":0.051689839572192475,"ece15":0.05256684491978605,"language_ms":{"n":875,"mean":143.88934914894136,"p95":186.15067296195775},"requests":875,"failed_requests":0,"all_hard_judgments":935,"all_natural_judgments":800,"families":{"paws":[87,100],"winogrande":[97,100],"trec":[84,100],"ag_news":[92,100],"policy_oracle":[117,120],"boolq":[93,100],"ordinal_oracle":[15,15],"emotion":[59,100],"sst2":[97,100],"mrpc":[74,100]}}}},"paired":{"prism":{"1":{"overall":{"decoice_only":60,"comparator_only":41,"common":935,"delta_pp":2.0320855614973263},"natural":{"decoice_only":60,"comparator_only":41,"common":800,"delta_pp":2.375,"paired_exact_p":0.07275700687752425}},"2":{"overall":{"decoice_only":53,"comparator_only":41,"common":935,"delta_pp":1.2834224598930482},"natural":{"decoice_only":53,"comparator_only":41,"common":800,"delta_pp":1.5,"paired_exact_p":0.25644234893296985}},"3":{"overall":{"decoice_only":55,"comparator_only":54,"common":935,"delta_pp":0.10695187165775401},"natural":{"decoice_only":54,"comparator_only":54,"common":800,"delta_pp":0.0,"paired_exact_p":1.0}},"combined":{"overall":{"decoice_only":168,"comparator_only":136,"common":2805,"delta_pp":1.1408199643493762},"natural":{"decoice_only":167,"comparator_only":136,"common":2400,"delta_pp":1.2916666666666667,"paired_exact_p":0.08463736856943596}}},"jev114":{"1":{"overall":{"decoice_only":71,"comparator_only":30,"common":935,"delta_pp":4.385026737967914},"natural":{"decoice_only":67,"comparator_only":30,"common":800,"delta_pp":4.625,"paired_exact_p":0.00021890976301036633}},"2":{"overall":{"decoice_only":86,"comparator_only":31,"common":935,"delta_pp":5.882352941176471},"natural":{"decoice_only":83,"comparator_only":31,"common":800,"delta_pp":6.5,"paired_exact_p":1.1803379442916355e-06}},"3":{"overall":{"decoice_only":68,"comparator_only":39,"common":935,"delta_pp":3.1016042780748663},"natural":{"decoice_only":62,"comparator_only":39,"common":800,"delta_pp":2.875,"paired_exact_p":0.028089467947778165}},"combined":{"overall":{"decoice_only":225,"comparator_only":100,"common":2805,"delta_pp":4.45632798573975},"natural":{"decoice_only":212,"comparator_only":100,"common":2400,"delta_pp":4.666666666666667,"paired_exact_p":2.126729907047434e-10}}},"jev113":{"1":{"overall":{"decoice_only":78,"comparator_only":30,"common":935,"delta_pp":5.133689839572193},"natural":{"decoice_only":78,"comparator_only":30,"common":800,"delta_pp":6.0,"paired_exact_p":4.314751521794875e-06}},"2":{"overall":{"decoice_only":76,"comparator_only":32,"common":935,"delta_pp":4.705882352941177},"natural":{"decoice_only":73,"comparator_only":32,"common":800,"delta_pp":5.125,"paired_exact_p":7.808423662658282e-05}},"3":{"overall":{"decoice_only":63,"comparator_only":40,"common":935,"delta_pp":2.4598930481283423},"natural":{"decoice_only":60,"comparator_only":40,"common":800,"delta_pp":2.5,"paired_exact_p":0.05688793364098089}},"combined":{"overall":{"decoice_only":217,"comparator_only":102,"common":2805,"delta_pp":4.09982174688057},"natural":{"decoice_only":211,"comparator_only":102,"common":2400,"delta_pp":4.541666666666667,"paired_exact_p":6.996824398009984e-10}}}},"date":"2026-11-19","requests_per_model":2625,"confidence_thresholds":{"decoice":{"combined":{"0.5":{"accepted":2765,"errors":244,"expected_rate":0.12138960275292054,"error_rate":0.08824593128390597},"0.7":{"accepted":2342,"errors":101,"expected_rate":0.07112500129040485,"error_rate":0.04312553373185312},"0.9":{"accepted":1627,"errors":19,"expected_rate":0.02301623775769987,"error_rate":0.011677934849416103},"0.95":{"accepted":1358,"errors":12,"expected_rate":0.01280983416310919,"error_rate":0.008836524300441826}},"1":{"0.5":{"accepted":922,"errors":78,"expected_rate":0.11990267564676127,"error_rate":0.08459869848156182},"0.7":{"accepted":784,"errors":33,"expected_rate":0.0711335470031772,"error_rate":0.042091836734693876},"0.9":{"accepted":546,"errors":9,"expected_rate":0.024276030875557644,"error_rate":0.016483516483516484},"0.95":{"accepted":446,"errors":7,"expected_rate":0.013074409556161135,"error_rate":0.01569506726457399}},"2":{"0.5":{"accepted":923,"errors":77,"expected_rate":0.11663046946620714,"error_rate":0.08342361863488625},"0.7":{"accepted":794,"errors":40,"expected_rate":0.07003921590421959,"error_rate":0.05037783375314862},"0.9":{"accepted":557,"errors":7,"expected_rate":0.023810786989977958,"error_rate":0.012567324955116697},"0.95":{"accepted":455,"errors":4,"expected_rate":0.012445667459661847,"error_rate":0.008791208791208791}},"3":{"0.5":{"accepted":920,"errors":89,"expected_rate":0.12765441450891543,"error_rate":0.0967391304347826},"0.7":{"accepted":764,"errors":28,"expected_rate":0.07224465280587286,"error_rate":0.03664921465968586},"0.9":{"accepted":524,"errors":3,"expected_rate":0.020858964924247117,"error_rate":0.0057251908396946565},"0.95":{"accepted":457,"errors":1,"expected_rate":0.012914200081637362,"error_rate":0.002188183807439825}}},"prism":{"combined":{"0.5":{"accepted":2780,"errors":280,"expected_rate":0.09137057603355538,"error_rate":0.10071942446043165},"0.7":{"accepted":2527,"errors":173,"expected_rate":0.06075943210674879,"error_rate":0.06846062524732885},"0.9":{"accepted":2080,"errors":66,"expected_rate":0.03406832110624934,"error_rate":0.03173076923076923},"0.95":{"accepted":1717,"errors":28,"expected_rate":0.02655459713273788,"error_rate":0.016307513104251603}},"1":{"0.5":{"accepted":924,"errors":98,"expected_rate":0.08999379258714751,"error_rate":0.10606060606060606},"0.7":{"accepted":838,"errors":59,"expected_rate":0.058786046304162363,"error_rate":0.07040572792362769},"0.9":{"accepted":703,"errors":23,"expected_rate":0.034583289048759944,"error_rate":0.032716927453769556},"0.95":{"accepted":574,"errors":11,"expected_rate":0.026486174434787124,"error_rate":0.01916376306620209}},"2":{"0.5":{"accepted":927,"errors":88,"expected_rate":0.0888156850054651,"error_rate":0.09492988133764833},"0.7":{"accepted":854,"errors":58,"expected_rate":0.06218465618770471,"error_rate":0.06791569086651054},"0.9":{"accepted":696,"errors":20,"expected_rate":0.03412861833970909,"error_rate":0.028735632183908046},"0.95":{"accepted":572,"errors":6,"expected_rate":0.02656718789540554,"error_rate":0.01048951048951049}},"3":{"0.5":{"accepted":929,"errors":94,"expected_rate":0.09528934017512754,"error_rate":0.10118406889128095},"0.7":{"accepted":835,"errors":56,"expected_rate":0.06128225358870218,"error_rate":0.06706586826347305},"0.9":{"accepted":681,"errors":23,"expected_rate":0.03347509153492346,"error_rate":0.033773861967694566},"0.95":{"accepted":571,"errors":11,"expected_rate":0.02661076650642935,"error_rate":0.01926444833625219}}},"jev114":{"combined":{"0.5":{"accepted":2785,"errors":383,"expected_rate":0.08742190305206464,"error_rate":0.13752244165170557},"0.7":{"accepted":2506,"errors":251,"expected_rate":0.05183958499600958,"error_rate":0.10015961691939346},"0.9":{"accepted":2053,"errors":142,"expected_rate":0.02110569897710668,"error_rate":0.069167072576717},"0.95":{"accepted":1740,"errors":93,"expected_rate":0.011017241379310354,"error_rate":0.05344827586206897}},"1":{"0.5":{"accepted":927,"errors":126,"expected_rate":0.08836030204962245,"error_rate":0.13592233009708737},"0.7":{"accepted":837,"errors":90,"expected_rate":0.054026284348865,"error_rate":0.10752688172043011},"0.9":{"accepted":680,"errors":46,"expected_rate":0.021367647058823536,"error_rate":0.06764705882352941},"0.95":{"accepted":571,"errors":26,"expected_rate":0.010542907180385298,"error_rate":0.04553415061295972}},"2":{"0.5":{"accepted":930,"errors":134,"expected_rate":0.08816129032258065,"error_rate":0.14408602150537633},"0.7":{"accepted":824,"errors":76,"expected_rate":0.04720873786407768,"error_rate":0.09223300970873786},"0.9":{"accepted":692,"errors":43,"expected_rate":0.02037572254335261,"error_rate":0.06213872832369942},"0.95":{"accepted":596,"errors":33,"expected_rate":0.01122483221476511,"error_rate":0.05536912751677853}},"3":{"0.5":{"accepted":928,"errors":123,"expected_rate":0.08574353448275862,"error_rate":0.13254310344827586},"0.7":{"accepted":845,"errors":85,"expected_rate":0.05418934911242604,"error_rate":0.10059171597633136},"0.9":{"accepted":681,"errors":53,"expected_rate":0.021585903083700446,"error_rate":0.07782672540381791},"0.95":{"accepted":573,"errors":34,"expected_rate":0.011273996509598615,"error_rate":0.059336823734729496}}},"jev113":{"combined":{"0.5":{"accepted":2786,"errors":371,"expected_rate":0.08058506819813353,"error_rate":0.13316582914572864},"0.7":{"accepted":2515,"errors":244,"expected_rate":0.04587673956262426,"error_rate":0.09701789264413518},"0.9":{"accepted":2109,"errors":144,"expected_rate":0.019108582266477013,"error_rate":0.06827880512091039},"0.95":{"accepted":1856,"errors":106,"expected_rate":0.01130926724137932,"error_rate":0.057112068965517244}},"1":{"0.5":{"accepted":929,"errors":133,"expected_rate":0.08205597416576965,"error_rate":0.14316469321851452},"0.7":{"accepted":834,"errors":84,"expected_rate":0.046282973621103125,"error_rate":0.10071942446043165},"0.9":{"accepted":702,"errors":48,"expected_rate":0.019544159544159553,"error_rate":0.06837606837606838},"0.95":{"accepted":611,"errors":30,"expected_rate":0.01111292962356793,"error_rate":0.049099836333878884}},"2":{"0.5":{"accepted":927,"errors":121,"expected_rate":0.08006472491909386,"error_rate":0.13052858683926646},"0.7":{"accepted":833,"errors":77,"expected_rate":0.04346938775510205,"error_rate":0.09243697478991597},"0.9":{"accepted":705,"errors":48,"expected_rate":0.018212765957446815,"error_rate":0.06808510638297872},"0.95":{"accepted":633,"errors":38,"expected_rate":0.011358609794628763,"error_rate":0.06003159557661927}},"3":{"0.5":{"accepted":930,"errors":117,"expected_rate":0.07963440860215054,"error_rate":0.12580645161290321},"0.7":{"accepted":848,"errors":83,"expected_rate":0.04784198113207548,"error_rate":0.09787735849056604},"0.9":{"accepted":702,"errors":48,"expected_rate":0.01957264957264958,"error_rate":0.06837606837606838},"0.95":{"accepted":612,"errors":38,"expected_rate":0.011454248366013083,"error_rate":0.06209150326797386}}}}}