{
  "model": "decoice-1.0.0",
  "runDateUtc": "2026-11-18",
  "endpoint": "Production Decisions API via HTTPS loopback from the serving VM",
  "datasets": {
    "when2call": {
      "revision": "0582f7749df63a96fdc3070932e83e72396ace53",
      "source": "https://huggingface.co/datasets/nvidia/When2Call",
      "split": "MCQ test"
    },
    "belebele": {
      "revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
      "source": "https://huggingface.co/datasets/facebook/belebele",
      "split": "Test"
    }
  },
  "results": [
    {
      "id": "when2call",
      "name": "When2Call · Tool-use decisions",
      "source_sha256": "8c3694e583eeeb8dbc297e6cd90da70efc68efa4b6adb7227523e828c6b7b14c",
      "cases": 3652,
      "correct": 2296,
      "accuracy": 0.628696604600219,
      "ece15": 0.15898081574189263,
      "brier": 0.5188214176611197,
      "confidence_when_wrong": 0.6856544029245938,
      "wrong_at_90": 99,
      "latency_ms": {
        "mean": 446.23160220153284,
        "median": 397.76889149970884,
        "p95": 937.7726760003497,
        "p99": 2238.2185080005
      }
    },
    {
      "id": "eng_Latn",
      "name": "Belebele · English",
      "source_sha256": "15af884c2b5994fdc71199e79d553a1864b08ce1b31894aa87f6be925a064a31",
      "cases": 900,
      "correct": 722,
      "accuracy": 0.8022222222222222,
      "ece15": 0.15726506927048073,
      "brier": 0.28739685860711195,
      "confidence_when_wrong": 0.47043601717655004,
      "wrong_at_90": 0,
      "latency_ms": {
        "mean": 231.93395687004238,
        "median": 334.07973449993733,
        "p95": 362.47698300030606,
        "p99": 377.40298499920755
      }
    }
  ],
  "when2callFilteredAnalysis": {
    "methodology": "https://artificialanalysis.ai/methodology/decision-index#when2call",
    "scope": "Saved Decoice predictions rescored on the 3,423 cases retained by Artificial Analysis, using our Decisions API request format",
    "cases": 3423,
    "excluded": 229,
    "correct": 2253,
    "accuracy": 0.6581945661700263,
    "ties": 0,
    "decision_score": 73.45415880657411,
    "ece15": 0.1400621324192123,
    "wrong_at_90": 44,
    "latency_ms": {
      "median": 403.7655790007193,
      "p95": 929.4017650008755
    }
  },
  "comparators": {
    "when2call": {
      "prism": {
        "cases": 3652,
        "correct": 3280,
        "accuracy": 0.8981380065717415,
        "ece10": 0.03495365375528461,
        "ece15": 0.03645035360918379,
        "brier": 0.15519144531416093,
        "log_loss": 0.3093551238228957,
        "confidence_when_wrong": 0.753259235011634,
        "wrong_at_90": 95,
        "latency_ms": {
          "mean": 164.5581662517295,
          "median": 150.89758450631052,
          "p95": 264.1326740849763,
          "p99": 300.9666970465332
        },
        "source_sha256": "8c3694e583eeeb8dbc297e6cd90da70efc68efa4b6adb7227523e828c6b7b14c",
        "filtered": {
          "cases": 3423,
          "correct": 3175,
          "accuracy": 0.927548933683903,
          "decision_score": 93.63941439999121,
          "ece15": 0.038318810757579355,
          "wrong_at_90": 53
        }
      },
      "jev114": {
        "cases": 3652,
        "correct": 2960,
        "accuracy": 0.8105147864184009,
        "ece10": 0.06125684556407447,
        "ece15": 0.060566812705366915,
        "brier": 0.28707672508214677,
        "log_loss": 1.0991022995387996,
        "confidence_when_wrong": 0.738742774566474,
        "wrong_at_90": 194,
        "latency_ms": {
          "mean": 153.98617288493844,
          "median": 147.9224390059244,
          "p95": 201.41071802936494,
          "p99": 258.8714649900794
        },
        "source_sha256": "8c3694e583eeeb8dbc297e6cd90da70efc68efa4b6adb7227523e828c6b7b14c",
        "filtered": {
          "cases": 3423,
          "correct": 2904,
          "accuracy": 0.8483786152497809,
          "decision_score": 87.37793894244814,
          "ece15": 0.023835816535203034,
          "wrong_at_90": 93
        }
      },
      "jev113": {
        "cases": 3652,
        "correct": 2949,
        "accuracy": 0.8075027382256298,
        "ece10": 0.06030941949616648,
        "ece15": 0.06096111719605695,
        "brier": 0.29282437020810514,
        "log_loss": 1.0834109826976317,
        "confidence_when_wrong": 0.7364864864864865,
        "wrong_at_90": 196,
        "latency_ms": {
          "mean": 145.2298743549203,
          "median": 140.6706259585917,
          "p95": 186.21633597649634,
          "p99": 244.5824549649842
        },
        "source_sha256": "8c3694e583eeeb8dbc297e6cd90da70efc68efa4b6adb7227523e828c6b7b14c",
        "filtered": {
          "cases": 3423,
          "correct": 2896,
          "accuracy": 0.8460414840782939,
          "decision_score": 87.13620946538124,
          "ece15": 0.023526146654981008,
          "wrong_at_90": 89
        }
      }
    },
    "eng_Latn": {
      "prism": {
        "cases": 900,
        "correct": 868,
        "accuracy": 0.9644444444444444,
        "ece10": 0.03757381992207633,
        "ece15": 0.040280836522579196,
        "brier": 0.0615334500201821,
        "log_loss": 0.16620273874435967,
        "confidence_when_wrong": 0.7629223829135299,
        "wrong_at_90": 8,
        "latency_ms": {
          "mean": 83.74130057119247,
          "median": 83.81097251549363,
          "p95": 87.25209999829531,
          "p99": 96.58862301148474
        },
        "source_sha256": "15af884c2b5994fdc71199e79d553a1864b08ce1b31894aa87f6be925a064a31"
      },
      "jev114": {
        "cases": 900,
        "correct": 871,
        "accuracy": 0.9677777777777777,
        "ece10": 0.014222222222222221,
        "ece15": 0.0156,
        "brier": 0.049355555555555555,
        "log_loss": 0.403440131794342,
        "confidence_when_wrong": 0.7417241379310345,
        "wrong_at_90": 8,
        "latency_ms": {
          "mean": 149.45977811895622,
          "median": 142.2344470047392,
          "p95": 189.44699404528365,
          "p99": 298.72838902520016
        },
        "source_sha256": "15af884c2b5994fdc71199e79d553a1864b08ce1b31894aa87f6be925a064a31"
      },
      "jev113": {
        "cases": 900,
        "correct": 875,
        "accuracy": 0.9722222222222222,
        "ece10": 0.012188888888888888,
        "ece15": 0.016277777777777776,
        "brier": 0.04599,
        "log_loss": 0.4003144721918202,
        "confidence_when_wrong": 0.7916,
        "wrong_at_90": 10,
        "latency_ms": {
          "mean": 146.4196370452151,
          "median": 140.9782335103955,
          "p95": 192.30843702098355,
          "p99": 288.5723889921792
        },
        "source_sha256": "15af884c2b5994fdc71199e79d553a1864b08ce1b31894aa87f6be925a064a31"
      }
    }
  },
  "comparatorRuntimes": {
    "jev114": {
      "version": "typesafe/jev-1.14-20261014",
      "transport": "Hosted OpenRouter Decisions API",
      "concurrentRequests": 6
    },
    "jev113": {
      "version": "typesafe/jev-1.13-20260917",
      "transport": "Hosted OpenRouter Decisions API",
      "concurrentRequests": 6
    },
    "prism": {
      "version": "Sarvam Prism",
      "transport": "Resident native Prism service on one NVIDIA H200 via Unix socket",
      "concurrentRequests": 1
    }
  },
  "comparatorRunDateUtc": "2026-11-19",
  "review": {
    "caseCoverage": "Complete for all four models on the two pinned English splits",
    "requestFormattersSha256NormalizedAst": {
      "request_for": "fabb9a7c397d92705fec88d8cca74351bd3ec2f98b217b1a606862d1662d4207",
      "reading_request": "7570886c58cb1cd4ea328720d9a23e97407d71576c440416c74136fd0a564512"
    },
    "method": "Dataset hashes and source case membership verified. A second scoring implementation checked comparator accuracy, ECE15, confident errors and the filtered Decision Score."
  }
}
