{
  "run_index": 1,
  "evaluated_utc": "2026-09-22T06:32:28Z",
  "test_file": "all/test-00000-of-00001.parquet",
  "test_sha256": "4f294f218ea1da27f3efef936359389c62ea4d3973a41457732990f1d31b647c",
  "n_cases": 400,
  "n_decisions": 2000,
  "latency_ms_per_case_cpu": 134.04,
  "raw": {
    "overall": {
      "n": 2000,
      "accuracy": 0.751,
      "soft_accuracy": 0.5634989456557318,
      "brier": 0.05914920990215329,
      "kl": 0.10238191010682485,
      "tv": 0.1433847785776465,
      "nll": 0.6367698398652486,
      "ece": 0.12722164873538963,
      "score_mae": 0.2746505811582656,
      "within_1": 0.98375
    },
    "by_workflow": {
      "agent_trace_observability": {
        "n": 500,
        "accuracy": 0.732,
        "soft_accuracy": 0.4724501203294106,
        "brier": 0.05628207400044921,
        "kl": 0.09350560088046769,
        "tv": 0.15358126436337113,
        "nll": 0.7006394391227598,
        "ece": 0.1601141162832159,
        "score_mae": 0.2100908384556247,
        "within_1": 1.0
      },
      "customer_service": {
        "n": 500,
        "accuracy": 0.728,
        "soft_accuracy": 0.5469157003096765,
        "brier": 0.092352802056536,
        "kl": 0.16993479269228093,
        "tv": 0.1916863908237952,
        "nll": 0.6922485950267834,
        "ece": 0.11188648161232355,
        "score_mae": 0.2972645950098066,
        "within_1": 0.975
      },
      "invoice_processing": {
        "n": 500,
        "accuracy": 0.808,
        "soft_accuracy": 0.7196525728518464,
        "brier": 0.04959251157667264,
        "kl": 0.0847333468242194,
        "tv": 0.11005123578831648,
        "nll": 0.4740942175604576,
        "ece": 0.09652193226496708,
        "score_mae": 0.2800105412177077,
        "within_1": 0.975
      },
      "security_incidents": {
        "n": 500,
        "accuracy": 0.736,
        "soft_accuracy": 0.5149773891319935,
        "brier": 0.038369451974955324,
        "kl": 0.06135390003033135,
        "tv": 0.11822022333510321,
        "nll": 0.6800971077509935,
        "ece": 0.14461991061586713,
        "score_mae": 0.3112363499499235,
        "within_1": 0.985
      }
    },
    "by_qtype": {
      "choice": {
        "n": 600,
        "accuracy": 0.7383333333333333,
        "soft_accuracy": 0.4576608436665159,
        "brier": 0.07886955158977817,
        "kl": 0.15330907850375597,
        "tv": 0.18865741057538912,
        "nll": 0.7129818449599276,
        "ece": 0.1392636251597695,
        "score_mae": null,
        "within_1": null
      },
      "noul": {
        "n": 600,
        "accuracy": 0.8366666666666667,
        "soft_accuracy": 0.6693370476449475,
        "brier": 0.03942886821452841,
        "kl": 0.05145474170989372,
        "tv": 0.09811214657990391,
        "nll": 0.3812379372014114,
        "ece": 0.0843618453812399,
        "score_mae": null,
        "within_1": null
      },
      "score": {
        "n": 800,
        "accuracy": 0.69625,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.7712597630421172,
        "ece": 0.15033501893271706,
        "score_mae": 0.2746505811582656,
        "within_1": 0.98375
      }
    },
    "by_schema_id": {
      "agent_trace_observability/action": {
        "n": 100,
        "accuracy": 0.67,
        "soft_accuracy": 0.37864703694239377,
        "brier": 0.049903900779301506,
        "kl": 0.09724056949034372,
        "tv": 0.15841980300574204,
        "nll": 0.841429970588603,
        "ece": 0.16207918840409982,
        "score_mae": null,
        "within_1": null
      },
      "agent_trace_observability/needs_review": {
        "n": 100,
        "accuracy": 0.82,
        "soft_accuracy": 0.6271667824819649,
        "brier": 0.039836710289119205,
        "kl": 0.05140740089529728,
        "tv": 0.10942551114017018,
        "nll": 0.41184288208503156,
        "ece": 0.1275521930036992,
        "score_mae": null,
        "within_1": null
      },
      "agent_trace_observability/outcome": {
        "n": 100,
        "accuracy": 0.68,
        "soft_accuracy": 0.41153654156387304,
        "brier": 0.07910561093292694,
        "kl": 0.13186883225576213,
        "tv": 0.19289847894420123,
        "nll": 0.7898371925373076,
        "ece": 0.13892615370006228,
        "score_mae": null,
        "within_1": null
      },
      "agent_trace_observability/risk": {
        "n": 100,
        "accuracy": 0.8,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.6549731615156594,
        "ece": 0.24708902020558798,
        "score_mae": 0.1947352688400199,
        "within_1": 1.0
      },
      "agent_trace_observability/urgency": {
        "n": 100,
        "accuracy": 0.69,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.8051139888871975,
        "ece": 0.1755901741662443,
        "score_mae": 0.22544640807122945,
        "within_1": 1.0
      },
      "customer_service/action": {
        "n": 100,
        "accuracy": 0.63,
        "soft_accuracy": 0.3888590717902884,
        "brier": 0.11445938581035998,
        "kl": 0.23031132103353194,
        "tv": 0.24566652927218924,
        "nll": 0.9319729586843554,
        "ece": 0.08587540619343464,
        "score_mae": null,
        "within_1": null
      },
      "customer_service/category": {
        "n": 100,
        "accuracy": 0.9,
        "soft_accuracy": 0.6704960256392977,
        "brier": 0.0947763938424711,
        "kl": 0.20248588253857602,
        "tv": 0.18808011295017835,
        "nll": 0.38370140372630757,
        "ece": 0.14613724059962907,
        "score_mae": null,
        "within_1": null
      },
      "customer_service/churn_risk": {
        "n": 100,
        "accuracy": 0.67,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.83026776181706,
        "ece": 0.16804481624874312,
        "score_mae": 0.3211845585193193,
        "within_1": 0.95
      },
      "customer_service/needs_human": {
        "n": 100,
        "accuracy": 0.75,
        "soft_accuracy": 0.5813920034994436,
        "brier": 0.06782262651677691,
        "kl": 0.07700717450473488,
        "tv": 0.14131253024901802,
        "nll": 0.5149443876497118,
        "ece": 0.0901062252733414,
        "score_mae": null,
        "within_1": null
      },
      "customer_service/urgency": {
        "n": 100,
        "accuracy": 0.69,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.8003564632564825,
        "ece": 0.1752485705339351,
        "score_mae": 0.2733446315002939,
        "within_1": 1.0
      },
      "invoice_processing/discrepancy_severity": {
        "n": 100,
        "accuracy": 0.78,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.5601792853430406,
        "ece": 0.14199146682039665,
        "score_mae": 0.338526231566432,
        "within_1": 0.95
      },
      "invoice_processing/disposition": {
        "n": 100,
        "accuracy": 0.75,
        "soft_accuracy": 0.48504161823816555,
        "brier": 0.08354235197127319,
        "kl": 0.1496560986469237,
        "tv": 0.19025085349788567,
        "nll": 0.6579466495886376,
        "ece": 0.1524891184511141,
        "score_mae": null,
        "within_1": null
      },
      "invoice_processing/duplicate": {
        "n": 100,
        "accuracy": 0.93,
        "soft_accuracy": 0.8669995078963517,
        "brier": 0.018845183257522446,
        "kl": 0.027196929038003314,
        "tv": 0.047014968294512044,
        "nll": 0.16610251723167443,
        "ece": 0.0660903240070419,
        "score_mae": null,
        "within_1": null
      },
      "invoice_processing/matches_order": {
        "n": 100,
        "accuracy": 0.94,
        "soft_accuracy": 0.8069165924210221,
        "brier": 0.0463899995012223,
        "kl": 0.07734701278773116,
        "tv": 0.09288788557255173,
        "nll": 0.20546349749709156,
        "ece": 0.07770465780110603,
        "score_mae": null,
        "within_1": null
      },
      "invoice_processing/urgency": {
        "n": 100,
        "accuracy": 0.64,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.7807791381418437,
        "ece": 0.12403058303108795,
        "score_mae": 0.22149485086898335,
        "within_1": 1.0
      },
      "security_incidents/credential_compromise": {
        "n": 100,
        "accuracy": 0.77,
        "soft_accuracy": 0.5431074459153316,
        "brier": 0.03405358380994987,
        "kl": 0.03940214977263347,
        "tv": 0.10374847615088617,
        "nll": 0.5468801338498097,
        "ece": 0.145414782843676,
        "score_mae": null,
        "within_1": null
      },
      "security_incidents/disposition": {
        "n": 100,
        "accuracy": 0.8,
        "soft_accuracy": 0.41138476782507716,
        "brier": 0.051429666202336376,
        "kl": 0.10829176705739842,
        "tv": 0.1566286857821381,
        "nll": 0.6730028946343545,
        "ece": 0.21373141384977584,
        "score_mae": null,
        "within_1": null
      },
      "security_incidents/severity": {
        "n": 100,
        "accuracy": 0.71,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.8968579196544322,
        "ece": 0.21102490765200058,
        "score_mae": 0.3344534301686142,
        "within_1": 0.98
      },
      "security_incidents/true_positive": {
        "n": 100,
        "accuracy": 0.81,
        "soft_accuracy": 0.5904399536555718,
        "brier": 0.02962510591257973,
        "kl": 0.036367783260962155,
        "tv": 0.09428350807228533,
        "nll": 0.4421942048951493,
        "ece": 0.13079891069657626,
        "score_mae": null,
        "within_1": null
      },
      "security_incidents/urgency": {
        "n": 100,
        "accuracy": 0.59,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.841550385721222,
        "ece": 0.08675957856276363,
        "score_mae": 0.28801926973123276,
        "within_1": 0.99
      }
    }
  },
  "calibrated": {
    "overall": {
      "n": 2000,
      "accuracy": 0.751,
      "soft_accuracy": 0.6178046717519351,
      "brier": 0.11362459879677686,
      "kl": 0.30095497190224685,
      "tv": 0.20176915755108013,
      "nll": 0.5680045157373079,
      "ece": 0.04043277735449942,
      "score_mae": 0.3198573015138221,
      "within_1": 0.97
    },
    "by_workflow": {
      "agent_trace_observability": {
        "n": 500,
        "accuracy": 0.732,
        "soft_accuracy": 0.5271686746301996,
        "brier": 0.12098650868702557,
        "kl": 0.31954329306493995,
        "tv": 0.22993239481802855,
        "nll": 0.5973505916529968,
        "ece": 0.03588033188979682,
        "score_mae": 0.28590820356756197,
        "within_1": 1.0
      },
      "customer_service": {
        "n": 500,
        "accuracy": 0.728,
        "soft_accuracy": 0.6138200792710129,
        "brier": 0.145541356967536,
        "kl": 0.3411004238382146,
        "tv": 0.22775900057478538,
        "nll": 0.6500724208735882,
        "ece": 0.07012955999535193,
        "score_mae": 0.3169925538594844,
        "within_1": 0.965
      },
      "invoice_processing": {
        "n": 500,
        "accuracy": 0.808,
        "soft_accuracy": 0.7639967805225397,
        "brier": 0.08197299545550225,
        "kl": 0.2698870870596405,
        "tv": 0.14169172035189762,
        "nll": 0.4160005525805608,
        "ece": 0.04359159746369926,
        "score_mae": 0.27941288373058354,
        "within_1": 0.95
      },
      "security_incidents": {
        "n": 500,
        "accuracy": 0.736,
        "soft_accuracy": 0.5662331525839882,
        "brier": 0.10599753407704361,
        "kl": 0.2732890836461923,
        "tv": 0.20769351445960893,
        "nll": 0.608594497842086,
        "ece": 0.046897431140645096,
        "score_mae": 0.3971155648976584,
        "within_1": 0.965
      }
    },
    "by_qtype": {
      "choice": {
        "n": 600,
        "accuracy": 0.7383333333333333,
        "soft_accuracy": 0.5294220048812253,
        "brier": 0.15886700352925193,
        "kl": 0.4576430916821466,
        "tv": 0.2703961043025863,
        "nll": 0.6390465026947252,
        "ece": 0.06568009288736831,
        "score_mae": null,
        "within_1": null
      },
      "noul": {
        "n": 600,
        "accuracy": 0.8366666666666667,
        "soft_accuracy": 0.7061873386226449,
        "brier": 0.06838219406430175,
        "kl": 0.14426685212234702,
        "tv": 0.13314221079957392,
        "nll": 0.3443171239150436,
        "ece": 0.03072768695903312,
        "score_mae": null,
        "within_1": null
      },
      "score": {
        "n": 800,
        "accuracy": 0.69625,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.6824885693859432,
        "ece": 0.043157208326225695,
        "score_mae": 0.3198573015138221,
        "within_1": 0.97
      }
    },
    "by_schema_id": {
      "agent_trace_observability/action": {
        "n": 100,
        "accuracy": 0.67,
        "soft_accuracy": 0.43377777225260944,
        "brier": 0.12744161649559468,
        "kl": 0.3826436703933884,
        "tv": 0.2633365085097515,
        "nll": 0.7352119058432045,
        "ece": 0.08080262060002472,
        "score_mae": null,
        "within_1": null
      },
      "agent_trace_observability/needs_review": {
        "n": 100,
        "accuracy": 0.82,
        "soft_accuracy": 0.6719805168552689,
        "brier": 0.06233964264866126,
        "kl": 0.1482735079470231,
        "tv": 0.13895963722913257,
        "nll": 0.3464312756882847,
        "ece": 0.09750878535037706,
        "score_mae": null,
        "within_1": null
      },
      "agent_trace_observability/outcome": {
        "n": 100,
        "accuracy": 0.68,
        "soft_accuracy": 0.47574773478272037,
        "brier": 0.17317826691682073,
        "kl": 0.4277127008544081,
        "tv": 0.28750103871520155,
        "nll": 0.7251843727780874,
        "ece": 0.12738973196466497,
        "score_mae": null,
        "within_1": null
      },
      "agent_trace_observability/risk": {
        "n": 100,
        "accuracy": 0.8,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.49726936882265277,
        "ece": 0.08128467110179902,
        "score_mae": 0.256063240195547,
        "within_1": 1.0
      },
      "agent_trace_observability/urgency": {
        "n": 100,
        "accuracy": 0.69,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.6826560351327541,
        "ece": 0.04947163723661896,
        "score_mae": 0.31575316693957683,
        "within_1": 1.0
      },
      "customer_service/action": {
        "n": 100,
        "accuracy": 0.63,
        "soft_accuracy": 0.45212191867191825,
        "brier": 0.2110131519451015,
        "kl": 0.5188205492002642,
        "tv": 0.3187358639074713,
        "nll": 0.9683544302994465,
        "ece": 0.17665053743921832,
        "score_mae": null,
        "within_1": null
      },
      "customer_service/category": {
        "n": 100,
        "accuracy": 0.9,
        "soft_accuracy": 0.7640711181753796,
        "brier": 0.09520275546506135,
        "kl": 0.3121675177388705,
        "tv": 0.15788651204535845,
        "nll": 0.27464683825519653,
        "ece": 0.036861447188785965,
        "score_mae": null,
        "within_1": null
      },
      "customer_service/churn_risk": {
        "n": 100,
        "accuracy": 0.67,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.7903341057001046,
        "ece": 0.0936859953048758,
        "score_mae": 0.3403095011269979,
        "within_1": 0.94
      },
      "customer_service/needs_human": {
        "n": 100,
        "accuracy": 0.75,
        "soft_accuracy": 0.6252672009657406,
        "brier": 0.1304081634924451,
        "kl": 0.19231320457550907,
        "tv": 0.20665462577152632,
        "nll": 0.5138780280031838,
        "ece": 0.12367391544275927,
        "score_mae": null,
        "within_1": null
      },
      "customer_service/urgency": {
        "n": 100,
        "accuracy": 0.69,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.7031487021100088,
        "ece": 0.06809865228312525,
        "score_mae": 0.2936756065919708,
        "within_1": 0.99
      },
      "invoice_processing/discrepancy_severity": {
        "n": 100,
        "accuracy": 0.78,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.493989389662816,
        "ece": 0.0799121179160796,
        "score_mae": 0.3374784670461055,
        "within_1": 0.9
      },
      "invoice_processing/disposition": {
        "n": 100,
        "accuracy": 0.75,
        "soft_accuracy": 0.554000467328609,
        "brier": 0.16784104793395124,
        "kl": 0.5187716339663901,
        "tv": 0.2735823324908618,
        "nll": 0.6012390024595231,
        "ece": 0.1087785620484807,
        "score_mae": null,
        "within_1": null
      },
      "invoice_processing/duplicate": {
        "n": 100,
        "accuracy": 0.93,
        "soft_accuracy": 0.8906173029033013,
        "brier": 0.029733229585660614,
        "kl": 0.10844597304603973,
        "tv": 0.06165477710925789,
        "nll": 0.17549498045278247,
        "ece": 0.06917040917004692,
        "score_mae": null,
        "within_1": null
      },
      "invoice_processing/matches_order": {
        "n": 100,
        "accuracy": 0.94,
        "soft_accuracy": 0.8473725713357088,
        "brier": 0.048344708846894936,
        "kl": 0.1824436541664917,
        "tv": 0.08983805145557319,
        "nll": 0.17492864391876922,
        "ece": 0.055859921108077235,
        "score_mae": null,
        "within_1": null
      },
      "invoice_processing/urgency": {
        "n": 100,
        "accuracy": 0.64,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.634350746408913,
        "ece": 0.08940732025394861,
        "score_mae": 0.22134730041506162,
        "within_1": 1.0
      },
      "security_incidents/credential_compromise": {
        "n": 100,
        "accuracy": 0.77,
        "soft_accuracy": 0.5728015621293402,
        "brier": 0.05996805088332501,
        "kl": 0.08639044321714845,
        "tv": 0.13661466424427768,
        "nll": 0.48320685975458416,
        "ece": 0.06143237512009606,
        "score_mae": null,
        "within_1": null
      },
      "security_incidents/disposition": {
        "n": 100,
        "accuracy": 0.8,
        "soft_accuracy": 0.4968130180761153,
        "brier": 0.17852518241898221,
        "kl": 0.5857424779395584,
        "tv": 0.3213343701468733,
        "nll": 0.5296424665328927,
        "ece": 0.07959175670498408,
        "score_mae": null,
        "within_1": null
      },
      "security_incidents/severity": {
        "n": 100,
        "accuracy": 0.71,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.8210523363985104,
        "ece": 0.14468211188621938,
        "score_mae": 0.38530313622199197,
        "within_1": 0.97
      },
      "security_incidents/true_positive": {
        "n": 100,
        "accuracy": 0.81,
        "soft_accuracy": 0.6290848775465093,
        "brier": 0.07949936892882356,
        "kl": 0.1477343297818701,
        "tv": 0.16513150898767584,
        "nll": 0.3719629556726569,
        "ece": 0.0533904871823985,
        "score_mae": null,
        "within_1": null
      },
      "security_incidents/urgency": {
        "n": 100,
        "accuracy": 0.59,
        "soft_accuracy": null,
        "brier": null,
        "kl": null,
        "tv": null,
        "nll": 0.8371078708517858,
        "ece": 0.12910757991997132,
        "score_mae": 0.408927993573325,
        "within_1": 0.96
      }
    }
  },
  "laya_reference": {
    "accuracy": 0.766,
    "soft_accuracy": 0.471,
    "brier": 0.062,
    "ece": 0.213,
    "score_mae": 0.242
  },
  "gate": {
    "accuracy_gt_laya": false,
    "brier_lt_laya": true,
    "ece_raw_lt_laya_raw": true,
    "score_mae_lt_laya": false,
    "probability_quality_improved": true,
    "broad_claim_allowed": false,
    "rule": "accuracy > Laya AND (Brier < Laya OR raw ECE < Laya); score MAE is informational"
  },
  "workflow_gate_accuracy_gt_laya": {
    "agent_trace_observability": true,
    "customer_service": false,
    "invoice_processing": true,
    "security_incidents": false
  },
  "override_justification": ""
}