Text Classification
Safetensors
English
primus-decision
aame
local-inference
model-evaluation
typed-decisions
system-one
calibrated-decisions
probabilistic-classification
state-space-model
s4d
gru
cpu-inference
non-transformer
agent-observability
invoice-processing
security-incidents
customer-service
Eval Results (legacy)
| { | |
| "run_index": 1, | |
| "evaluated_utc": "2026-09-22T06:32:28Z", | |
| "test_file": "all/test-00000-of-00001.parquet", | |
| "test_sha256": "4f294f218ea1da27f3efef936359389c62ea4d3973a41457732990f1d31b647c", | |
| "n_cases": 400, | |
| "n_decisions": 2000, | |
| "latency_ms_per_case_cpu": 134.04, | |
| "raw": { | |
| "overall": { | |
| "n": 2000, | |
| "accuracy": 0.751, | |
| "soft_accuracy": 0.5634989456557318, | |
| "brier": 0.05914920990215329, | |
| "kl": 0.10238191010682485, | |
| "tv": 0.1433847785776465, | |
| "nll": 0.6367698398652486, | |
| "ece": 0.12722164873538963, | |
| "score_mae": 0.2746505811582656, | |
| "within_1": 0.98375 | |
| }, | |
| "by_workflow": { | |
| "agent_trace_observability": { | |
| "n": 500, | |
| "accuracy": 0.732, | |
| "soft_accuracy": 0.4724501203294106, | |
| "brier": 0.05628207400044921, | |
| "kl": 0.09350560088046769, | |
| "tv": 0.15358126436337113, | |
| "nll": 0.7006394391227598, | |
| "ece": 0.1601141162832159, | |
| "score_mae": 0.2100908384556247, | |
| "within_1": 1.0 | |
| }, | |
| "customer_service": { | |
| "n": 500, | |
| "accuracy": 0.728, | |
| "soft_accuracy": 0.5469157003096765, | |
| "brier": 0.092352802056536, | |
| "kl": 0.16993479269228093, | |
| "tv": 0.1916863908237952, | |
| "nll": 0.6922485950267834, | |
| "ece": 0.11188648161232355, | |
| "score_mae": 0.2972645950098066, | |
| "within_1": 0.975 | |
| }, | |
| "invoice_processing": { | |
| "n": 500, | |
| "accuracy": 0.808, | |
| "soft_accuracy": 0.7196525728518464, | |
| "brier": 0.04959251157667264, | |
| "kl": 0.0847333468242194, | |
| "tv": 0.11005123578831648, | |
| "nll": 0.4740942175604576, | |
| "ece": 0.09652193226496708, | |
| "score_mae": 0.2800105412177077, | |
| "within_1": 0.975 | |
| }, | |
| "security_incidents": { | |
| "n": 500, | |
| "accuracy": 0.736, | |
| "soft_accuracy": 0.5149773891319935, | |
| "brier": 0.038369451974955324, | |
| "kl": 0.06135390003033135, | |
| "tv": 0.11822022333510321, | |
| "nll": 0.6800971077509935, | |
| "ece": 0.14461991061586713, | |
| "score_mae": 0.3112363499499235, | |
| "within_1": 0.985 | |
| } | |
| }, | |
| "by_qtype": { | |
| "choice": { | |
| "n": 600, | |
| "accuracy": 0.7383333333333333, | |
| "soft_accuracy": 0.4576608436665159, | |
| "brier": 0.07886955158977817, | |
| "kl": 0.15330907850375597, | |
| "tv": 0.18865741057538912, | |
| "nll": 0.7129818449599276, | |
| "ece": 0.1392636251597695, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "noul": { | |
| "n": 600, | |
| "accuracy": 0.8366666666666667, | |
| "soft_accuracy": 0.6693370476449475, | |
| "brier": 0.03942886821452841, | |
| "kl": 0.05145474170989372, | |
| "tv": 0.09811214657990391, | |
| "nll": 0.3812379372014114, | |
| "ece": 0.0843618453812399, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "score": { | |
| "n": 800, | |
| "accuracy": 0.69625, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.7712597630421172, | |
| "ece": 0.15033501893271706, | |
| "score_mae": 0.2746505811582656, | |
| "within_1": 0.98375 | |
| } | |
| }, | |
| "by_schema_id": { | |
| "agent_trace_observability/action": { | |
| "n": 100, | |
| "accuracy": 0.67, | |
| "soft_accuracy": 0.37864703694239377, | |
| "brier": 0.049903900779301506, | |
| "kl": 0.09724056949034372, | |
| "tv": 0.15841980300574204, | |
| "nll": 0.841429970588603, | |
| "ece": 0.16207918840409982, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "agent_trace_observability/needs_review": { | |
| "n": 100, | |
| "accuracy": 0.82, | |
| "soft_accuracy": 0.6271667824819649, | |
| "brier": 0.039836710289119205, | |
| "kl": 0.05140740089529728, | |
| "tv": 0.10942551114017018, | |
| "nll": 0.41184288208503156, | |
| "ece": 0.1275521930036992, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "agent_trace_observability/outcome": { | |
| "n": 100, | |
| "accuracy": 0.68, | |
| "soft_accuracy": 0.41153654156387304, | |
| "brier": 0.07910561093292694, | |
| "kl": 0.13186883225576213, | |
| "tv": 0.19289847894420123, | |
| "nll": 0.7898371925373076, | |
| "ece": 0.13892615370006228, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "agent_trace_observability/risk": { | |
| "n": 100, | |
| "accuracy": 0.8, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.6549731615156594, | |
| "ece": 0.24708902020558798, | |
| "score_mae": 0.1947352688400199, | |
| "within_1": 1.0 | |
| }, | |
| "agent_trace_observability/urgency": { | |
| "n": 100, | |
| "accuracy": 0.69, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.8051139888871975, | |
| "ece": 0.1755901741662443, | |
| "score_mae": 0.22544640807122945, | |
| "within_1": 1.0 | |
| }, | |
| "customer_service/action": { | |
| "n": 100, | |
| "accuracy": 0.63, | |
| "soft_accuracy": 0.3888590717902884, | |
| "brier": 0.11445938581035998, | |
| "kl": 0.23031132103353194, | |
| "tv": 0.24566652927218924, | |
| "nll": 0.9319729586843554, | |
| "ece": 0.08587540619343464, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "customer_service/category": { | |
| "n": 100, | |
| "accuracy": 0.9, | |
| "soft_accuracy": 0.6704960256392977, | |
| "brier": 0.0947763938424711, | |
| "kl": 0.20248588253857602, | |
| "tv": 0.18808011295017835, | |
| "nll": 0.38370140372630757, | |
| "ece": 0.14613724059962907, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "customer_service/churn_risk": { | |
| "n": 100, | |
| "accuracy": 0.67, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.83026776181706, | |
| "ece": 0.16804481624874312, | |
| "score_mae": 0.3211845585193193, | |
| "within_1": 0.95 | |
| }, | |
| "customer_service/needs_human": { | |
| "n": 100, | |
| "accuracy": 0.75, | |
| "soft_accuracy": 0.5813920034994436, | |
| "brier": 0.06782262651677691, | |
| "kl": 0.07700717450473488, | |
| "tv": 0.14131253024901802, | |
| "nll": 0.5149443876497118, | |
| "ece": 0.0901062252733414, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "customer_service/urgency": { | |
| "n": 100, | |
| "accuracy": 0.69, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.8003564632564825, | |
| "ece": 0.1752485705339351, | |
| "score_mae": 0.2733446315002939, | |
| "within_1": 1.0 | |
| }, | |
| "invoice_processing/discrepancy_severity": { | |
| "n": 100, | |
| "accuracy": 0.78, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.5601792853430406, | |
| "ece": 0.14199146682039665, | |
| "score_mae": 0.338526231566432, | |
| "within_1": 0.95 | |
| }, | |
| "invoice_processing/disposition": { | |
| "n": 100, | |
| "accuracy": 0.75, | |
| "soft_accuracy": 0.48504161823816555, | |
| "brier": 0.08354235197127319, | |
| "kl": 0.1496560986469237, | |
| "tv": 0.19025085349788567, | |
| "nll": 0.6579466495886376, | |
| "ece": 0.1524891184511141, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "invoice_processing/duplicate": { | |
| "n": 100, | |
| "accuracy": 0.93, | |
| "soft_accuracy": 0.8669995078963517, | |
| "brier": 0.018845183257522446, | |
| "kl": 0.027196929038003314, | |
| "tv": 0.047014968294512044, | |
| "nll": 0.16610251723167443, | |
| "ece": 0.0660903240070419, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "invoice_processing/matches_order": { | |
| "n": 100, | |
| "accuracy": 0.94, | |
| "soft_accuracy": 0.8069165924210221, | |
| "brier": 0.0463899995012223, | |
| "kl": 0.07734701278773116, | |
| "tv": 0.09288788557255173, | |
| "nll": 0.20546349749709156, | |
| "ece": 0.07770465780110603, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "invoice_processing/urgency": { | |
| "n": 100, | |
| "accuracy": 0.64, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.7807791381418437, | |
| "ece": 0.12403058303108795, | |
| "score_mae": 0.22149485086898335, | |
| "within_1": 1.0 | |
| }, | |
| "security_incidents/credential_compromise": { | |
| "n": 100, | |
| "accuracy": 0.77, | |
| "soft_accuracy": 0.5431074459153316, | |
| "brier": 0.03405358380994987, | |
| "kl": 0.03940214977263347, | |
| "tv": 0.10374847615088617, | |
| "nll": 0.5468801338498097, | |
| "ece": 0.145414782843676, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "security_incidents/disposition": { | |
| "n": 100, | |
| "accuracy": 0.8, | |
| "soft_accuracy": 0.41138476782507716, | |
| "brier": 0.051429666202336376, | |
| "kl": 0.10829176705739842, | |
| "tv": 0.1566286857821381, | |
| "nll": 0.6730028946343545, | |
| "ece": 0.21373141384977584, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "security_incidents/severity": { | |
| "n": 100, | |
| "accuracy": 0.71, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.8968579196544322, | |
| "ece": 0.21102490765200058, | |
| "score_mae": 0.3344534301686142, | |
| "within_1": 0.98 | |
| }, | |
| "security_incidents/true_positive": { | |
| "n": 100, | |
| "accuracy": 0.81, | |
| "soft_accuracy": 0.5904399536555718, | |
| "brier": 0.02962510591257973, | |
| "kl": 0.036367783260962155, | |
| "tv": 0.09428350807228533, | |
| "nll": 0.4421942048951493, | |
| "ece": 0.13079891069657626, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "security_incidents/urgency": { | |
| "n": 100, | |
| "accuracy": 0.59, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.841550385721222, | |
| "ece": 0.08675957856276363, | |
| "score_mae": 0.28801926973123276, | |
| "within_1": 0.99 | |
| } | |
| } | |
| }, | |
| "calibrated": { | |
| "overall": { | |
| "n": 2000, | |
| "accuracy": 0.751, | |
| "soft_accuracy": 0.6178046717519351, | |
| "brier": 0.11362459879677686, | |
| "kl": 0.30095497190224685, | |
| "tv": 0.20176915755108013, | |
| "nll": 0.5680045157373079, | |
| "ece": 0.04043277735449942, | |
| "score_mae": 0.3198573015138221, | |
| "within_1": 0.97 | |
| }, | |
| "by_workflow": { | |
| "agent_trace_observability": { | |
| "n": 500, | |
| "accuracy": 0.732, | |
| "soft_accuracy": 0.5271686746301996, | |
| "brier": 0.12098650868702557, | |
| "kl": 0.31954329306493995, | |
| "tv": 0.22993239481802855, | |
| "nll": 0.5973505916529968, | |
| "ece": 0.03588033188979682, | |
| "score_mae": 0.28590820356756197, | |
| "within_1": 1.0 | |
| }, | |
| "customer_service": { | |
| "n": 500, | |
| "accuracy": 0.728, | |
| "soft_accuracy": 0.6138200792710129, | |
| "brier": 0.145541356967536, | |
| "kl": 0.3411004238382146, | |
| "tv": 0.22775900057478538, | |
| "nll": 0.6500724208735882, | |
| "ece": 0.07012955999535193, | |
| "score_mae": 0.3169925538594844, | |
| "within_1": 0.965 | |
| }, | |
| "invoice_processing": { | |
| "n": 500, | |
| "accuracy": 0.808, | |
| "soft_accuracy": 0.7639967805225397, | |
| "brier": 0.08197299545550225, | |
| "kl": 0.2698870870596405, | |
| "tv": 0.14169172035189762, | |
| "nll": 0.4160005525805608, | |
| "ece": 0.04359159746369926, | |
| "score_mae": 0.27941288373058354, | |
| "within_1": 0.95 | |
| }, | |
| "security_incidents": { | |
| "n": 500, | |
| "accuracy": 0.736, | |
| "soft_accuracy": 0.5662331525839882, | |
| "brier": 0.10599753407704361, | |
| "kl": 0.2732890836461923, | |
| "tv": 0.20769351445960893, | |
| "nll": 0.608594497842086, | |
| "ece": 0.046897431140645096, | |
| "score_mae": 0.3971155648976584, | |
| "within_1": 0.965 | |
| } | |
| }, | |
| "by_qtype": { | |
| "choice": { | |
| "n": 600, | |
| "accuracy": 0.7383333333333333, | |
| "soft_accuracy": 0.5294220048812253, | |
| "brier": 0.15886700352925193, | |
| "kl": 0.4576430916821466, | |
| "tv": 0.2703961043025863, | |
| "nll": 0.6390465026947252, | |
| "ece": 0.06568009288736831, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "noul": { | |
| "n": 600, | |
| "accuracy": 0.8366666666666667, | |
| "soft_accuracy": 0.7061873386226449, | |
| "brier": 0.06838219406430175, | |
| "kl": 0.14426685212234702, | |
| "tv": 0.13314221079957392, | |
| "nll": 0.3443171239150436, | |
| "ece": 0.03072768695903312, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "score": { | |
| "n": 800, | |
| "accuracy": 0.69625, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.6824885693859432, | |
| "ece": 0.043157208326225695, | |
| "score_mae": 0.3198573015138221, | |
| "within_1": 0.97 | |
| } | |
| }, | |
| "by_schema_id": { | |
| "agent_trace_observability/action": { | |
| "n": 100, | |
| "accuracy": 0.67, | |
| "soft_accuracy": 0.43377777225260944, | |
| "brier": 0.12744161649559468, | |
| "kl": 0.3826436703933884, | |
| "tv": 0.2633365085097515, | |
| "nll": 0.7352119058432045, | |
| "ece": 0.08080262060002472, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "agent_trace_observability/needs_review": { | |
| "n": 100, | |
| "accuracy": 0.82, | |
| "soft_accuracy": 0.6719805168552689, | |
| "brier": 0.06233964264866126, | |
| "kl": 0.1482735079470231, | |
| "tv": 0.13895963722913257, | |
| "nll": 0.3464312756882847, | |
| "ece": 0.09750878535037706, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "agent_trace_observability/outcome": { | |
| "n": 100, | |
| "accuracy": 0.68, | |
| "soft_accuracy": 0.47574773478272037, | |
| "brier": 0.17317826691682073, | |
| "kl": 0.4277127008544081, | |
| "tv": 0.28750103871520155, | |
| "nll": 0.7251843727780874, | |
| "ece": 0.12738973196466497, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "agent_trace_observability/risk": { | |
| "n": 100, | |
| "accuracy": 0.8, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.49726936882265277, | |
| "ece": 0.08128467110179902, | |
| "score_mae": 0.256063240195547, | |
| "within_1": 1.0 | |
| }, | |
| "agent_trace_observability/urgency": { | |
| "n": 100, | |
| "accuracy": 0.69, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.6826560351327541, | |
| "ece": 0.04947163723661896, | |
| "score_mae": 0.31575316693957683, | |
| "within_1": 1.0 | |
| }, | |
| "customer_service/action": { | |
| "n": 100, | |
| "accuracy": 0.63, | |
| "soft_accuracy": 0.45212191867191825, | |
| "brier": 0.2110131519451015, | |
| "kl": 0.5188205492002642, | |
| "tv": 0.3187358639074713, | |
| "nll": 0.9683544302994465, | |
| "ece": 0.17665053743921832, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "customer_service/category": { | |
| "n": 100, | |
| "accuracy": 0.9, | |
| "soft_accuracy": 0.7640711181753796, | |
| "brier": 0.09520275546506135, | |
| "kl": 0.3121675177388705, | |
| "tv": 0.15788651204535845, | |
| "nll": 0.27464683825519653, | |
| "ece": 0.036861447188785965, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "customer_service/churn_risk": { | |
| "n": 100, | |
| "accuracy": 0.67, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.7903341057001046, | |
| "ece": 0.0936859953048758, | |
| "score_mae": 0.3403095011269979, | |
| "within_1": 0.94 | |
| }, | |
| "customer_service/needs_human": { | |
| "n": 100, | |
| "accuracy": 0.75, | |
| "soft_accuracy": 0.6252672009657406, | |
| "brier": 0.1304081634924451, | |
| "kl": 0.19231320457550907, | |
| "tv": 0.20665462577152632, | |
| "nll": 0.5138780280031838, | |
| "ece": 0.12367391544275927, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "customer_service/urgency": { | |
| "n": 100, | |
| "accuracy": 0.69, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.7031487021100088, | |
| "ece": 0.06809865228312525, | |
| "score_mae": 0.2936756065919708, | |
| "within_1": 0.99 | |
| }, | |
| "invoice_processing/discrepancy_severity": { | |
| "n": 100, | |
| "accuracy": 0.78, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.493989389662816, | |
| "ece": 0.0799121179160796, | |
| "score_mae": 0.3374784670461055, | |
| "within_1": 0.9 | |
| }, | |
| "invoice_processing/disposition": { | |
| "n": 100, | |
| "accuracy": 0.75, | |
| "soft_accuracy": 0.554000467328609, | |
| "brier": 0.16784104793395124, | |
| "kl": 0.5187716339663901, | |
| "tv": 0.2735823324908618, | |
| "nll": 0.6012390024595231, | |
| "ece": 0.1087785620484807, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "invoice_processing/duplicate": { | |
| "n": 100, | |
| "accuracy": 0.93, | |
| "soft_accuracy": 0.8906173029033013, | |
| "brier": 0.029733229585660614, | |
| "kl": 0.10844597304603973, | |
| "tv": 0.06165477710925789, | |
| "nll": 0.17549498045278247, | |
| "ece": 0.06917040917004692, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "invoice_processing/matches_order": { | |
| "n": 100, | |
| "accuracy": 0.94, | |
| "soft_accuracy": 0.8473725713357088, | |
| "brier": 0.048344708846894936, | |
| "kl": 0.1824436541664917, | |
| "tv": 0.08983805145557319, | |
| "nll": 0.17492864391876922, | |
| "ece": 0.055859921108077235, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "invoice_processing/urgency": { | |
| "n": 100, | |
| "accuracy": 0.64, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.634350746408913, | |
| "ece": 0.08940732025394861, | |
| "score_mae": 0.22134730041506162, | |
| "within_1": 1.0 | |
| }, | |
| "security_incidents/credential_compromise": { | |
| "n": 100, | |
| "accuracy": 0.77, | |
| "soft_accuracy": 0.5728015621293402, | |
| "brier": 0.05996805088332501, | |
| "kl": 0.08639044321714845, | |
| "tv": 0.13661466424427768, | |
| "nll": 0.48320685975458416, | |
| "ece": 0.06143237512009606, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "security_incidents/disposition": { | |
| "n": 100, | |
| "accuracy": 0.8, | |
| "soft_accuracy": 0.4968130180761153, | |
| "brier": 0.17852518241898221, | |
| "kl": 0.5857424779395584, | |
| "tv": 0.3213343701468733, | |
| "nll": 0.5296424665328927, | |
| "ece": 0.07959175670498408, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "security_incidents/severity": { | |
| "n": 100, | |
| "accuracy": 0.71, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.8210523363985104, | |
| "ece": 0.14468211188621938, | |
| "score_mae": 0.38530313622199197, | |
| "within_1": 0.97 | |
| }, | |
| "security_incidents/true_positive": { | |
| "n": 100, | |
| "accuracy": 0.81, | |
| "soft_accuracy": 0.6290848775465093, | |
| "brier": 0.07949936892882356, | |
| "kl": 0.1477343297818701, | |
| "tv": 0.16513150898767584, | |
| "nll": 0.3719629556726569, | |
| "ece": 0.0533904871823985, | |
| "score_mae": null, | |
| "within_1": null | |
| }, | |
| "security_incidents/urgency": { | |
| "n": 100, | |
| "accuracy": 0.59, | |
| "soft_accuracy": null, | |
| "brier": null, | |
| "kl": null, | |
| "tv": null, | |
| "nll": 0.8371078708517858, | |
| "ece": 0.12910757991997132, | |
| "score_mae": 0.408927993573325, | |
| "within_1": 0.96 | |
| } | |
| } | |
| }, | |
| "laya_reference": { | |
| "accuracy": 0.766, | |
| "soft_accuracy": 0.471, | |
| "brier": 0.062, | |
| "ece": 0.213, | |
| "score_mae": 0.242 | |
| }, | |
| "gate": { | |
| "accuracy_gt_laya": false, | |
| "brier_lt_laya": true, | |
| "ece_raw_lt_laya_raw": true, | |
| "score_mae_lt_laya": false, | |
| "probability_quality_improved": true, | |
| "broad_claim_allowed": false, | |
| "rule": "accuracy > Laya AND (Brier < Laya OR raw ECE < Laya); score MAE is informational" | |
| }, | |
| "workflow_gate_accuracy_gt_laya": { | |
| "agent_trace_observability": true, | |
| "customer_service": false, | |
| "invoice_processing": true, | |
| "security_incidents": false | |
| }, | |
| "override_justification": "" | |
| } | |