{
 "model": "Sieve-9B",
 "decision_index_leaderboard": {
  "edition": "0.4.1",
  "board_data": "2026-10-28",
  "full_score": 41.73,
  "public_score": 41.88,
  "url": "https://huggingface.co/spaces/multimodalart/jev-decision-index",
  "full_score_weights": {
   "private_same_skills": 0.6,
   "private_new_domains": 0.4
  }
 },
 "protocol": {
  "temperature": 1.6311,
  "temperature_fit": "negative log-likelihood on the decision-v7 calibration split (1,148 questions); that split is used only to fit T and choose the checkpoint and is not reported",
  "rows": "decision-v7 and transfer-v4: original rows only (the constructed none-present / none-absent / permuted variants excluded). transfer-v9: original rows with every unknowable* family removed.",
  "metrics": "accuracy; error_rate = 1 - accuracy; NLL; Brier (squared error summed over options); ECE (15 equal-width confidence bins); wrong_at_p_ge_0.9 = share of all questions answered wrongly with top probability >= 0.9. 'calibrated' is at T = 1.6311 (as served), 'raw' at T = 1.",
  "intervals": "accuracy_ci95: bootstrap over groups of related records (4,000 resamples, seed 0)",
  "adapter": "suite, held-out, public-test and BFCL rows were scored in bf16 with the adapter unmerged; the Decision Index run and the latency use the merged model, as served",
  "partitions": "development partitions only; no decision-v7 or transfer test partition has been scored"
 },
 "decision_index_0.2.1": {
  "status": "our own run of the 0.2.1 suite",
  "kit": "github.com/apolinario/decision-index at 87d4650b42b377c0291a89c1f1a879f9b31082bf, the kit's own scorer",
  "results_dataset": "https://huggingface.co/datasets/sthanika-ai/Sieve-9B-decision-index-results",
  "decision_index": 41.71,
  "raw_index": 56.47,
  "breadth": 40.06,
  "requests_scored": 150317,
  "answered": 150317,
  "areas_skill_x100": {
   "Knowledge & Reasoning": 27.4,
   "Language Understanding": 46.1,
   "Retrieval & Classification": 46.1,
   "Tools & Automation": 61.3,
   "Arts & Human Taste": 22.8
  },
  "benchmarks": [
   {
    "dataset": "ACOS",
    "metric": "per-review F1",
    "score": 0.2409,
    "skill": 0.2166,
    "requests": 1565,
    "in_index": true
   },
   {
    "dataset": "ANLI",
    "metric": "macro-F1",
    "score": 0.5615,
    "skill": 0.3432,
    "requests": 3200,
    "in_index": true
   },
   {
    "dataset": "API-Bank",
    "metric": "accuracy",
    "score": 0.7264,
    "skill": 0.7211,
    "requests": 508,
    "in_index": true
   },
   {
    "dataset": "Amazon ESCI",
    "metric": "macro-F1",
    "score": 0.4931,
    "skill": 0.3643,
    "requests": 5000,
    "in_index": true
   },
   {
    "dataset": "BANKING77",
    "metric": "macro-F1",
    "score": 0.8362,
    "skill": 0.8341,
    "requests": 3080,
    "in_index": true
   },
   {
    "dataset": "BBH fixed-option tasks",
    "metric": "accuracy",
    "score": 0.6715,
    "skill": 0.5238,
    "requests": 5507,
    "in_index": true
   },
   {
    "dataset": "BFCL",
    "metric": "case exact accuracy",
    "score": 0.954,
    "skill": 0.9378,
    "requests": 1694,
    "in_index": true
   },
   {
    "dataset": "BPoMP",
    "metric": "accuracy",
    "score": 0.6654,
    "skill": 0.352,
    "requests": 5000,
    "in_index": true
   },
   {
    "dataset": "BRIGHT",
    "metric": "nDCG@10",
    "score": 0.4202,
    "skill": 0.344,
    "requests": 220,
    "in_index": true
   },
   {
    "dataset": "CLINC150+OOS",
    "metric": "macro-F1",
    "score": 0.7603,
    "skill": 0.7589,
    "requests": 5500,
    "in_index": true
   },
   {
    "dataset": "CLadder",
    "metric": "accuracy",
    "score": 0.633,
    "skill": 0.266,
    "requests": 5000,
    "in_index": true
   },
   {
    "dataset": "CRUXEval",
    "metric": "accuracy",
    "score": 0.5368,
    "skill": 0.2652,
    "requests": 570,
    "in_index": true
   },
   {
    "dataset": "ChessBench",
    "metric": "accuracy",
    "score": 0.1144,
    "skill": 0.0354,
    "requests": 5000,
    "in_index": true
   },
   {
    "dataset": "ContractNLI",
    "metric": "macro-F1",
    "score": 0.6145,
    "skill": 0.4425,
    "requests": 123,
    "in_index": true
   },
   {
    "dataset": "FinEntity",
    "metric": "macro-F1",
    "score": 0.8882,
    "skill": 0.8356,
    "requests": 979,
    "in_index": true
   },
   {
    "dataset": "ForecastBench",
    "metric": "Brier (lower is better)",
    "score": 0.181,
    "skill": 0.276,
    "requests": 10139,
    "in_index": true
   },
   {
    "dataset": "GPQA Diamond",
    "metric": "accuracy",
    "score": 0.4235,
    "skill": 0.2313,
    "requests": 196,
    "in_index": true
   },
   {
    "dataset": "GSM8K",
    "metric": "accuracy",
    "score": 0.4602,
    "skill": 0.3552,
    "requests": 2638,
    "in_index": true
   },
   {
    "dataset": "HLE",
    "metric": "accuracy",
    "score": 0.1058,
    "skill": 0.0,
    "requests": 501,
    "in_index": true
   },
   {
    "dataset": "Habermas Machine",
    "metric": "accuracy",
    "score": 0.4039,
    "skill": 0.1349,
    "requests": 1676,
    "in_index": true
   },
   {
    "dataset": "HellaSwag",
    "metric": "accuracy",
    "score": 0.8297,
    "skill": 0.7729,
    "requests": 10042,
    "in_index": true
   },
   {
    "dataset": "HoVer claim verification",
    "metric": "accuracy",
    "score": 0.6348,
    "skill": 0.2696,
    "requests": 4000,
    "in_index": true
   },
   {
    "dataset": "Home appliance simulator",
    "metric": "case exact accuracy",
    "score": 0.3068,
    "skill": 0.3068,
    "requests": 88,
    "in_index": true
   },
   {
    "dataset": "Humicroedit",
    "metric": "accuracy",
    "score": 0.5506,
    "skill": 0.1012,
    "requests": 2628,
    "in_index": true
   },
   {
    "dataset": "MMLU-Pro",
    "metric": "accuracy",
    "score": 0.5081,
    "skill": 0.4467,
    "requests": 12032,
    "in_index": true
   },
   {
    "dataset": "MuSR",
    "metric": "accuracy",
    "score": 0.5665,
    "skill": 0.3108,
    "requests": 752,
    "in_index": true
   },
   {
    "dataset": "NLI4CT",
    "metric": "macro-F1",
    "score": 0.779,
    "skill": 0.57,
    "requests": 5500,
    "in_index": true
   },
   {
    "dataset": "New Yorker caption matching",
    "metric": "accuracy",
    "score": 0.5814,
    "skill": 0.4768,
    "requests": 528,
    "in_index": true
   },
   {
    "dataset": "POP909-CL",
    "metric": "accuracy",
    "score": 0.129,
    "skill": 0.1171,
    "requests": 2000,
    "in_index": true
   },
   {
    "dataset": "PhishNChips phishing decisions",
    "metric": "accuracy",
    "score": 0.541,
    "skill": 0.082,
    "requests": 2000,
    "in_index": true
   },
   {
    "dataset": "RAGTruth response-level hallucination",
    "metric": "F1 on hallucinated class",
    "score": 0.6285,
    "skill": 0.2297,
    "requests": 2700,
    "in_index": true
   },
   {
    "dataset": "SATA-Bench",
    "metric": "case exact accuracy",
    "score": 0.2958,
    "skill": 0.2865,
    "requests": 1650,
    "in_index": true
   },
   {
    "dataset": "ToolRet",
    "metric": "nDCG@10",
    "score": 0.6514,
    "skill": 0.5975,
    "requests": 685,
    "in_index": true
   },
   {
    "dataset": "VAST",
    "metric": "macro-F1",
    "score": 0.5222,
    "skill": 0.2834,
    "requests": 3006,
    "in_index": true
   },
   {
    "dataset": "When2Call MCQ",
    "metric": "accuracy",
    "score": 0.5613,
    "skill": 0.4151,
    "requests": 3652,
    "in_index": true
   },
   {
    "dataset": "WinoGrande",
    "metric": "accuracy",
    "score": 0.7506,
    "skill": 0.5012,
    "requests": 1267,
    "in_index": true
   },
   {
    "dataset": "cfcolor",
    "metric": "accuracy",
    "score": 0.5804,
    "skill": 0.1299,
    "requests": 5000,
    "in_index": true
   },
   {
    "dataset": "iSarcasmEval",
    "metric": "Sarcasm F1 · track A, English",
    "score": 0.5064,
    "skill": 0.365,
    "requests": 4600,
    "in_index": true
   },
   {
    "dataset": "ARC-Challenge",
    "metric": "accuracy",
    "score": 0.9377,
    "skill": 0.9169,
    "requests": 1172,
    "in_index": false
   },
   {
    "dataset": "ARC-Easy",
    "metric": "accuracy",
    "score": 0.9777,
    "skill": 0.9703,
    "requests": 2376,
    "in_index": false
   },
   {
    "dataset": "MMLU",
    "metric": "accuracy",
    "score": 0.7494,
    "skill": 0.6659,
    "requests": 14033,
    "in_index": false
   },
   {
    "dataset": "RouterBench",
    "metric": "selected quality (quality objective)",
    "score": 0.793,
    "skill": 0.5147,
    "requests": 10000,
    "in_index": false
   },
   {
    "dataset": "SGD/SGD-X",
    "metric": "macro-F1",
    "score": 0.6267,
    "skill": 0.3789,
    "requests": 2500,
    "in_index": false
   },
   {
    "dataset": "SimpleBench",
    "metric": "accuracy",
    "score": 0.2,
    "skill": 0.04,
    "requests": 10,
    "in_index": false
   }
  ]
 },
 "suites": {
  "decision-v7 development (in-domain)": {
   "n": 1264,
   "accuracy": 0.8718,
   "accuracy_ci95": [
    0.8507,
    0.892
   ],
   "error_rate": 0.1282,
   "calibrated": {
    "nll": 0.3297,
    "brier": 0.1763,
    "ece": 0.0284,
    "wrong_at_p_ge_0.9": 0.0174
   },
   "raw": {
    "nll": 0.4006,
    "brier": 0.1869,
    "ece": 0.0623,
    "wrong_at_p_ge_0.9": 0.0396
   },
   "per_family_accuracy": {
    "agnews": {
     "n": 80,
     "accuracy": 0.85
    },
    "agnews_yn": {
     "n": 160,
     "accuracy": 0.9187
    },
    "amazon": {
     "n": 80,
     "accuracy": 0.6
    },
    "banking77": {
     "n": 80,
     "accuracy": 0.9
    },
    "boolq": {
     "n": 80,
     "accuracy": 0.8625
    },
    "composition_atom": {
     "n": 16,
     "accuracy": 1.0
    },
    "composition_conditional": {
     "n": 16,
     "accuracy": 0.8125
    },
    "composition_conjunction": {
     "n": 16,
     "accuracy": 1.0
    },
    "composition_disjunction": {
     "n": 16,
     "accuracy": 1.0
    },
    "composition_exception": {
     "n": 16,
     "accuracy": 1.0
    },
    "composition_negation": {
     "n": 16,
     "accuracy": 1.0
    },
    "composition_nested_and": {
     "n": 16,
     "accuracy": 1.0
    },
    "composition_nested_or": {
     "n": 16,
     "accuracy": 1.0
    },
    "contrastive_age_eligibility": {
     "n": 24,
     "accuracy": 1.0
    },
    "contrastive_quantity_limit": {
     "n": 24,
     "accuracy": 0.9583
    },
    "contrastive_return_window": {
     "n": 24,
     "accuracy": 0.875
    },
    "contrastive_spend_threshold": {
     "n": 24,
     "accuracy": 1.0
    },
    "dbpedia14": {
     "n": 80,
     "accuracy": 0.9625
    },
    "imdb": {
     "n": 80,
     "accuracy": 0.95
    },
    "mnli": {
     "n": 80,
     "accuracy": 0.9125
    },
    "sst5": {
     "n": 80,
     "accuracy": 0.6375
    },
    "trec": {
     "n": 80,
     "accuracy": 0.95
    },
    "yelp": {
     "n": 80,
     "accuracy": 0.6875
    },
    "yelp_yn": {
     "n": 80,
     "accuracy": 0.9125
    }
   }
  },
  "transfer-v4 development (out-of-domain)": {
   "n": 656,
   "accuracy": 0.8293,
   "accuracy_ci95": [
    0.7969,
    0.8599
   ],
   "error_rate": 0.1707,
   "calibrated": {
    "nll": 0.4468,
    "brier": 0.2416,
    "ece": 0.0449,
    "wrong_at_p_ge_0.9": 0.0366
   },
   "raw": {
    "nll": 0.5531,
    "brier": 0.2572,
    "ece": 0.0885,
    "wrong_at_p_ge_0.9": 0.0595
   },
   "per_family_accuracy": {
    "composition_held_and_or": {
     "n": 32,
     "accuracy": 0.8125
    },
    "composition_held_conditional": {
     "n": 32,
     "accuracy": 0.9062
    },
    "composition_held_or_not": {
     "n": 32,
     "accuracy": 0.875
    },
    "contrastive_authorization": {
     "n": 40,
     "accuracy": 1.0
    },
    "contrastive_deadline": {
     "n": 40,
     "accuracy": 0.9
    },
    "emotion": {
     "n": 80,
     "accuracy": 0.575
    },
    "mmlu": {
     "n": 80,
     "accuracy": 0.7625
    },
    "paws": {
     "n": 80,
     "accuracy": 0.7875
    },
    "qnli": {
     "n": 80,
     "accuracy": 0.925
    },
    "sciq": {
     "n": 80,
     "accuracy": 0.975
    },
    "tweet_offensive": {
     "n": 80,
     "accuracy": 0.7875
    }
   }
  },
  "transfer-v9 development, strict (out-of-domain; unknowable families removed)": {
   "n": 936,
   "accuracy": 0.7532,
   "accuracy_ci95": [
    0.7236,
    0.7814
   ],
   "error_rate": 0.2468,
   "calibrated": {
    "nll": 0.6888,
    "brier": 0.3372,
    "ece": 0.0496,
    "wrong_at_p_ge_0.9": 0.0385
   },
   "raw": {
    "nll": 0.8158,
    "brier": 0.3558,
    "ece": 0.1053,
    "wrong_at_p_ge_0.9": 0.062
   },
   "per_family_accuracy": {
    "buried_emotion": {
     "n": 20,
     "accuracy": 0.75
    },
    "buried_paws": {
     "n": 20,
     "accuracy": 0.7
    },
    "buried_qnli": {
     "n": 20,
     "accuracy": 0.85
    },
    "buried_tweet_offensive": {
     "n": 20,
     "accuracy": 0.8
    },
    "composition_held_and_or": {
     "n": 32,
     "accuracy": 0.8125
    },
    "composition_held_conditional": {
     "n": 32,
     "accuracy": 0.9062
    },
    "composition_held_or_not": {
     "n": 32,
     "accuracy": 0.9062
    },
    "contrastive_authorization": {
     "n": 40,
     "accuracy": 1.0
    },
    "contrastive_deadline": {
     "n": 40,
     "accuracy": 0.9
    },
    "emotion": {
     "n": 80,
     "accuracy": 0.575
    },
    "mmlu": {
     "n": 80,
     "accuracy": 0.7625
    },
    "mmlu_pro": {
     "n": 200,
     "accuracy": 0.49
    },
    "paws": {
     "n": 80,
     "accuracy": 0.7875
    },
    "qnli": {
     "n": 80,
     "accuracy": 0.925
    },
    "sciq": {
     "n": 80,
     "accuracy": 0.975
    },
    "tweet_offensive": {
     "n": 80,
     "accuracy": 0.7875
    }
   }
  }
 },
 "held_out_decision_benchmark": {
  "note": "6,233 questions over 10 synthetic decision domains. 6,224 were scored; 9 were dropped because their options exceed the evaluation's 12,288-token question limit. 77 scored questions have 256 or 512 options, more than the served API accepts (255). States over 384 tokens were truncated to 384, the training limit.",
  "n": 6224,
  "accuracy": 0.8567,
  "accuracy_ci95": [
   0.8476,
   0.8655
  ],
  "error_rate": 0.1433,
  "calibrated": {
   "nll": 0.4507,
   "brier": 0.207,
   "ece": 0.0546,
   "wrong_at_p_ge_0.9": 0.0039
  },
  "raw": {
   "nll": 0.4273,
   "brier": 0.2006,
   "ece": 0.0196,
   "wrong_at_p_ge_0.9": 0.0182
  },
  "by_state_length": {
   "state <= 384 tokens": {
    "n": 5186,
    "accuracy": 0.8887
   },
   "state > 384 tokens (truncated)": {
    "n": 1038,
    "accuracy": 0.6965
   }
  },
  "by_option_count": {
   "2 options": {
    "n": 2452,
    "accuracy": 0.9755
   },
   "3-10 options": {
    "n": 2969,
    "accuracy": 0.8228
   },
   "11-50 options": {
    "n": 522,
    "accuracy": 0.728
   },
   "51-255 options": {
    "n": 204,
    "accuracy": 0.4314
   },
   "256-512 options": {
    "n": 77,
    "accuracy": 0.3766
   }
  }
 },
 "public_benchmarks": {
  "mmlu": {
   "n": 14042,
   "accuracy": 0.758,
   "accuracy_ci95": [
    0.751,
    0.7651
   ],
   "error_rate": 0.242,
   "calibrated": {
    "nll": 0.6424,
    "brier": 0.3331,
    "ece": 0.0077,
    "wrong_at_p_ge_0.9": 0.017
   },
   "raw": {
    "nll": 0.7303,
    "brier": 0.3494,
    "ece": 0.0913,
    "wrong_at_p_ge_0.9": 0.0526
   }
  },
  "sciq": {
   "n": 1000,
   "accuracy": 0.984,
   "accuracy_ci95": [
    0.976,
    0.991
   ],
   "error_rate": 0.016,
   "calibrated": {
    "nll": 0.0629,
    "brier": 0.0258,
    "ece": 0.0241,
    "wrong_at_p_ge_0.9": 0.001
   },
   "raw": {
    "nll": 0.0483,
    "brier": 0.0239,
    "ece": 0.0057,
    "wrong_at_p_ge_0.9": 0.004
   }
  },
  "banking77": {
   "n": 3079,
   "accuracy": 0.8568,
   "accuracy_ci95": [
    0.8441,
    0.8694
   ],
   "error_rate": 0.1432,
   "calibrated": {
    "nll": 0.5634,
    "brier": 0.2236,
    "ece": 0.0431,
    "wrong_at_p_ge_0.9": 0.0156
   },
   "raw": {
    "nll": 0.6024,
    "brier": 0.2246,
    "ece": 0.0638,
    "wrong_at_p_ge_0.9": 0.0477
   },
   "note": "in-distribution: the training data contains 1,000 Banking77 training questions"
  },
  "massive_intent": {
   "n": 2974,
   "accuracy": 0.7646,
   "accuracy_ci95": [
    0.7495,
    0.7801
   ],
   "error_rate": 0.2354,
   "calibrated": {
    "nll": 0.8861,
    "brier": 0.3448,
    "ece": 0.0933,
    "wrong_at_p_ge_0.9": 0.005
   },
   "raw": {
    "nll": 0.8602,
    "brier": 0.343,
    "ece": 0.0679,
    "wrong_at_p_ge_0.9": 0.0434
   }
  },
  "massive_scenario": {
   "n": 2974,
   "accuracy": 0.7112,
   "accuracy_ci95": [
    0.695,
    0.7276
   ],
   "error_rate": 0.2888,
   "calibrated": {
    "nll": 0.8888,
    "brier": 0.401,
    "ece": 0.0585,
    "wrong_at_p_ge_0.9": 0.0017
   },
   "raw": {
    "nll": 0.9417,
    "brier": 0.4156,
    "ece": 0.0962,
    "wrong_at_p_ge_0.9": 0.0393
   }
  },
  "typed_decisions": {
   "n": 2000,
   "accuracy": 0.7035,
   "accuracy_ci95": [
    0.681,
    0.7255
   ],
   "error_rate": 0.2965,
   "calibrated": {
    "nll": 0.7038,
    "brier": 0.4037,
    "ece": 0.0396,
    "wrong_at_p_ge_0.9": 0.005
   },
   "raw": {
    "nll": 0.7269,
    "brier": 0.4106,
    "ece": 0.0692,
    "wrong_at_p_ge_0.9": 0.0355
   }
  }
 },
 "bfcl_decision_form": {
  "note": "BFCL v3 single-turn Python items in decision form: pick the function to call from the offered ones, or 'none of these'. Argument values are not generated or scored, so these are not BFCL leaderboard numbers. Items that offer no function at all are dropped (3,045 of 3,049 scored). live_relevance counts any offered function as correct, as BFCL does.",
  "per_category": {
   "simple": {
    "n": 399,
    "accuracy": 0.9925
   },
   "multiple": {
    "n": 200,
    "accuracy": 0.99
   },
   "live_simple": {
    "n": 258,
    "accuracy": 0.9302
   },
   "live_multiple": {
    "n": 1052,
    "accuracy": 0.9544
   },
   "irrelevance": {
    "n": 240,
    "accuracy": 0.8917
   },
   "live_irrelevance": {
    "n": 878,
    "accuracy": 0.6731
   },
   "live_relevance": {
    "n": 18,
    "accuracy": 0.9444
   }
  },
  "function_selection": {
   "n": 1909,
   "accuracy": 0.9628
  },
  "irrelevance_detection": {
   "n": 1118,
   "accuracy": 0.72
  }
 },
 "latency": {
  "hardware": "1x NVIDIA A100 80GB PCIe, idle (utilisation < 5% for a minute before measuring), bf16, adapter merged",
  "model_only_min_ms (warm state prefix, 1 choice question)": {
   "2": {
    "eager": 68.8,
    "cuda_graphs": 18.4
   },
   "5": {
    "eager": 68.7,
    "cuda_graphs": 19.2
   },
   "10": {
    "eager": 69.2,
    "cuda_graphs": 22.0
   },
   "25": {
    "eager": 68.9,
    "cuda_graphs": 30.2
   },
   "50": {
    "eager": 70.9,
    "cuda_graphs": 53.4
   },
   "100": {
    "eager": 106.8,
    "cuda_graphs": 103.7
   },
   "255": {
    "eager": 232.2,
    "cuda_graphs": 231.9
   }
  },
  "graph_parity_vs_eager": {
   "n": 16,
   "max_abs_dp": 0.00529855489730835,
   "argmax_flips": 0
  },
  "server_reported_p50_ms (repeated support-ticket state, options 'dept_i: team handling category i'; excludes tokenisation and HTTP)": {
   "2": 18.6,
   "5": 19.3,
   "10": 22.8,
   "25": 42.8,
   "50": 74.5,
   "100": 128.4,
   "255": 330.5
  },
  "measured": {
   "model_only": "2026-09-26 02:18:58",
   "server_reported": "2026-09-26 08:13:47"
  }
 }
}