{
  "source_revision": "fb0fefdfad76f358edf9861f0c49e4ba2bdcab83",
  "records": {
    "evaluations/v2-rp105/low_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 176,
      "final_answer_pass_pct": 88.0,
      "completed_final_answer_pass_pct": 88.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-rp105/medium_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 176,
      "final_answer_pass_pct": 88.0,
      "completed_final_answer_pass_pct": 88.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-rp105/xhigh_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 174,
      "final_answer_pass_pct": 87.0,
      "completed_final_answer_pass_pct": 87.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v11-rp105/low_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 161,
      "final_answer_pass_pct": 80.5,
      "completed_final_answer_pass_pct": 80.5,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v11-rp105/medium_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 162,
      "final_answer_pass_pct": 81.0,
      "completed_final_answer_pass_pct": 81.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v11-rp105/xhigh_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 162,
      "final_answer_pass_pct": 81.0,
      "completed_final_answer_pass_pct": 81.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/qwen-rp105/low_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 178,
      "final_answer_pass_pct": 89.0,
      "completed_final_answer_pass_pct": 89.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/qwen-rp105/medium_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 178,
      "final_answer_pass_pct": 89.0,
      "completed_final_answer_pass_pct": 89.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/qwen-rp105/xhigh_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 176,
      "final_answer_pass_pct": 87.5,
      "completed_final_answer_pass_pct": 86.5,
      "passed_without_visible_final_answer": [
        "irrelevance_139"
      ],
      "passed_without_eos": [
        "irrelevance_89",
        "irrelevance_157",
        "irrelevance_139"
      ],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-mtp2/low_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 175,
      "final_answer_pass_pct": 87.5,
      "completed_final_answer_pass_pct": 87.5,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-mtp2/medium_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 180,
      "final_answer_pass_pct": 90.0,
      "completed_final_answer_pass_pct": 90.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-mtp2/xhigh_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 176,
      "final_answer_pass_pct": 88.0,
      "completed_final_answer_pass_pct": 88.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-candidate/low_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 173,
      "final_answer_pass_pct": 86.5,
      "completed_final_answer_pass_pct": 86.5,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-candidate/medium_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 179,
      "final_answer_pass_pct": 89.5,
      "completed_final_answer_pass_pct": 89.5,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v2-candidate/xhigh_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 177,
      "final_answer_pass_pct": 88.5,
      "completed_final_answer_pass_pct": 88.5,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v11-baseline/low_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 167,
      "final_answer_pass_pct": 83.5,
      "completed_final_answer_pass_pct": 83.5,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v11-baseline/medium_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 170,
      "final_answer_pass_pct": 85.0,
      "completed_final_answer_pass_pct": 85.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/v11-baseline/xhigh_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 166,
      "final_answer_pass_pct": 83.0,
      "completed_final_answer_pass_pct": 83.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/qwen-baseline/low_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 180,
      "final_answer_pass_pct": 90.0,
      "completed_final_answer_pass_pct": 90.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/qwen-baseline/medium_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 178,
      "final_answer_pass_pct": 89.0,
      "completed_final_answer_pass_pct": 89.0,
      "passed_without_visible_final_answer": [],
      "passed_without_eos": [],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    },
    "evaluations/qwen-baseline/xhigh_bfcl_generations.json": {
      "n": 200,
      "raw_passed": 176,
      "final_answer_pass_pct": 86.5,
      "completed_final_answer_pass_pct": 86.0,
      "passed_without_visible_final_answer": [
        "irrelevance_229",
        "irrelevance_132",
        "irrelevance_139"
      ],
      "passed_without_eos": [
        "irrelevance_229",
        "irrelevance_132",
        "irrelevance_157",
        "irrelevance_139"
      ],
      "completion_definition": "Require a visible final decision in addition to the stored official AST verdict; an unfinished thought that has not called a tool is not a completed irrelevance decision."
    }
  }
}