[ADD] Initial results and frontend for EHRSQL, MedCalc, MedEC

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

app.py

@@ -1,145 +1,71 @@
-import subprocess
-
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 import time
+import functools
+import gc
+
+import os
 
 from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT_1,
-    LLM_BENCHMARKS_TEXT_2,
-    CROSS_EVALUATION_METRICS,
-    NOTE_GENERATION_METRICS,
-    HEALTHBENCH_METRICS,
-    # EVALUATION_EXAMPLE_IMG,
-    # LLM_BENCHMARKS_TEXT_2,
-    # ENTITY_DISTRIBUTION_IMG,
-    # LLM_BENCHMARKS_TEXT_3,
-    TITLE,
-    LOGO,
-    FIVE_PILLAR_DIAGRAM
+    CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT_1, LLM_BENCHMARKS_TEXT_2, CROSS_EVALUATION_METRICS,
+    NOTE_GENERATION_METRICS, HEALTHBENCH_METRICS, TITLE, LOGO, FIVE_PILLAR_DIAGRAM
 )
 from src.display.css_html_js import custom_css
-# changes to be made here
 from src.display.utils import (
-    DATASET_BENCHMARK_COLS,
-    OPEN_ENDED_BENCHMARK_COLS,
-    MED_SAFETY_BENCHMARK_COLS,
-    MEDICAL_SUMMARIZATION_BENCHMARK_COLS,
-    ACI_BENCHMARK_COLS,
-    SOAP_BENCHMARK_COLS,
-    HEALTHBENCH_BENCHMARK_COLS,
-    HEALTHBENCH_HARD_BENCHMARK_COLS,
-    DATASET_COLS,
-    OPEN_ENDED_COLS,
-    MED_SAFETY_COLS,
-    MEDICAL_SUMMARIZATION_COLS,
-    ACI_COLS,
-    SOAP_COLS,
-    HEALTHBENCH_COLS,
-    HEALTHBENCH_HARD_COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    NUMERIC_INTERVALS,
-    TYPES,
-    AutoEvalColumn,
-    ModelType,
-    ModelArch,
-    PromptTemplateName,
-    Precision,
-    WeightType,
-    fields,
-    render_generation_templates,
-    OpenEndedArabic_COLS,
-    OpenEndedArabic_BENCHMARK_COLS,
-    OpenEndedFrench_COLS,
-    OpenEndedFrench_BENCHMARK_COLS,
-    OpenEndedPortuguese_COLS,
-    OpenEndedPortuguese_BENCHMARK_COLS,
-    OpenEndedRomanian_COLS,
-    OpenEndedRomanian_BENCHMARK_COLS,
-    OpenEndedGreek_COLS,
-    OpenEndedGreek_BENCHMARK_COLS,
-    OpenEndedSpanish_COLS,
-    OpenEndedSpanish_BENCHMARK_COLS,
-    ClosedEndedMultilingual_COLS,
-    ClosedEndedMultilingual_BENCHMARK_COLS,
+    DATASET_BENCHMARK_COLS, OPEN_ENDED_BENCHMARK_COLS, MED_SAFETY_BENCHMARK_COLS,
+    MEDICAL_SUMMARIZATION_BENCHMARK_COLS, ACI_BENCHMARK_COLS, SOAP_BENCHMARK_COLS,
+    HEALTHBENCH_BENCHMARK_COLS, HEALTHBENCH_HARD_BENCHMARK_COLS, DATASET_COLS,
+    OPEN_ENDED_COLS, MED_SAFETY_COLS, MEDICAL_SUMMARIZATION_COLS, ACI_COLS, SOAP_COLS,
+    HEALTHBENCH_COLS, HEALTHBENCH_HARD_COLS, EVAL_COLS, EVAL_TYPES, NUMERIC_INTERVALS,
+    TYPES, AutoEvalColumn, ModelType, Precision, WeightType, fields, render_generation_templates,
+    OpenEndedArabic_COLS, OpenEndedArabic_BENCHMARK_COLS, OpenEndedFrench_COLS,
+    OpenEndedFrench_BENCHMARK_COLS, OpenEndedPortuguese_COLS, OpenEndedPortuguese_BENCHMARK_COLS,
+    OpenEndedRomanian_COLS, OpenEndedRomanian_BENCHMARK_COLS, OpenEndedGreek_COLS,
+    OpenEndedGreek_BENCHMARK_COLS, OpenEndedSpanish_COLS, OpenEndedSpanish_BENCHMARK_COLS,
+    ClosedEndedMultilingual_COLS, ClosedEndedMultilingual_BENCHMARK_COLS,
+    EHRSQL_ZERO_SHOT_COLS, EHRSQL_ZERO_SHOT_BENCHMARK_COLS, 
+    EHRSQL_FEW_SHOT_COLS, EHRSQL_FEW_SHOT_BENCHMARK_COLS,
+    MEDCALC_DIRECT_ANSWER_COLS, MEDCALC_DIRECT_ANSWER_BENCHMARK_COLS,
+    MEDCALC_ONE_SHOT_COT_COLS, MEDCALC_ONE_SHOT_COT_BENCHMARK_COLS,
+    MEDCALC_ZERO_SHOT_COT_COLS, MEDCALC_ZERO_SHOT_COT_BENCHMARK_COLS,
+    MEDEC_ZERO_SHOT_COLS, MEDEC_ZERO_SHOT_BENCHMARK_COLS,
+    MEDEC_ONE_SHOT_COLS, MEDEC_ONE_SHOT_BENCHMARK_COLS,
 )
-
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN, PRIVATE_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval, PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG
+from src.submission.submit import add_new_eval
+
+# =====================================================================================
+# 1. SETUP AND DATA LOADING 
+# =====================================================================================
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-print(f"QUEUE_REPO: {QUEUE_REPO}")
-print(f"RESULTS_REPO: {RESULTS_REPO}")
-print(f"EVAL_REQUESTS_PATH: {EVAL_REQUESTS_PATH}")
-print(f"EVAL_RESULTS_PATH: {EVAL_RESULTS_PATH}")
-print(f"TOKEN: {TOKEN}")
-
-try:
-    print(f"EVAL_REQUESTS_PATH: {EVAL_REQUESTS_PATH}")
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-    print(f"EVAL_REQUESTS_PATH downloaded")
-except Exception:
-    print("An error occurred while downloading EVAL_REQUESTS_PATH. Please check the connection or the repository settings.")
-    restart_space()
+print("Downloading evaluation data...")
 try:
-    print(f"EVAL_RESULTS_PATH: {EVAL_RESULTS_PATH}")
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-    print(f"EVAL_RESULTS_PATH downloaded")
-except Exception:
-    print("An error occurred while downloading EVAL_RESULTS_PATH. Please check the connection or the repository settings.")
+    snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", token=TOKEN)
+    snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", token=TOKEN)
+    print("Downloads complete.")
+except Exception as e:
+    print(f"An error occurred during download: {e}")
     restart_space()
 
-# Span based results
-# changes to be made here
-
+print("Loading all dataframes into a central dictionary...")
 start_time = time.time()
 
 _, harness_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "accuracy", "datasets")
-harness_datasets_leaderboard_df = harness_datasets_original_df.copy()
-print("Closed ended English results loaded")
-
 _, open_ended_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OPEN_ENDED_COLS, OPEN_ENDED_BENCHMARK_COLS, "score", "open_ended")
-open_ended_leaderboard_df = open_ended_original_df.copy()
-print("Open ended English results loaded")
-
 _, med_safety_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MED_SAFETY_COLS, MED_SAFETY_BENCHMARK_COLS, "score", "med_safety")
-med_safety_leaderboard_df = med_safety_original_df.copy()
-print("Med safety results loaded")
-
 _, medical_summarization_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDICAL_SUMMARIZATION_COLS, MEDICAL_SUMMARIZATION_BENCHMARK_COLS, "score", "medical_summarization")
-medical_summarization_leaderboard_df = medical_summarization_original_df.copy()
-print("Medical summarization results loaded")
-
 _, aci_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ACI_COLS, ACI_BENCHMARK_COLS, "score", "aci")
-aci_leaderboard_df = aci_original_df.copy()
-print("ACI results loaded")
-
 _, soap_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, SOAP_COLS, SOAP_BENCHMARK_COLS, "score", "soap")
-soap_leaderboard_df = soap_original_df.copy()
-print("SOAP results loaded")
-
 _, healthbench_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, HEALTHBENCH_COLS, HEALTHBENCH_BENCHMARK_COLS, "score", "healthbench")
-healthbench_leaderboard_df = healthbench_original_df.copy()
-
 _, healthbench_hard_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, HEALTHBENCH_HARD_COLS, HEALTHBENCH_HARD_BENCHMARK_COLS, "score", "healthbench_hard")
-healthbench_hard_leaderboard_df = healthbench_hard_original_df.copy()
-print("Healthbench results loaded")
-
 _, open_ended_arabic_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedArabic_COLS, OpenEndedArabic_BENCHMARK_COLS, "score", "open_ended_arabic")
 _, open_ended_french_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedFrench_COLS, OpenEndedFrench_BENCHMARK_COLS, "score", "open_ended_french")
 _, open_ended_portuguese_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedPortuguese_COLS, OpenEndedPortuguese_BENCHMARK_COLS, "score", "open_ended_portuguese")
@@ -147,129 +73,53 @@ _, open_ended_romanian_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_
 _, open_ended_greek_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedGreek_COLS, OpenEndedGreek_BENCHMARK_COLS, "score", "open_ended_greek")
 _, open_ended_spanish_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedSpanish_COLS, OpenEndedSpanish_BENCHMARK_COLS, "score", "open_ended_spanish")
 _, closed_ended_multilingual_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ClosedEndedMultilingual_COLS, ClosedEndedMultilingual_BENCHMARK_COLS, "score", "closed_ended_multilingual")
-
-
-open_ended_arabic_leaderboard_df = open_ended_arabic_df.copy()
-open_ended_french_leaderboard_df = open_ended_french_df.copy()
-open_ended_portuguese_leaderboard_df = open_ended_portuguese_df.copy()
-open_ended_romanian_leaderboard_df = open_ended_romanian_df.copy()
-open_ended_greek_leaderboard_df = open_ended_greek_df.copy()
-open_ended_spanish_leaderboard_df = open_ended_spanish_df.copy()
-print("Open ended multilingual results loaded")
-
-closed_ended_multilingual_leaderboard_df = closed_ended_multilingual_df.copy()
-print("Closed ended multilingual results loaded")
-
+_, ehrsql_zero_shot_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, EHRSQL_ZERO_SHOT_COLS, EHRSQL_ZERO_SHOT_BENCHMARK_COLS, "score", "ehrsql_zero_shot")
+_, ehrsql_few_shot_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, EHRSQL_FEW_SHOT_COLS, EHRSQL_FEW_SHOT_BENCHMARK_COLS, "score", "ehrsql_few_shot")
+_, medcalc_direct_answer_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDCALC_DIRECT_ANSWER_COLS, MEDCALC_DIRECT_ANSWER_BENCHMARK_COLS, "score", "medcalc_direct_answer")
+_, medcalc_one_shot_cot_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDCALC_ONE_SHOT_COT_COLS, MEDCALC_ONE_SHOT_COT_BENCHMARK_COLS, "score", "medcalc_one_shot_cot")
+_, medcalc_zero_shot_cot_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDCALC_ZERO_SHOT_COT_COLS, MEDCALC_ZERO_SHOT_COT_BENCHMARK_COLS, "score", "medcalc_zero_shot_cot")
+_, medec_zero_shot_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDEC_ZERO_SHOT_COLS, MEDEC_ZERO_SHOT_BENCHMARK_COLS, "score", "medec_zero_shot")
+_, medec_one_shot_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDEC_ONE_SHOT_COLS, MEDEC_ONE_SHOT_BENCHMARK_COLS, "score", "medec_one_shot")
+
+# Debug check for openai/gpt-oss-20b
+
+ALL_DATASETS = {
+    "datasets": harness_datasets_original_df,
+    "open_ended": open_ended_original_df,
+    "med_safety": med_safety_original_df,
+    "medical_summarization": medical_summarization_original_df,
+    "aci": aci_original_df,
+    "soap": soap_original_df,
+    "healthbench": healthbench_original_df,
+    "healthbench_hard": healthbench_hard_original_df,
+    "open_ended_arabic": open_ended_arabic_df,
+    "open_ended_french": open_ended_french_df,
+    "open_ended_portuguese": open_ended_portuguese_df,
+    "open_ended_romanian": open_ended_romanian_df,
+    "open_ended_greek": open_ended_greek_df,
+    "open_ended_spanish": open_ended_spanish_df,
+    "closed_ended_multilingual": closed_ended_multilingual_df,
+    "ehrsql_zero_shot": ehrsql_zero_shot_df,
+    "ehrsql_few_shot": ehrsql_few_shot_df,
+    "medcalc_direct_answer": medcalc_direct_answer_df,
+    "medcalc_one_shot_cot": medcalc_one_shot_cot_df,
+    "medcalc_zero_shot_cot": medcalc_zero_shot_cot_df,
+    "medec_zero_shot": medec_zero_shot_df,
+    "medec_one_shot": medec_one_shot_df,
+}
 end_time = time.time()
-total_time = end_time - start_time
-print(f"Total time taken to load all results: {total_time:.2f} seconds")
-
-# breakpoint()
-# # Token based results
-# _, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
-# token_based_datasets_leaderboard_df = token_based_datasets_original_df.copy()
+print(f"Dataframes loaded in {end_time - start_time:.2f} seconds.")
 
-# _, token_based_types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "clinical_types")
-# token_based_types_leaderboard_df = token_based_types_original_df.copy()
-
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-# breakpoint()
-def update_df(shown_columns, subset="datasets"):
-    # changes to be made here
-    if subset == "datasets":
-        leaderboard_table_df = harness_datasets_leaderboard_df.copy()
-        hidden_leader_board_df = harness_datasets_original_df
-    elif subset == "open_ended":
-        leaderboard_table_df = open_ended_leaderboard_df.copy()
-        hidden_leader_board_df = open_ended_original_df
-    elif subset == "med_safety":
-        leaderboard_table_df = med_safety_leaderboard_df.copy()
-        hidden_leader_board_df = med_safety_original_df
-    elif subset == "medical_summarization":
-        leaderboard_table_df = medical_summarization_leaderboard_df.copy()
-        hidden_leader_board_df = medical_summarization_original_df
-    elif subset == "aci":
-        leaderboard_table_df = aci_leaderboard_df.copy()
-        hidden_leader_board_df = aci_original_df
-    elif subset == "soap":
-        leaderboard_table_df = soap_leaderboard_df.copy()
-        hidden_leader_board_df = soap_original_df
-    elif subset == "healthbench":
-        leaderboard_table_df = healthbench_leaderboard_df.copy()
-        hidden_leader_board_df = healthbench_original_df
-    elif subset == "healthbench_hard":
-        leaderboard_table_df = healthbench_hard_leaderboard_df.copy()
-        hidden_leader_board_df = healthbench_hard_original_df
-    elif subset == "open_ended_arabic": 
-        leaderboard_table_df = open_ended_arabic_df.copy()
-        hidden_leader_board_df = open_ended_arabic_df
-    elif subset == "open_ended_french":
-        leaderboard_table_df = open_ended_french_df.copy()
-        hidden_leader_board_df = open_ended_french_df
-    elif subset == "open_ended_portuguese":
-        leaderboard_table_df = open_ended_portuguese_df.copy()
-        hidden_leader_board_df = open_ended_portuguese_df
-    elif subset == "open_ended_romanian":
-        leaderboard_table_df = open_ended_romanian_df.copy()
-        hidden_leader_board_df = open_ended_romanian_df
-    elif subset == "open_ended_greek":
-        leaderboard_table_df = open_ended_greek_df.copy()
-        hidden_leader_board_df = open_ended_greek_df
-    elif subset == "open_ended_spanish":
-        leaderboard_table_df = open_ended_spanish_df.copy()
-        hidden_leader_board_df = open_ended_spanish_df
-    elif subset == "closed_ended_multilingual":
-        leaderboard_table_df = closed_ended_multilingual_df.copy()
-        hidden_leader_board_df = closed_ended_multilingual_df
-    
-
-    value_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns
-    # breakpoint()
-    return leaderboard_table_df[value_cols], hidden_leader_board_df
-
-
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    query: str = "",
-    # type_query: list = None,
-    domain_specific_query: list = None,
-    size_query: list = None,
-    precision_query: str = None,
-    show_deleted: bool = False,
-):
-    # breakpoint()
-    type_query = None
-    filtered_df = filter_models(hidden_df, type_query, domain_specific_query, size_query, precision_query, show_deleted)
-    # breakpoint()
-    filtered_df = filter_queries(query, filtered_df)
-    # breakpoint()
-    df = select_columns(filtered_df, columns, list(hidden_df.columns))
-    # breakpoint()
-    return df
+# Evaluation Queue DataFrames
+(finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
+# =====================================================================================
+# 2. EFFICIENT FILTERING LOGIC
+# =====================================================================================
 
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
 
-
-def select_columns(df: pd.DataFrame, columns: list, cols:list) -> pd.DataFrame:
-    always_here_cols = [
-        # AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
-    ]
-    # We use COLS to maintain sorting
-    filtered_df = df[always_here_cols + [c for c in cols if c in df.columns and c in columns]]
-    return filtered_df
-
-
 def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
     final_df = []
     if query != "":
@@ -285,8 +135,6 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
             filtered_df = filtered_df.drop_duplicates(
                 subset=[
                     AutoEvalColumn.model.name,
-                    # AutoEvalColumn.precision.name,
-                    # AutoEvalColumn.revision.name,
                 ]
             )
 
@@ -296,11 +144,6 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
 def filter_models(
     df: pd.DataFrame, type_query: list, domain_specific_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
-    # Show all models
-    # if show_deleted:
-    #     filtered_df = df
-    # else:  # Show only still on the hub models
-    #     filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
     filtered_df = df
 
@@ -310,17 +153,12 @@ def filter_models(
 
     if domain_specific_query is not None:
         domain_specifics = []
-        if "🏥  Clinical models" in domain_specific_query:
+        if "🏥 Clinical models" in domain_specific_query:
             domain_specifics.append(True)
         if "Generic models" in domain_specific_query:
             domain_specifics.append(False)
         filtered_df = filtered_df.loc[df[AutoEvalColumn.is_domain_specific.name].isin(domain_specifics)]
-        
-    # if architecture_query is not None:
-    #     arch_types = [t for t in architecture_query]
-    #     filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(arch_types)]
-    #         # filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(architecture_query + ["None"])]
-    
+
     if precision_query is not None:
         if AutoEvalColumn.precision.name in df.columns:
             filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
@@ -333,1075 +171,276 @@ def filter_models(
 
     return filtered_df
 
-            
+def get_filtered_table(
+    shown_columns: list,
+    query: str,
+    domain_specific_query: list,
+    size_query: list,
+    *, # force subset_name to be a keyword-only argument
+    subset_name: str
+):
+    original_df = ALL_DATASETS[subset_name]
+    
+    type_query = None 
+    filtered_df = filter_models(original_df, type_query, domain_specific_query, size_query, None, False)
+    filtered_df = filter_queries(query, filtered_df)
+
+    always_here_cols = [AutoEvalColumn.model.name]
+    available_cols = [c for c in shown_columns if c in filtered_df.columns]
+    final_df = filtered_df[always_here_cols + available_cols]
+
+    del filtered_df
+    gc.collect()
+
+    
+    return final_df
+
+# =====================================================================================
+# 3. REUSABLE UI CREATION FUNCTION
+# =====================================================================================
+
+def create_leaderboard_ui(subset_name: str, column_choices: list, default_columns: list):
+    """Creates a full leaderboard UI block for a given subset."""
+    with gr.Row():
+        with gr.Column():
+            with gr.Row():
+                search_bar = gr.Textbox(
+                    placeholder=f"🔍 Search for models...",
+                    show_label=False,
+                    elem_id=f"search-bar-{subset_name}",
+                )
+            with gr.Row():
+                shown_columns = gr.CheckboxGroup(
+                    choices=column_choices,
+                    value=default_columns,
+                    label="Select columns to show",
+                    elem_id=f"column-select-{subset_name}",
+                    interactive=True,
+                )
+        with gr.Column(min_width=320):
+            filter_domain_specific = gr.CheckboxGroup(
+                label="Domain Specificity",
+                choices=["🏥 Clinical models", "Generic models"],
+                value=["🏥 Clinical models", "Generic models"],
+                interactive=True,
+                elem_id=f"filter-domain-{subset_name}",
+            )
+            filter_columns_size = gr.CheckboxGroup(
+                label="Model sizes (in billions of parameters)",
+                choices=list(NUMERIC_INTERVALS.keys()),
+                value=list(NUMERIC_INTERVALS.keys()),
+                interactive=True,
+                elem_id=f"filter-size-{subset_name}",
+            )
+    
+    update_fn = functools.partial(get_filtered_table, subset_name=subset_name)
+
+    initial_df = update_fn(
+        shown_columns=default_columns,
+        query="",
+        domain_specific_query=["🏥 Clinical models", "Generic models"],
+        size_query=list(NUMERIC_INTERVALS.keys())
+    )
+
+    leaderboard_table = gr.Dataframe(
+        value=initial_df,
+        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + default_columns,
+        datatype=TYPES,
+        elem_id=f"leaderboard-table-{subset_name}",
+        interactive=False,
+    )
+
+    inputs = [shown_columns, search_bar, filter_domain_specific, filter_columns_size]
+    
+    # Attach listeners to all input components
+    for component in inputs:
+        if isinstance(component, gr.Textbox):
+            component.submit(update_fn, inputs, leaderboard_table)
+        else:
+            component.change(update_fn, inputs, leaderboard_table)
+
+    return leaderboard_table
+
+# =====================================================================================
+# 4. GRADIO DEMO UI (Main application layout)
+# =====================================================================================
+
 demo = gr.Blocks(css=custom_css)
+
 with demo:
-    print("hello")
     gr.HTML(LOGO)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-    filter_columns_type = None
+    
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=1):
             with gr.Tabs(elem_classes="tab-buttons6") as language_tabs:
                 LANGUAGES = {
-                    "🇺🇸 English": "open_ended",
-                    "🇦🇪 Arabic": "open_ended_arabic",
-                    "🇫🇷 French": "open_ended_french",
-                    "🇪🇸 Spanish": "open_ended_spanish",
-                    "🇵🇹 Portuguese": "open_ended_portuguese",
-                    "🇷🇴 Romanian": "open_ended_romanian",
+                    "🇺🇸 English": "open_ended", "🇦🇪 Arabic": "open_ended_arabic",
+                    "🇫🇷 French": "open_ended_french", "🇪🇸 Spanish": "open_ended_spanish",
+                    "🇵🇹 Portuguese": "open_ended_portuguese", "🇷🇴 Romanian": "open_ended_romanian",
                     "🇬🇷 Greek": "open_ended_greek",
                 }
-
                 for idx, (label, subset) in enumerate(LANGUAGES.items()):
                     with gr.TabItem(label, elem_id=f"llm-benchmark-tab-open-{subset}", id=idx):
-                        # Custom judge information for each language
-                        if label == "🇺🇸 English":
-                            judge_text = "**Note:** Llama 3.1 70B Instruct has been used as judge for English."
-                        else:
-                            judge_text = "**Note:** Qwen 2.5 72B Instruct has been used as judge for this language."
-                        
+                        judge_text = "**Note:** Llama 3.1 70B Instruct has been used as judge for English." if label == "🇺🇸 English" else "**Note:** Qwen 2.5 72B Instruct has been used as judge for this language."
                         gr.Markdown(judge_text, elem_classes="markdown-text")
                         
-                        with gr.Row():
-                            with gr.Column():
-                                with gr.Row():
-                                    search_bar = gr.Textbox(
-                                        placeholder=f"🔍 Search for your model in {label}...",
-                                        show_label=False,
-                                        elem_id=f"search-bar-{subset}",
-                                    )
-                                with gr.Row():
-                                    shown_columns = gr.CheckboxGroup(
-                                        choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)],
-                                        value=[
-                                            c.name
-                                            for c in fields(AutoEvalColumn)
-                                            if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)
-                                        ],
-                                        label="Select columns to show",
-                                        elem_id=f"column-select-{subset}",
-                                        interactive=True,
-                                    )
-                            with gr.Column(min_width=320):
-                                # filter_columns_type = gr.CheckboxGroup(
-                                #     label="Model Types",
-                                #     choices=[t.to_str() for t in ModelType],
-                                #     value=[t.to_str() for t in ModelType],
-                                #     interactive=True,
-                                #     elem_id=f"filter-columns-type-{subset}",
-                                # )
-
-                                filter_domain_specific = gr.CheckboxGroup(
-                                    label="Domain Specificity",
-                                    choices=["🏥  Clinical models", "Generic models"],
-                                    value=["🏥  Clinical models", "Generic models"],
-                                    interactive=True,
-                                    elem_id=f"filter-columns-domain-{subset}",
-                                )
-                                filter_columns_size = gr.CheckboxGroup(
-                                    label="Model sizes (in billions of parameters)",
-                                    choices=list(NUMERIC_INTERVALS.keys()),
-                                    value=list(NUMERIC_INTERVALS.keys()),
-                                    interactive=True,
-                                    elem_id=f"filter-columns-size-{subset}",
-                                )
-
-                        datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset=subset)
-
-                        leaderboard_table = gr.Dataframe(
-                            value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                            headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                            datatype=TYPES,
-                            elem_id=f"leaderboard-table-{subset}",
-                            interactive=False,
-                            visible=True,
+                        create_leaderboard_ui(
+                            subset_name=subset,
+                            column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)],
+                            default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)]
                         )
-
-                        hidden_leaderboard_table_for_search = gr.Dataframe(
-                            value=datasets_original_df[OPEN_ENDED_COLS],
-                            headers=OPEN_ENDED_COLS,
-                            datatype=TYPES,
-                            visible=False,
-                        )
-
-                        search_bar.submit(
-                            update_table,
-                            [
-                                hidden_leaderboard_table_for_search,
-                                shown_columns,
-                                search_bar,
-                                # filter_columns_type,
-                                filter_domain_specific,
-                                filter_columns_size
-                            ],
-                            leaderboard_table,
-                        )
-
-                        for selector in [
-                            shown_columns,
-                            # filter_columns_type,
-                            filter_domain_specific,
-                            filter_columns_size,
-                        ]:
-                            selector.change(
-                                update_table,
-                                [
-                                    hidden_leaderboard_table_for_search,
-                                    shown_columns,
-                                    search_bar,
-                                    # filter_columns_type,
-                                    filter_domain_specific,
-                                    filter_columns_size
-                                ],
-                                leaderboard_table,
-                                queue=True,
-                            )
-
                         with gr.Accordion("💬 Generation templates", open=False):
                             with gr.Accordion("Response generation", open=False):
                                 render_generation_templates(task="open_ended", generation_type="response_generation")
                             with gr.Accordion("Scoring Rubric", open=False):
                                 render_generation_templates(task="open_ended", generation_type="scoring_rubric")
-
+        
         with gr.TabItem("🏅 Medical Summarization", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(CROSS_EVALUATION_METRICS, elem_classes="markdown-text")
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    # filter_columns_type = gr.CheckboxGroup(
-                    #     label="Model Types",
-                    #     choices=[t.to_str() for t in ModelType],
-                    #     value=[t.to_str() for t in ModelType],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-type",
-                    # )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
-                    filter_domain_specific = gr.CheckboxGroup(
-                        label="Domain Specificity",
-                        choices=["🏥  Clinical models", "Generic models"],
-                        value=["🏥  Clinical models", "Generic models"],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="medical_summarization")
-
-            leaderboard_table = gr.components.Dataframe(
-                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=datasets_original_df[MEDICAL_SUMMARIZATION_COLS],
-                headers=MEDICAL_SUMMARIZATION_COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-
-                        
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    search_bar,
-                    # filter_columns_type,
-                    filter_domain_specific,
-                    filter_columns_size
-                    # filter_columns_architecture
-                ],
-                leaderboard_table,
+            create_leaderboard_ui(
+                subset_name="medical_summarization",
+                column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
+                default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)]
             )
-            for selector in [
-                shown_columns,
-                # filter_columns_type,
-                filter_domain_specific,
-                filter_columns_size,
-                # deleted_models_visibility,
-            ]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        search_bar,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
             with gr.Accordion("💬 Generation templates", open=False):
                 with gr.Accordion("Response generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="medical_summarization", generation_type="response_generation")
+                    render_generation_templates(task="medical_summarization", generation_type="response_generation")
                 with gr.Accordion("Question generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
+                    render_generation_templates(task="ce", generation_type="question_generation")
                 with gr.Accordion("Cross Examination", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")  
-        
+                    render_generation_templates(task="ce", generation_type="cross_examination")
+                    
         with gr.TabItem("🏅 Note generation", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown(NOTE_GENERATION_METRICS, elem_classes="markdown-text")
-            with gr.Tabs(elem_classes="tab-buttons2") as tabs:
-                with gr.TabItem("ACI Bench", elem_id="llm-benchmark-tab-table2", id=0):
-                    with gr.Row():
-                        with gr.Column():
-                            with gr.Row():
-                                search_bar = gr.Textbox(
-                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                    show_label=False,
-                                    elem_id="search-bar",
-                                )
-                            with gr.Row():
-                                shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)],
-                                    value=[
-                                        c.name
-                                        for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)
-                                    ],
-                                    label="Select columns to show",
-                                    elem_id="column-select",
-                                    interactive=True,
-                                )
-                            # with gr.Row():
-                            #     deleted_models_visibility = gr.Checkbox(
-                            #         value=False, label="Show gated/private/deleted models", interactive=True
-                            #     )
-                        with gr.Column(min_width=320):
-                            # with gr.Box(elem_id="box-filter"):
-                            # filter_columns_type = gr.CheckboxGroup(
-                            #     label="Model Types",
-                            #     choices=[t.to_str() for t in ModelType],
-                            #     value=[t.to_str() for t in ModelType],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-type",
-                            # )
-                            # filter_columns_architecture = gr.CheckboxGroup(
-                            #     label="Architecture Types",
-                            #     choices=[i.value.name for i in ModelArch],
-                            #     value=[i.value.name for i in ModelArch],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-architecture",
-                            # )
-                            filter_domain_specific = gr.CheckboxGroup(
-                                label="Domain Specificity",
-                                choices=["🏥  Clinical models", "Generic models"],
-                                value=["🏥  Clinical models", "Generic models"],
-                                interactive=True,
-                                elem_id="filter-columns-type",
-                            )
-                            filter_columns_size = gr.CheckboxGroup(
-                                label="Model sizes (in billions of parameters)",
-                                choices=list(NUMERIC_INTERVALS.keys()),
-                                value=list(NUMERIC_INTERVALS.keys()),
-                                interactive=True,
-                                elem_id="filter-columns-size",
-                            )
-
-                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="aci")
-
-                    leaderboard_table = gr.components.Dataframe(
-                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                        datatype=TYPES,
-                        elem_id="leaderboard-table",
-                        interactive=False,
-                        visible=True,
+            with gr.Tabs(elem_classes="tab-buttons2"):
+                with gr.TabItem("ACI Bench", id=0):
+                    create_leaderboard_ui(
+                        subset_name="aci",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)]
                     )
-
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                        value=datasets_original_df[ACI_COLS],
-                        headers=ACI_COLS,
-                        datatype=TYPES,
-                        visible=False,
-                    )
-
-                                
-                    search_bar.submit(
-                        update_table,
-                        [
-                            hidden_leaderboard_table_for_search,
-                            shown_columns,
-                            search_bar,
-                            # filter_columns_type,
-                            filter_domain_specific,
-                            filter_columns_size
-                            # filter_columns_architecture
-                        ],
-                        leaderboard_table,
-                    )
-                    for selector in [
-                        shown_columns,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size,
-                        # deleted_models_visibility,
-                    ]:
-                        selector.change(
-                            update_table,
-                            [
-                                hidden_leaderboard_table_for_search,
-                                shown_columns,
-                                search_bar,
-                                # filter_columns_type,
-                                filter_domain_specific,
-                                filter_columns_size
-                            ],
-                            leaderboard_table,
-                            queue=True,
-                        )
-                with gr.TabItem("SOAP Notes", elem_id="llm-benchmark-tab-table2", id=1):
-                    with gr.Row():
-                        with gr.Column():
-                            with gr.Row():
-                                search_bar = gr.Textbox(
-                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                    show_label=False,
-                                    elem_id="search-bar",
-                                )
-                            with gr.Row():
-                                shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)],
-                                    value=[
-                                        c.name
-                                        for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)
-                                    ],
-                                    label="Select columns to show",
-                                    elem_id="column-select",
-                                    interactive=True,
-                                )
-                            # with gr.Row():
-                            #     deleted_models_visibility = gr.Checkbox(
-                            #         value=False, label="Show gated/private/deleted models", interactive=True
-                            #     )
-                        with gr.Column(min_width=320):
-                            # with gr.Box(elem_id="box-filter"):
-                            # filter_columns_type = gr.CheckboxGroup(
-                            #     label="Model Types",
-                            #     choices=[t.to_str() for t in ModelType],
-                            #     value=[t.to_str() for t in ModelType],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-type",
-                            # )
-                            # filter_columns_architecture = gr.CheckboxGroup(
-                            #     label="Architecture Types",
-                            #     choices=[i.value.name for i in ModelArch],
-                            #     value=[i.value.name for i in ModelArch],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-architecture",
-                            # )
-                            filter_domain_specific = gr.CheckboxGroup(
-                                label="Domain Specificity",
-                                choices=["🏥  Clinical models", "Generic models"],
-                                value=["🏥  Clinical models", "Generic models"],
-                                interactive=True,
-                                elem_id="filter-columns-type",
-                            )
-                            filter_columns_size = gr.CheckboxGroup(
-                                label="Model sizes (in billions of parameters)",
-                                choices=list(NUMERIC_INTERVALS.keys()),
-                                value=list(NUMERIC_INTERVALS.keys()),
-                                interactive=True,
-                                elem_id="filter-columns-size",
-                            )
-
-                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="soap")
-
-                    leaderboard_table = gr.components.Dataframe(
-                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                        datatype=TYPES,
-                        elem_id="leaderboard-table",
-                        interactive=False,
-                        visible=True,
+                with gr.TabItem("SOAP Notes", id=1):
+                    create_leaderboard_ui(
+                        subset_name="soap",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)]
                     )
-
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                        value=datasets_original_df[SOAP_COLS],
-                        headers=SOAP_COLS,
-                        datatype=TYPES,
-                        visible=False,
-                    )
-
-                                
-                    search_bar.submit(
-                        update_table,
-                        [
-                            hidden_leaderboard_table_for_search,
-                            shown_columns,
-                            search_bar,
-                            # filter_columns_type,
-                            filter_domain_specific,
-                            filter_columns_size
-                            # filter_columns_architecture
-                        ],
-                        leaderboard_table,
-                    )
-                    for selector in [
-                        shown_columns,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size,
-                        # deleted_models_visibility,
-                    ]:
-                        selector.change(
-                            update_table,
-                            [
-                                hidden_leaderboard_table_for_search,
-                                shown_columns,
-                                search_bar,
-                                # filter_columns_type,
-                                filter_domain_specific,
-                                filter_columns_size
-                            ],
-                            leaderboard_table,
-                            queue=True,
-                        )
-            with gr.Accordion("💬 Generation templates", open=False):
-                with gr.Accordion("ACI-Bench Response generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="aci", generation_type="response_generation")
-                with gr.Accordion("SOAP Notes Response generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="soap", generation_type="response_generation")
-                with gr.Accordion("Question generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
-                with gr.Accordion("Cross Examination", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
+            # Add accordions for this section if needed, similar to other tabs
         
         with gr.TabItem("🏅 HealthBench", elem_id="llm-benchmark-tab-table", id=4):
             gr.Markdown(HEALTHBENCH_METRICS, elem_classes="markdown-text")
-            with gr.Tabs(elem_classes="tab-buttons2") as tabs:
-                with gr.TabItem("HealthBench", elem_id="llm-benchmark-tab-table3", id=0):
-                    with gr.Row():
-                        with gr.Column():
-                            with gr.Row():
-                                search_bar = gr.Textbox(
-                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                    show_label=False,
-                                    elem_id="search-bar",
-                                )
-                            with gr.Row():
-                                shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_col)],
-                                    value=[
-                                        c.name
-                                        for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_col)
-                                    ],
-                                    label="Select columns to show",
-                                    elem_id="column-select",
-                                    interactive=True,
-                                )
-                            # with gr.Row():
-                            #     deleted_models_visibility = gr.Checkbox(
-                            #         value=False, label="Show gated/private/deleted models", interactive=True
-                            #     )
-                        with gr.Column(min_width=320):
-                            # with gr.Box(elem_id="box-filter"):
-                            # filter_columns_type = gr.CheckboxGroup(
-                            #     label="Model Types",
-                            #     choices=[t.to_str() for t in ModelType],
-                            #     value=[t.to_str() for t in ModelType],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-type",
-                            # )
-                            # filter_columns_architecture = gr.CheckboxGroup(
-                            #     label="Architecture Types",
-                            #     choices=[i.value.name for i in ModelArch],
-                            #     value=[i.value.name for i in ModelArch],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-architecture",
-                            # )
-                            filter_domain_specific = gr.CheckboxGroup(
-                                label="Domain Specificity",
-                                choices=["🏥  Clinical models", "Generic models"],
-                                value=["🏥  Clinical models", "Generic models"],
-                                interactive=True,
-                                elem_id="filter-columns-type",
-                            )
-                            filter_columns_size = gr.CheckboxGroup(
-                                label="Model sizes (in billions of parameters)",
-                                choices=list(NUMERIC_INTERVALS.keys()),
-                                value=list(NUMERIC_INTERVALS.keys()),
-                                interactive=True,
-                                elem_id="filter-columns-size",
-                            )
-
-                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="healthbench")
-
-                    leaderboard_table = gr.components.Dataframe(
-                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                        datatype=TYPES,
-                        elem_id="leaderboard-table",
-                        interactive=False,
-                        visible=True,
+            with gr.Tabs(elem_classes="tab-buttons2"):
+                with gr.TabItem("HealthBench", id=0):
+                    create_leaderboard_ui(
+                        subset_name="healthbench",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_col)]
                     )
-
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                        value=datasets_original_df[HEALTHBENCH_COLS],
-                        headers=HEALTHBENCH_COLS,
-                        datatype=TYPES,
-                        visible=False,
-                    )
-
-                                
-                    search_bar.submit(
-                        update_table,
-                        [
-                            hidden_leaderboard_table_for_search,
-                            shown_columns,
-                            search_bar,
-                            # filter_columns_type,
-                            filter_domain_specific,
-                            filter_columns_size
-                            # filter_columns_architecture
-                        ],
-                        leaderboard_table,
+                with gr.TabItem("HealthBench-Hard", id=1):
+                    create_leaderboard_ui(
+                        subset_name="healthbench_hard",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_hard_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_hard_col)]
                     )
-                    for selector in [
-                        shown_columns,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size,
-                        # deleted_models_visibility,
-                    ]:
-                        selector.change(
-                            update_table,
-                            [
-                                hidden_leaderboard_table_for_search,
-                                shown_columns,
-                                search_bar,
-                                # filter_columns_type,
-                                filter_domain_specific,
-                                filter_columns_size
-                            ],
-                            leaderboard_table,
-                            queue=True,
-                        )
-                with gr.TabItem("HealthBench-Hard", elem_id="llm-benchmark-tab-table3", id=1):
-                    with gr.Row():
-                        with gr.Column():
-                            with gr.Row():
-                                search_bar = gr.Textbox(
-                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                    show_label=False,
-                                    elem_id="search-bar",
-                                )
-                            with gr.Row():
-                                shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_hard_col)],
-                                    value=[
-                                        c.name
-                                        for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_hard_col)
-                                    ],
-                                    label="Select columns to show",
-                                    elem_id="column-select",
-                                    interactive=True,
-                                )
-                            # with gr.Row():
-                            #     deleted_models_visibility = gr.Checkbox(
-                            #         value=False, label="Show gated/private/deleted models", interactive=True
-                            #     )
-                        with gr.Column(min_width=320):
-                            # with gr.Box(elem_id="box-filter"):
-                            # filter_columns_type = gr.CheckboxGroup(
-                            #     label="Model Types",
-                            #     choices=[t.to_str() for t in ModelType],
-                            #     value=[t.to_str() for t in ModelType],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-type",
-                            # )
-                            # filter_columns_architecture = gr.CheckboxGroup(
-                            #     label="Architecture Types",
-                            #     choices=[i.value.name for i in ModelArch],
-                            #     value=[i.value.name for i in ModelArch],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-architecture",
-                            # )
-                            filter_domain_specific = gr.CheckboxGroup(
-                                label="Domain Specificity",
-                                choices=["🏥  Clinical models", "Generic models"],
-                                value=["🏥  Clinical models", "Generic models"],
-                                interactive=True,
-                                elem_id="filter-columns-type",
-                            )
-                            filter_columns_size = gr.CheckboxGroup(
-                                label="Model sizes (in billions of parameters)",
-                                choices=list(NUMERIC_INTERVALS.keys()),
-                                value=list(NUMERIC_INTERVALS.keys()),
-                                interactive=True,
-                                elem_id="filter-columns-size",
-                            )
-
-                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="healthbench_hard")
-
-                    leaderboard_table = gr.components.Dataframe(
-                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                        datatype=TYPES,
-                        elem_id="leaderboard-table",
-                        interactive=False,
-                        visible=True,
-                    )
-
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                        value=datasets_original_df[HEALTHBENCH_HARD_COLS],
-                        headers=HEALTHBENCH_HARD_COLS,
-                        datatype=TYPES,
-                        visible=False,
-                    )
-
-                                
-                    search_bar.submit(
-                        update_table,
-                        [
-                            hidden_leaderboard_table_for_search,
-                            shown_columns,
-                            search_bar,
-                            # filter_columns_type,
-                            filter_domain_specific,
-                            filter_columns_size
-                            # filter_columns_architecture
-                        ],
-                        leaderboard_table,
-                    )
-                    for selector in [
-                        shown_columns,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size,
-                        # deleted_models_visibility,
-                    ]:
-                        selector.change(
-                            update_table,
-                            [
-                                hidden_leaderboard_table_for_search,
-                                shown_columns,
-                                search_bar,
-                                # filter_columns_type,
-                                filter_domain_specific,
-                                filter_columns_size
-                            ],
-                            leaderboard_table,
-                            queue=True,
-                        )        
 
         with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=5):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    # filter_columns_type = gr.CheckboxGroup(
-                    #     label="Model Types",
-                    #     choices=[t.to_str() for t in ModelType],
-                    #     value=[t.to_str() for t in ModelType],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-type",
-                    # )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
-                    filter_domain_specific = gr.CheckboxGroup(
-                        label="Domain Specificity",
-                        choices=["🏥  Clinical models", "Generic models"],
-                        value=["🏥  Clinical models", "Generic models"],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="med_safety")
-
-            leaderboard_table = gr.components.Dataframe(
-                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
+            create_leaderboard_ui(
+                subset_name="med_safety",
+                column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)],
+                default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)]
             )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=datasets_original_df[MED_SAFETY_COLS],
-                headers=MED_SAFETY_COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-
-                        
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    search_bar,
-                    # filter_columns_type,
-                    filter_domain_specific,
-                    filter_columns_size
-                    # filter_columns_architecture
-                ],
-                leaderboard_table,
-            )
-            for selector in [
-                shown_columns,
-                # filter_columns_type,
-                filter_domain_specific,
-                filter_columns_size,
-                # deleted_models_visibility,
-            ]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        search_bar,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
             with gr.Accordion("💬 Generation templates", open=False):
                 with gr.Accordion("Response generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="response_generation")
+                    render_generation_templates(task="med_safety", generation_type="response_generation")
                 with gr.Accordion("Scoring Rubric", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="scoring_rubric")
-
+                    render_generation_templates(task="med_safety", generation_type="scoring_rubric")
+        
         with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-closed", id=6):
-            with gr.Tabs(elem_classes="tab-buttons2") as closed_tabs:
-                # ENGLISH TAB
-                with gr.TabItem("English", elem_id="llm-benchmark-tab-closed-english", id=0):
-                    with gr.Row():
-                        with gr.Column():
-                            with gr.Row():
-                                search_bar = gr.Textbox(
-                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                    show_label=False,
-                                    elem_id="search-bar-closed-english",
-                                )
-                            with gr.Row():
-                                shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
-                                    value=[
-                                        c.name
-                                        for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)
-                                    ],
-                                    label="Select columns to show",
-                                    elem_id="column-select-closed-english",
-                                    interactive=True,
-                                )
-                        with gr.Column(min_width=320):
-                            # filter_columns_type = gr.CheckboxGroup(
-                            #     label="Model Types",
-                            #     choices=[t.to_str() for t in ModelType],
-                            #     value=[t.to_str() for t in ModelType],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-type-closed-english",
-                            # )
-                            filter_domain_specific = gr.CheckboxGroup(
-                                label="Domain Specificity",
-                                choices=["🏥  Clinical models", "Generic models"],
-                                value=["🏥  Clinical models", "Generic models"],
-                                interactive=True,
-                                elem_id="filter-domain-specific-closed-english",
-                            )
-                            filter_columns_size = gr.CheckboxGroup(
-                                label="Model sizes (in billions of parameters)",
-                                choices=list(NUMERIC_INTERVALS.keys()),
-                                value=list(NUMERIC_INTERVALS.keys()),
-                                interactive=True,
-                                elem_id="filter-columns-size-closed-english",
-                            )
-
-                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="datasets")
-                    leaderboard_table = gr.components.Dataframe(
-                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                        datatype=TYPES,
-                        elem_id="leaderboard-table-english",
-                        interactive=False,
-                        visible=True,
+            with gr.Tabs(elem_classes="tab-buttons2"):
+                with gr.TabItem("English", id=0):
+                    create_leaderboard_ui(
+                        subset_name="datasets",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)]
+                    )
+                with gr.TabItem("🌍 Multilingual", id=1):
+                    gr.Markdown("📊 **Dataset Information:** This tab uses the Global MMLU dataset filtering only the subcategory: medical (10.7%)")
+                    create_leaderboard_ui(
+                        subset_name="closed_ended_multilingual",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)]
                     )
 
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                        value=datasets_original_df[DATASET_COLS],
-                        headers=DATASET_COLS,
-                        datatype=TYPES,
-                        visible=False,
+        with gr.TabItem("🏅 EHRSQL", elem_id="llm-benchmark-tab-table", id=7):
+            with gr.Tabs(elem_classes="tab-buttons2"):
+                with gr.TabItem("Zero Shot", id=0):
+                    create_leaderboard_ui(
+                        subset_name="ehrsql_zero_shot",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.ehrsql_zero_shot_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.ehrsql_zero_shot_col)]
                     )
-                                
-                    search_bar.submit(
-                        update_table,
-                        [
-                            hidden_leaderboard_table_for_search,
-                            shown_columns,
-                            search_bar,
-                            # filter_columns_type,
-                            filter_domain_specific,
-                            filter_columns_size
-                        ],
-                        leaderboard_table,
+                with gr.TabItem("Few Shot", id=1):
+                    create_leaderboard_ui(
+                        subset_name="ehrsql_few_shot",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.ehrsql_few_shot_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.ehrsql_few_shot_col)]
                     )
-                    
-                    for selector in [
-                        shown_columns,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size,
-                    ]:
-                        selector.change(
-                            update_table,
-                            [
-                                hidden_leaderboard_table_for_search,
-                                shown_columns,
-                                search_bar,
-                                # filter_columns_type,
-                                filter_domain_specific,
-                                filter_columns_size
-                            ],
-                            leaderboard_table,
-                            queue=True,
-                        )
-                
-                #MULTILINGUAL TAB - Same level as English tab
-                with gr.TabItem("🌍 Multilingual", elem_id="llm-benchmark-tab-table9", id=1):
-                    with gr.Row():
-                        gr.Markdown("📊 **Dataset Information:** This tab uses the Global MMLU dataset filtering only the subcategory: medical (10.7%)")
-                    
-                    with gr.Row():
-                        with gr.Column():
-                            with gr.Row():
-                                search_bar = gr.Textbox(
-                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                    show_label=False,
-                                    elem_id="search-bar",
-                                )
-                                
-                            with gr.Row():
-                                shown_columns = gr.CheckboxGroup(
-                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)],
-                                    value=[
-                                        c.name
-                                        for c in fields(AutoEvalColumn)
-                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)
-                                    ],
-                                    label="Select columns to show",
-                                    elem_id="column-select",
-                                    interactive=True,
-                                )
-                        with gr.Column(min_width=320):
-                            # with gr.Box(elem_id="box-filter"):
-                            # filter_columns_type = gr.CheckboxGroup(
-                            #     label="Model Types",
-                            #     choices=[t.to_str() for t in ModelType],
-                            #     value=[t.to_str() for t in ModelType],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-type",
-                            # )
-                            filter_domain_specific = gr.CheckboxGroup(
-                                label="Domain Specificity",
-                                choices=["🏥  Clinical models", "Generic models"],
-                                value=["🏥  Clinical models", "Generic models"],
-                                interactive=True,
-                                elem_id="filter-columns-type",
-                            )
-                            filter_columns_size = gr.CheckboxGroup(
-                                label="Model sizes (in billions of parameters)",
-                                choices=list(NUMERIC_INTERVALS.keys()),
-                                value=list(NUMERIC_INTERVALS.keys()),
-                                interactive=True,
-                                elem_id="filter-columns-size",
-                            )
 
-                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="closed_ended_multilingual")
-                    leaderboard_table = gr.components.Dataframe(
-                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                        datatype=TYPES,
-                        elem_id="leaderboard-table",
-                        interactive=False,
-                        visible=True,
+        with gr.TabItem("🏅 MedCalc", elem_id="llm-benchmark-tab-table", id=8):
+            with gr.Tabs(elem_classes="tab-buttons2"):
+                with gr.TabItem("Direct Answer", id=0):
+                    create_leaderboard_ui(
+                        subset_name="medcalc_direct_answer",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medcalc_direct_answer_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medcalc_direct_answer_col)]
                     )
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                        value=datasets_original_df[ClosedEndedMultilingual_COLS],
-                        headers=ClosedEndedMultilingual_COLS,
-                        datatype=TYPES,
-                        visible=False,
+                with gr.TabItem("One Shot CoT", id=1):
+                    create_leaderboard_ui(
+                        subset_name="medcalc_one_shot_cot",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medcalc_one_shot_cot_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medcalc_one_shot_cot_col)]
                     )
-                        
-                    search_bar.submit(
-                        update_table,
-                        [
-                            hidden_leaderboard_table_for_search,
-                            shown_columns,
-                            search_bar,
-                            # filter_columns_type,
-                            filter_domain_specific,
-                            filter_columns_size
-                            # filter_columns_architecture
-                        ],
-                        leaderboard_table,
+                with gr.TabItem("Zero Shot CoT", id=2):
+                    create_leaderboard_ui(
+                        subset_name="medcalc_zero_shot_cot",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medcalc_zero_shot_cot_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medcalc_zero_shot_cot_col)]
                     )
-                    for selector in [
-                        shown_columns,
-                        # filter_columns_type,
-                        filter_domain_specific,
-                        # filter_columns_architecture,
-                        filter_columns_size,
-                        # deleted_models_visibility,
-                    ]:
-                        selector.change(
-                            update_table,
-                            [
-                                hidden_leaderboard_table_for_search,
-                                shown_columns,
-                                search_bar,
-                                # filter_columns_type,
-                                filter_domain_specific,
-                                filter_columns_size
-                                # filter_columns_architecture,
-                            ],
-                            leaderboard_table,
-                            queue=True,
-                        )
-    
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=7):
+
+        with gr.TabItem("🏅 MedEC", elem_id="llm-benchmark-tab-table", id=9):
+            with gr.Tabs(elem_classes="tab-buttons2"):
+                with gr.TabItem("Zero Shot", id=0):
+                    create_leaderboard_ui(
+                        subset_name="medec_zero_shot",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medec_zero_shot_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medec_zero_shot_col)]
+                    )
+                with gr.TabItem("One Shot", id=1):
+                    create_leaderboard_ui(
+                        subset_name="medec_one_shot",
+                        column_choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medec_one_shot_col)],
+                        default_columns=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medec_one_shot_col)]
+                    )
+
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=10):
             gr.Markdown(LLM_BENCHMARKS_TEXT_1, elem_classes="markdown-text")
             gr.HTML(FIVE_PILLAR_DIAGRAM)
             gr.Markdown(LLM_BENCHMARKS_TEXT_2, elem_classes="markdown-text")
-            # gr.HTML(EVALUATION_EXAMPLE_IMG, elem_classes="logo")
-            # gr.Markdown(LLM_BENCHMARKS_TEXT_2, elem_classes="markdown-text")
-            # gr.HTML(ENTITY_DISTRIBUTION_IMG, elem_classes="logo")
-            # gr.Markdown(LLM_BENCHMARKS_TEXT_3, elem_classes="markdown-text")
             
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=8):
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=11):
+            
             with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+                gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+                with gr.Accordion(f"✅ Finished Evaluations ({len(finished_eval_queue_df)})", open=False):
+                    gr.Dataframe(value=finished_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5)
+                with gr.Accordion(f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})", open=False):
+                    gr.Dataframe(value=running_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5)
+                with gr.Accordion(f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})", open=False):
+                    gr.Dataframe(value=pending_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5)
+            
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
@@ -1459,10 +498,9 @@ with demo:
                 submission_result,
             )
 
-
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
+            gr.Textbox(
                 value=CITATION_BUTTON_TEXT,
                 label=CITATION_BUTTON_LABEL,
                 lines=20,
@@ -1470,7 +508,9 @@ with demo:
                 show_copy_button=True,
             )
 
+
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=86400) 
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch(allowed_paths=['./assets/'])
+
+demo.queue(default_concurrency_limit=40).launch(allowed_paths=['./assets/'], share=True , ssr_mode=False)

requirements.txt

@@ -11,8 +11,6 @@ pandas
 python-dateutil
 requests
 tqdm
-git+https://github.com/huggingface/transformers.git
+transformers
 tokenizers>=0.15.0
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
-accelerate
 sentencepiece

src/about.py

@@ -209,6 +209,100 @@ class HealthbenchHardColumns(Enum):
     healthbench_hard_column12 = HealthbenchHardColumn("Axis: Instruction following", "score", "Axis: Instruction following")
     healthbench_hard_column13 = HealthbenchHardColumn("Axis: Communication quality", "score", "Axis: Communication quality")
 
+@dataclass
+class EHRSQLZeroShotColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class EHRSQLZeroShotColumns(Enum):
+    ehrsql_zero_shot_column0 = EHRSQLZeroShotColumn("RS (0)", "score", "RS (0)")
+    ehrsql_zero_shot_column1 = EHRSQLZeroShotColumn("Abstains correct %", "score", "Abstains correct %")
+    ehrsql_zero_shot_column2 = EHRSQLZeroShotColumn("Abstains incorrect %", "score", "Abstains incorrect %")
+    ehrsql_zero_shot_column3 = EHRSQLZeroShotColumn("Abstains failed %", "score", "Abstains failed %")
+
+@dataclass
+class EHRSQLFewShotColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class EHRSQLFewShotColumns(Enum):
+    ehrsql_few_shot_column0 = EHRSQLFewShotColumn("RS (0)", "score", "RS (0)")
+    ehrsql_few_shot_column1 = EHRSQLFewShotColumn("Abstains correct %", "score", "Abstains correct %")
+    ehrsql_few_shot_column2 = EHRSQLFewShotColumn("Abstains incorrect %", "score", "Abstains incorrect %")
+    ehrsql_few_shot_column3 = EHRSQLFewShotColumn("Abstains failed %", "score", "Abstains failed %")
+
+@dataclass
+class MedCalcDirectAnswerColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class MedCalcDirectAnswerColumns(Enum):
+    medcalc_direct_answer_column0 = MedCalcDirectAnswerColumn("lab", "score", "Lab")
+    medcalc_direct_answer_column1 = MedCalcDirectAnswerColumn("risk", "score", "Risk")
+    medcalc_direct_answer_column2 = MedCalcDirectAnswerColumn("physical", "score", "Physical")
+    medcalc_direct_answer_column3 = MedCalcDirectAnswerColumn("severity", "score", "Severity")
+    medcalc_direct_answer_column4 = MedCalcDirectAnswerColumn("diagnosis", "score", "Diagnosis")
+    medcalc_direct_answer_column5 = MedCalcDirectAnswerColumn("date", "score", "Date")
+    medcalc_direct_answer_column6 = MedCalcDirectAnswerColumn("dosage", "score", "Dosage")
+    medcalc_direct_answer_column7 = MedCalcDirectAnswerColumn("overall", "score", "Overall")
+
+@dataclass
+class MedCalcOneShotCotColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class MedCalcOneShotCotColumns(Enum):
+    medcalc_one_shot_cot_column0 = MedCalcOneShotCotColumn("lab", "score", "Lab")
+    medcalc_one_shot_cot_column1 = MedCalcOneShotCotColumn("risk", "score", "Risk")
+    medcalc_one_shot_cot_column2 = MedCalcOneShotCotColumn("physical", "score", "Physical")
+    medcalc_one_shot_cot_column3 = MedCalcOneShotCotColumn("severity", "score", "Severity")
+    medcalc_one_shot_cot_column4 = MedCalcOneShotCotColumn("diagnosis", "score", "Diagnosis")
+    medcalc_one_shot_cot_column5 = MedCalcOneShotCotColumn("date", "score", "Date")
+    medcalc_one_shot_cot_column6 = MedCalcOneShotCotColumn("dosage", "score", "Dosage")
+    medcalc_one_shot_cot_column7 = MedCalcOneShotCotColumn("overall", "score", "Overall")
+
+@dataclass
+class MedCalcZeroShotCotColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class MedCalcZeroShotCotColumns(Enum):
+    medcalc_zero_shot_cot_column0 = MedCalcZeroShotCotColumn("lab", "score", "Lab")
+    medcalc_zero_shot_cot_column1 = MedCalcZeroShotCotColumn("risk", "score", "Risk")
+    medcalc_zero_shot_cot_column2 = MedCalcZeroShotCotColumn("physical", "score", "Physical")
+    medcalc_zero_shot_cot_column3 = MedCalcZeroShotCotColumn("severity", "score", "Severity")
+    medcalc_zero_shot_cot_column4 = MedCalcZeroShotCotColumn("diagnosis", "score", "Diagnosis")
+    medcalc_zero_shot_cot_column5 = MedCalcZeroShotCotColumn("date", "score", "Date")
+    medcalc_zero_shot_cot_column6 = MedCalcZeroShotCotColumn("dosage", "score", "Dosage")
+    medcalc_zero_shot_cot_column7 = MedCalcZeroShotCotColumn("overall", "score", "Overall")
+
+@dataclass
+class MedECZeroShotColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class MedECZeroShotColumns(Enum):
+    medec_zero_shot_column0 = MedECZeroShotColumn("Error Flag Accuracy (%)", "score", "Error Flag Accuracy (%)")
+    medec_zero_shot_column1 = MedECZeroShotColumn("Error Sentence ID Accuracy (%)", "score", "Error Sentence ID Accuracy (%)")
+    medec_zero_shot_column2 = MedECZeroShotColumn("Invalid Responses (%)", "score", "Invalid Responses (%)")
+
+@dataclass
+class MedECOneShotColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+
+class MedECOneShotColumns(Enum):
+    medec_one_shot_column0 = MedECOneShotColumn("Error Flag Accuracy (%)", "score", "Error Flag Accuracy (%)")
+    medec_one_shot_column1 = MedECOneShotColumn("Error Sentence ID Accuracy (%)", "score", "Error Sentence ID Accuracy (%)")
+    medec_one_shot_column2 = MedECOneShotColumn("Invalid Responses (%)", "score", "Invalid Responses (%)")
+
 NUM_FEWSHOT = 0  # Change with your few shot
 # ---------------------------------------------------
 

src/display/utils.py

@@ -4,7 +4,7 @@ from enum import Enum
 import pandas as pd
 
 # changes to be made here
-from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns, ClosedEndedMultilingualColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn
+from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns, ClosedEndedMultilingualColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn, EHRSQLZeroShotColumns, EHRSQLFewShotColumns, MedCalcDirectAnswerColumns, MedCalcOneShotCotColumns, MedCalcZeroShotCotColumns, MedECZeroShotColumns, MedECOneShotColumns
 from src.envs import PRIVATE_REPO
 import json
 import gradio as gr
@@ -41,6 +41,13 @@ class ColumnContent:
     open_ended_romanian_col: bool = False
     open_ended_greek_col: bool = False
     closed_ended_multilingual_col: bool = False
+    ehrsql_zero_shot_col: bool = False
+    ehrsql_few_shot_col: bool = False
+    medcalc_direct_answer_col: bool = False
+    medcalc_one_shot_cot_col: bool = False
+    medcalc_zero_shot_cot_col: bool = False
+    medec_zero_shot_col: bool = False
+    medec_one_shot_col: bool = False
 
 
 ## Leaderboard columns
@@ -78,6 +85,20 @@ for column in HealthbenchHardColumns:
         auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", False, False, healthbench_hard_col=True, invariant=False)])
     else:
         auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, healthbench_hard_col=True, invariant=False)])
+for column in EHRSQLZeroShotColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, ehrsql_zero_shot_col=True, invariant=False)])
+for column in EHRSQLFewShotColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, ehrsql_few_shot_col=True, invariant=False)])
+for column in MedCalcDirectAnswerColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, medcalc_direct_answer_col=True, invariant=False)])
+for column in MedCalcOneShotCotColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, medcalc_one_shot_cot_col=True, invariant=False)])
+for column in MedCalcZeroShotCotColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, medcalc_zero_shot_cot_col=True, invariant=False)])
+for column in MedECZeroShotColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, medec_zero_shot_col=True, invariant=False)])
+for column in MedECOneShotColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, medec_one_shot_col=True, invariant=False)])
 for column in OpenEndedArabicColumn:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_arabic_col=True, invariant=False)])
 for column in OpenEndedFrenchColumn:
@@ -261,7 +282,14 @@ OpenEndedPortuguese_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidd
 OpenEndedRomanian_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_romanian_col or c.invariant)]
 OpenEndedGreek_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_greek_col or c.invariant)]
 
+EHRSQL_ZERO_SHOT_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.ehrsql_zero_shot_col or c.invariant)]
+EHRSQL_FEW_SHOT_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.ehrsql_few_shot_col or c.invariant)]
 
+MEDCALC_DIRECT_ANSWER_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medcalc_direct_answer_col or c.invariant)]
+MEDCALC_ONE_SHOT_COT_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medcalc_one_shot_cot_col or c.invariant)]
+MEDCALC_ZERO_SHOT_COT_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medcalc_zero_shot_cot_col or c.invariant)]
+MEDEC_ZERO_SHOT_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medec_zero_shot_col or c.invariant)]
+MEDEC_ONE_SHOT_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medec_one_shot_col or c.invariant)]
 
 ClosedEndedMultilingual_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.closed_ended_multilingual_col or c.invariant)]
 
@@ -301,6 +329,14 @@ OpenEndedSpanish_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedSpanishCol
 OpenEndedRomanian_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedRomanianColumn]
 OpenEndedGreek_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedGreekColumn]
 
+EHRSQL_ZERO_SHOT_BENCHMARK_COLS = [t.value.col_name for t in EHRSQLZeroShotColumns]
+EHRSQL_FEW_SHOT_BENCHMARK_COLS = [t.value.col_name for t in EHRSQLFewShotColumns]
+
+MEDCALC_DIRECT_ANSWER_BENCHMARK_COLS = [t.value.col_name for t in MedCalcDirectAnswerColumns]
+MEDCALC_ONE_SHOT_COT_BENCHMARK_COLS = [t.value.col_name for t in MedCalcOneShotCotColumns]
+MEDCALC_ZERO_SHOT_COT_BENCHMARK_COLS = [t.value.col_name for t in MedCalcZeroShotCotColumns]
+MEDEC_ZERO_SHOT_BENCHMARK_COLS = [t.value.col_name for t in MedECZeroShotColumns]
+MEDEC_ONE_SHOT_BENCHMARK_COLS = [t.value.col_name for t in MedECOneShotColumns]
 
 ClosedEndedMultilingual_BENCHMARK_COLS = [t.value.col_name for t in ClosedEndedMultilingualColumns]
 

src/leaderboard/read_evals.py

@@ -9,7 +9,7 @@ import numpy as np
 
 from src.display.formatting import make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns, ClosedEndedMultilingualColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns, ClosedEndedMultilingualColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn, EHRSQLZeroShotColumns, EHRSQLFewShotColumns, MedCalcDirectAnswerColumns, MedCalcOneShotCotColumns, MedCalcZeroShotCotColumns, MedECZeroShotColumns, MedECOneShotColumns
 from src.submission.check_validity import is_model_on_hub
 from src.envs import PRIVATE_REPO
 
@@ -38,6 +38,13 @@ class EvalResult:
     open_ended_romanian_results: dict
     open_ended_greek_results: dict
     closed_ended_multilingual_results: dict
+    ehrsql_zero_shot_results: dict
+    ehrsql_few_shot_results: dict
+    medcalc_direct_answer_results: dict
+    medcalc_one_shot_cot_results: dict
+    medcalc_zero_shot_cot_results: dict
+    medec_zero_shot_results: dict
+    medec_one_shot_results: dict
     is_domain_specific: bool
     use_chat_template: bool
     # clinical_type_results:dict
@@ -269,6 +276,55 @@ class EvalResult:
                 task = task.value
                 accs = data["results"]["closed-ended-multilingual"][task.benchmark]["accuracy"] if task.benchmark in data["results"]["closed-ended-multilingual"] else None
                 closed_ended_multilingual_results[task.benchmark] = accs
+        
+        ehrsql_zero_shot_results = {}
+        if "ehrsql" in data["results"] and "zero_shot" in data["results"]["ehrsql"]:
+            for task in EHRSQLZeroShotColumns:
+                task = task.value
+                accs = data["results"]["ehrsql"]["zero_shot"][task.benchmark] if task.benchmark in data["results"]["ehrsql"]["zero_shot"] else None
+                ehrsql_zero_shot_results[task.benchmark] = accs
+        
+        ehrsql_few_shot_results = {}
+        if "ehrsql" in data["results"] and "few_shot" in data["results"]["ehrsql"]:
+            for task in EHRSQLFewShotColumns:
+                task = task.value
+                accs = data["results"]["ehrsql"]["few_shot"][task.benchmark] if task.benchmark in data["results"]["ehrsql"]["few_shot"] else None
+                ehrsql_few_shot_results[task.benchmark] = accs
+        
+        medcalc_direct_answer_results = {}
+        if "medcalc" in data["results"] and "direct_answer" in data["results"]["medcalc"]:
+            for task in MedCalcDirectAnswerColumns:
+                task = task.value
+                accs = data["results"]["medcalc"]["direct_answer"].get(task.benchmark, {}).get("average", None)
+                medcalc_direct_answer_results[task.benchmark] = accs
+        
+        medcalc_one_shot_cot_results = {}
+        if "medcalc" in data["results"] and "one_shot_cot" in data["results"]["medcalc"]:
+            for task in MedCalcOneShotCotColumns:
+                task = task.value
+                accs = data["results"]["medcalc"]["one_shot_cot"].get(task.benchmark, {}).get("average", None)
+                medcalc_one_shot_cot_results[task.benchmark] = accs
+        
+        medcalc_zero_shot_cot_results = {}
+        if "medcalc" in data["results"] and "zero_shot_cot" in data["results"]["medcalc"]:
+            for task in MedCalcZeroShotCotColumns:
+                task = task.value
+                accs = data["results"]["medcalc"]["zero_shot_cot"].get(task.benchmark, {}).get("average", None)
+                medcalc_zero_shot_cot_results[task.benchmark] = accs
+        
+        medec_zero_shot_results = {}
+        if "medec" in data["results"] and "zero_shot" in data["results"]["medec"]:
+            for task in MedECZeroShotColumns:
+                task = task.value
+                accs = data["results"]["medec"]["zero_shot"].get(task.benchmark, None)
+                medec_zero_shot_results[task.benchmark] = accs
+        
+        medec_one_shot_results = {}
+        if "medec" in data["results"] and "one_shot" in data["results"]["medec"]:
+            for task in MedECOneShotColumns:
+                task = task.value
+                accs = data["results"]["medec"]["one_shot"].get(task.benchmark, None)
+                medec_one_shot_results[task.benchmark] = accs
             
         #         #add the 
         # closed_ended_arabic_results = {}
@@ -327,6 +383,13 @@ class EvalResult:
             open_ended_romanian_results=open_ended_romanian_results,
             open_ended_greek_results=open_ended_greek_results,
             closed_ended_multilingual_results=closed_ended_multilingual_results,
+            ehrsql_zero_shot_results=ehrsql_zero_shot_results,
+            ehrsql_few_shot_results=ehrsql_few_shot_results,
+            medcalc_direct_answer_results=medcalc_direct_answer_results,
+            medcalc_one_shot_cot_results=medcalc_one_shot_cot_results,
+            medcalc_zero_shot_cot_results=medcalc_zero_shot_cot_results,
+            medec_zero_shot_results=medec_zero_shot_results,
+            medec_one_shot_results=medec_one_shot_results,
             is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
             use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,
@@ -476,6 +539,41 @@ class EvalResult:
                 for task in ClosedEndedMultilingualColumns:
                     data_dict[task.value.col_name] = self.closed_ended_multilingual_results[task.value.benchmark]
             return data_dict
+        if subset == "ehrsql_zero_shot":
+            if len(self.ehrsql_zero_shot_results) > 0:
+                for task in EHRSQLZeroShotColumns:
+                    data_dict[task.value.col_name] = self.ehrsql_zero_shot_results[task.value.benchmark]
+            return data_dict
+        if subset == "ehrsql_few_shot":
+            if len(self.ehrsql_few_shot_results) > 0:
+                for task in EHRSQLFewShotColumns:
+                    data_dict[task.value.col_name] = self.ehrsql_few_shot_results[task.value.benchmark]
+            return data_dict
+        if subset == "medcalc_direct_answer":
+            if len(self.medcalc_direct_answer_results) > 0:
+                for task in MedCalcDirectAnswerColumns:
+                    data_dict[task.value.col_name] = self.medcalc_direct_answer_results[task.value.benchmark]
+            return data_dict
+        if subset == "medcalc_one_shot_cot":
+            if len(self.medcalc_one_shot_cot_results) > 0:
+                for task in MedCalcOneShotCotColumns:
+                    data_dict[task.value.col_name] = self.medcalc_one_shot_cot_results[task.value.benchmark]
+            return data_dict
+        if subset == "medcalc_zero_shot_cot":
+            if len(self.medcalc_zero_shot_cot_results) > 0:
+                for task in MedCalcZeroShotCotColumns:
+                    data_dict[task.value.col_name] = self.medcalc_zero_shot_cot_results[task.value.benchmark]
+            return data_dict
+        if subset == "medec_zero_shot":
+            if len(self.medec_zero_shot_results) > 0:
+                for task in MedECZeroShotColumns:
+                    data_dict[task.value.col_name] = self.medec_zero_shot_results[task.value.benchmark]
+            return data_dict
+        if subset == "medec_one_shot":
+            if len(self.medec_one_shot_results) > 0:
+                for task in MedECOneShotColumns:
+                    data_dict[task.value.col_name] = self.medec_one_shot_results[task.value.benchmark]
+            return data_dict
 
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
@@ -539,12 +637,8 @@ def get_raw_eval_results(results_path: str, requests_path: str, evaluation_metri
     results = []
     # clinical_type_results = []
     for v in eval_results.values():
-        try:
-            v.to_dict(subset="dataset")  # we test if the dict version is complete
-            if not v.display_result:
-                continue
-            results.append(v)
-        except KeyError:  # not all eval values present
+        if not v.display_result:
             continue
+        results.append(v)
     # breakpoint()
     return results

src/populate.py

@@ -5,17 +5,13 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn, ClosedEndedMultilingualColumns
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn, ClosedEndedMultilingualColumns, EHRSQLZeroShotColumns, EHRSQLFewShotColumns, MedCalcDirectAnswerColumns, MedCalcOneShotCotColumns, MedCalcZeroShotCotColumns, MedECZeroShotColumns, MedECOneShotColumns
 from src.leaderboard.read_evals import get_raw_eval_results
 from src.envs import PRIVATE_REPO
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, evaluation_metric:str, subset:str) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data =  get_raw_eval_results(results_path, requests_path, evaluation_metric)
-    # print(raw_data)
-    # raise Exception("stop")
-    # if subset.startswith("healthbench"):
-    #     breakpoint()
     all_data_json = [v.to_dict(subset=subset) for v in raw_data if not v.full_model.startswith("/models_llm")]
     df = pd.DataFrame.from_records(all_data_json)
     # changes to be made here
@@ -39,25 +35,43 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         df = df.sort_values(by=["Overall Score"], ascending=False)
     elif subset == "closed_ended_multilingual":
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    elif subset == "ehrsql_zero_shot":
+        df = df.sort_values(by=["RS (0)"], ascending=False)
+    elif subset == "ehrsql_few_shot":
+        df = df.sort_values(by=["RS (0)"], ascending=False)
+    elif subset == "medcalc_direct_answer":
+        df = df.sort_values(by=["Overall"], ascending=False)
+    elif subset == "medcalc_one_shot_cot":
+        df = df.sort_values(by=["Overall"], ascending=False)
+    elif subset == "medcalc_zero_shot_cot":
+        df = df.sort_values(by=["Overall"], ascending=False)
+    elif subset == "medec_zero_shot":
+        df = df.sort_values(by=["Error Flag Accuracy (%)"], ascending=False)
+    elif subset == "medec_one_shot":
+        df = df.sort_values(by=["Error Flag Accuracy (%)"], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
     return raw_data, df
 
-
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     """Creates the different dataframes for the evaluation queues requestes"""
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
+    
     for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
+        full_path = os.path.join(save_path, entry)
+
+        if os.path.isdir(full_path):
+            continue
+
+        if entry.endswith(".json"):
+            with open(full_path) as fp:
                 data = json.load(fp)
+
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"]) if not data["private"] else data["model_name"]
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-            # changes to be made here
             data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
             data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
             data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
@@ -66,42 +80,24 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             if PRIVATE_REPO:
                 data[EvalQueueColumn.closed_ended_arabic_status.name] = data["status"]["closed-ended-arabic"]
             all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-                # print(data)
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model_name"]) if not data["private"] else data["model_name"]
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                data[EvalQueueColumn.closed_ended_status.name] = data["status"]["closed-ended"]
-                data[EvalQueueColumn.open_ended_status.name] = data["status"]["open-ended"]
-                data[EvalQueueColumn.med_safety_status.name] = data["status"]["med-safety"]
-                data[EvalQueueColumn.medical_summarization_status.name] = data["status"]["medical-summarization"]
-                data[EvalQueueColumn.note_generation_status.name] = data["status"]["note-generation"]
-                if PRIVATE_REPO:
-                    data[EvalQueueColumn.closed_ended_arabic_status.name] = data["status"]["closed-ended-arabic"]
-                all_evals.append(data)
-    # breakpoint()
+    
     pending_list = []
     running_list = []
     finished_list = []
     for run in all_evals:
-        # changes to be made here
         status_list = [run["status"]["closed-ended"], run["status"]["open-ended"], run["status"]["med-safety"], run["status"]["medical-summarization"], run["status"]["note-generation"]]
         if PRIVATE_REPO:
             status_list.append(run["status"]["closed-ended-arabic"])
-        # status_list = status_list
+        
         if "RUNNING" in status_list:
             running_list.append(run)
         elif "PENDING" in status_list or "RERUN" in status_list:
             pending_list.append(run)
         else:
             finished_list.append(run)
-        # breakpoint()
+
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    
     return df_finished[cols], df_running[cols], df_pending[cols]