[ADD] Healthbench

app.py

@@ -14,6 +14,7 @@ from src.about import (
     LLM_BENCHMARKS_TEXT_2,
     CROSS_EVALUATION_METRICS,
     NOTE_GENERATION_METRICS,
+    HEALTHBENCH_METRICS,
     # EVALUATION_EXAMPLE_IMG,
     # LLM_BENCHMARKS_TEXT_2,
     # ENTITY_DISTRIBUTION_IMG,
@@ -31,14 +32,16 @@ from src.display.utils import (
     MEDICAL_SUMMARIZATION_BENCHMARK_COLS,
     ACI_BENCHMARK_COLS,
     SOAP_BENCHMARK_COLS,
-    CLOSED_ENDED_ARABIC_BENCHMARK_COLS,
+    HEALTHBENCH_BENCHMARK_COLS,
+    HEALTHBENCH_HARD_BENCHMARK_COLS,
     DATASET_COLS,
     OPEN_ENDED_COLS,
     MED_SAFETY_COLS,
     MEDICAL_SUMMARIZATION_COLS,
     ACI_COLS,
     SOAP_COLS,
-    CLOSED_ENDED_ARABIC_COLS,
+    HEALTHBENCH_COLS,
+    HEALTHBENCH_HARD_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
@@ -96,9 +99,12 @@ aci_leaderboard_df = aci_original_df.copy()
 _, soap_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, SOAP_COLS, SOAP_BENCHMARK_COLS, "score", "soap")
 soap_leaderboard_df = soap_original_df.copy()
 
-if PRIVATE_REPO:
-    _, closed_ended_arabic_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, CLOSED_ENDED_ARABIC_COLS, CLOSED_ENDED_ARABIC_BENCHMARK_COLS, "score", "closed_ended_arabic")
-    closed_ended_arabic_leaderboard_df = closed_ended_arabic_original_df.copy()
+_, healthbench_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, HEALTHBENCH_COLS, HEALTHBENCH_BENCHMARK_COLS, "score", "healthbench")
+healthbench_leaderboard_df = healthbench_original_df.copy()
+
+_, healthbench_hard_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, HEALTHBENCH_HARD_COLS, HEALTHBENCH_HARD_BENCHMARK_COLS, "score", "healthbench_hard")
+healthbench_hard_leaderboard_df = healthbench_hard_original_df.copy()
+
 
 # breakpoint()
 # # Token based results
@@ -136,9 +142,12 @@ def update_df(shown_columns, subset="datasets"):
     elif subset == "soap":
         leaderboard_table_df = soap_leaderboard_df.copy()
         hidden_leader_board_df = soap_original_df
-    elif PRIVATE_REPO and subset == "closed-ended-arabic":
-        leaderboard_table_df = closed_ended_arabic_leaderboard_df.copy()
-        hidden_leader_board_df = closed_ended_arabic_original_df
+    elif subset == "healthbench":
+        leaderboard_table_df = healthbench_leaderboard_df.copy()
+        hidden_leader_board_df = healthbench_original_df
+    elif subset == "healthbench_hard":
+        leaderboard_table_df = healthbench_hard_leaderboard_df.copy()
+        hidden_leader_board_df = healthbench_hard_original_df
     # else:
     #     match evaluation_metric:
     #         case "Span Based":
@@ -380,121 +389,7 @@ with demo:
                     system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="response_generation")
                 with gr.Accordion("Scoring Rubric", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="scoring_rubric")
-        with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=2):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model Types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
-                    filter_domain_specific = gr.CheckboxGroup(
-                        label="Domain Specificity",
-                        choices=["🏥  Clinical models", "Generic models"],
-                        value=["🏥  Clinical models", "Generic models"],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="med_safety")
-
-            leaderboard_table = gr.components.Dataframe(
-                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=datasets_original_df[MED_SAFETY_COLS],
-                headers=MED_SAFETY_COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-
-                        
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    search_bar,
-                    filter_columns_type,
-                    filter_domain_specific,
-                    filter_columns_size
-                    # filter_columns_architecture
-                ],
-                leaderboard_table,
-            )
-            for selector in [
-                shown_columns,
-                filter_columns_type,
-                filter_domain_specific,
-                filter_columns_size,
-                # deleted_models_visibility,
-            ]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        search_bar,
-                        filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-            with gr.Accordion("💬 Generation templates", open=False):
-                with gr.Accordion("Response generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="response_generation")
-                with gr.Accordion("Scoring Rubric", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="scoring_rubric")
-        with gr.TabItem("🏅 Medical Summarization", elem_id="llm-benchmark-tab-table", id=3):
+        with gr.TabItem("🏅 Medical Summarization", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(CROSS_EVALUATION_METRICS, elem_classes="markdown-text")
             with gr.Row():
                 with gr.Column():
@@ -611,7 +506,7 @@ with demo:
                     system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
                 with gr.Accordion("Cross Examination", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")  
-        with gr.TabItem("🏅 Note generation", elem_id="llm-benchmark-tab-table", id=4):
+        with gr.TabItem("🏅 Note generation", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown(NOTE_GENERATION_METRICS, elem_classes="markdown-text")
             with gr.Tabs(elem_classes="tab-buttons2") as tabs:
                 with gr.TabItem("ACI Bench", elem_id="llm-benchmark-tab-table2", id=0):
@@ -840,87 +735,107 @@ with demo:
                 with gr.Accordion("Question generation", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
                 with gr.Accordion("Cross Examination", open=False):
-                        system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
-        if PRIVATE_REPO:
-            with gr.TabItem("Dev Evals", elem_id="llm-benchmark-tab-table", id=100):
-                with gr.Tabs(elem_classes="tab-buttons2") as tabs:
-                    with gr.TabItem("🏅 Arabic Closed Ended Evaluation", elem_id="llm-benchmark-tab-table100", id=0):
-                        with gr.Row():
-                            with gr.Column():
-                                with gr.Row():
-                                    search_bar = gr.Textbox(
-                                        placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                        show_label=False,
-                                        elem_id="search-bar",
-                                    )
-                                with gr.Row():
-                                    shown_columns = gr.CheckboxGroup(
-                                        choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_arabic_col)],
-                                        value=[
-                                            c.name
-                                            for c in fields(AutoEvalColumn)
-                                            if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_arabic_col)
-                                        ],
-                                        label="Select columns to show",
-                                        elem_id="column-select",
-                                        interactive=True,
-                                    )
-                                # with gr.Row():
-                                #     deleted_models_visibility = gr.Checkbox(
-                                #         value=False, label="Show gated/private/deleted models", interactive=True
-                                #     )
-                            with gr.Column(min_width=320):
-                                # with gr.Box(elem_id="box-filter"):
-                                filter_columns_type = gr.CheckboxGroup(
-                                    label="Model Types",
-                                    choices=[t.to_str() for t in ModelType],
-                                    value=[t.to_str() for t in ModelType],
-                                    interactive=True,
-                                    elem_id="filter-columns-type",
-                                )
-                                # filter_columns_architecture = gr.CheckboxGroup(
-                                #     label="Architecture Types",
-                                #     choices=[i.value.name for i in ModelArch],
-                                #     value=[i.value.name for i in ModelArch],
-                                #     interactive=True,
-                                #     elem_id="filter-columns-architecture",
-                                # )
-                                filter_domain_specific = gr.CheckboxGroup(
-                                    label="Domain Specificity",
-                                    choices=["🏥  Clinical models", "Generic models"],
-                                    value=["🏥  Clinical models", "Generic models"],
-                                    interactive=True,
-                                    elem_id="filter-columns-type",
+                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
+        with gr.TabItem("🏅 HealthBench", elem_id="llm-benchmark-tab-table", id=4):
+            gr.Markdown(HEALTHBENCH_METRICS, elem_classes="markdown-text")
+            with gr.Tabs(elem_classes="tab-buttons2") as tabs:
+                with gr.TabItem("HealthBench", elem_id="llm-benchmark-tab-table3", id=0):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
                                 )
-                                filter_columns_size = gr.CheckboxGroup(
-                                    label="Model sizes (in billions of parameters)",
-                                    choices=list(NUMERIC_INTERVALS.keys()),
-                                    value=list(NUMERIC_INTERVALS.keys()),
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
                                     interactive=True,
-                                    elem_id="filter-columns-size",
                                 )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
 
-                        closed_ended_arabic_leaderboard_df, closed_ended_arabic_original_df = update_df(shown_columns.value, subset="closed-ended-arabic")
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="healthbench")
 
-                        leaderboard_table = gr.components.Dataframe(
-                            value=closed_ended_arabic_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                            headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                            datatype=TYPES,
-                            elem_id="leaderboard-table",
-                            interactive=False,
-                            visible=True,
-                        )
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
 
-                        # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                            value=closed_ended_arabic_original_df[CLOSED_ENDED_ARABIC_COLS],
-                            headers=CLOSED_ENDED_ARABIC_COLS,
-                            datatype=TYPES,
-                            visible=False,
-                        )
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[HEALTHBENCH_COLS],
+                        headers=HEALTHBENCH_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
 
-                                    
-                        search_bar.submit(
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
                             update_table,
                             [
                                 hidden_leaderboard_table_for_search,
@@ -929,33 +844,236 @@ with demo:
                                 filter_columns_type,
                                 filter_domain_specific,
                                 filter_columns_size
-                                # filter_columns_architecture
                             ],
                             leaderboard_table,
+                            queue=True,
                         )
-                        for selector in [
+                with gr.TabItem("HealthBench-Hard", elem_id="llm-benchmark-tab-table3", id=1):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_hard_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.healthbench_hard_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
+
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="healthbench_hard")
+
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[HEALTHBENCH_HARD_COLS],
+                        headers=HEALTHBENCH_HARD_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
                             shown_columns,
+                            search_bar,
                             filter_columns_type,
                             filter_domain_specific,
-                            # filter_columns_architecture,
-                            filter_columns_size,
-                            # deleted_models_visibility,
-                        ]:
-                            selector.change(
-                                update_table,
-                                [
-                                    hidden_leaderboard_table_for_search,
-                                    shown_columns,
-                                    search_bar,
-                                    filter_columns_type,
-                                    filter_domain_specific,
-                                    filter_columns_size
-                                    # filter_columns_architecture,
-                                ],
-                                leaderboard_table,
-                                queue=True,
-                            )
-        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-table", id=0):
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )        
+
+        with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=5):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain Specificity",
+                        choices=["🏥  Clinical models", "Generic models"],
+                        value=["🏥  Clinical models", "Generic models"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="med_safety")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[MED_SAFETY_COLS],
+                headers=MED_SAFETY_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+            with gr.Accordion("💬 Generation templates", open=False):
+                with gr.Accordion("Response generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="response_generation")
+                with gr.Accordion("Scoring Rubric", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="scoring_rubric")
+
+        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-table", id=6):
             with gr.Row():
                 with gr.Column():
                     with gr.Row():
@@ -1067,7 +1185,7 @@ with demo:
                     queue=True,
                 )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=5):
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=7):
             gr.Markdown(LLM_BENCHMARKS_TEXT_1, elem_classes="markdown-text")
             gr.HTML(FIVE_PILLAR_DIAGRAM)
             gr.Markdown(LLM_BENCHMARKS_TEXT_2, elem_classes="markdown-text")
@@ -1076,7 +1194,7 @@ with demo:
             # gr.HTML(ENTITY_DISTRIBUTION_IMG, elem_classes="logo")
             # gr.Markdown(LLM_BENCHMARKS_TEXT_3, elem_classes="markdown-text")
             
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=6):
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=8):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")

src/about.py

@@ -97,22 +97,50 @@ class SOAPColumns(Enum):
     # soap_column3 = SOAPColumn("brief", "score", "Conciseness") 
 
 @dataclass
-class ClosedEndedArabicColumn:
+class HealthbenchColumn:
     benchmark: str
     metric: str
     col_name: str
 
-class ClosedEndedArabicColumns(Enum):
-    arabictask0 = ClosedEndedArabicColumn("MMLU-Arabic", "accuracy", "MMLU-Arabic")
-    arabictask2 = ClosedEndedArabicColumn("MedMCQA-Arabic", "accuracy", "MedMCQA-Arabic")
-    arabictask3 = ClosedEndedArabicColumn("MedQA-Arabic", "accuracy", "MedQA-Arabic")
-    arabictask5 = ClosedEndedArabicColumn("PubMedQA-Arabic", "accuracy", "PubMedQA-Arabic")
+class HealthbenchColumns(Enum):
+    healthbench_column0 = HealthbenchColumn("Overall Score", "score", "Overall Score")
+    healthbench_column2 = HealthbenchColumn("Responding under uncertainty", "score", "Responding under uncertainty")
+    healthbench_column3 = HealthbenchColumn("Health data tasks", "score", "Health data tasks")
+    healthbench_column4 = HealthbenchColumn("Global health", "score", "Global health")
+    healthbench_column5 = HealthbenchColumn("Expertise-tailored communication", "score", "Expertise-tailored communication")
+    healthbench_column6 = HealthbenchColumn("Context seeking", "score", "Context seeking")
+    healthbench_column7 = HealthbenchColumn("Emergency referrals", "score", "Emergency referrals")
+    healthbench_column8 = HealthbenchColumn("Response depth", "score", "Response depth")
+    healthbench_column9 = HealthbenchColumn("Axis: Completeness", "score", "Axis: Completeness")
+    healthbench_column10 = HealthbenchColumn("Axis: Context awareness", "score", "Axis: Context awareness")
+    healthbench_column11 = HealthbenchColumn("Axis: Accuracy", "score", "Axis: Accuracy")
+    healthbench_column12 = HealthbenchColumn("Axis: Instruction following", "score", "Axis: Instruction following")
+    healthbench_column13 = HealthbenchColumn("Axis: Communication quality", "score", "Axis: Communication quality")
+    
+@dataclass  
+class HealthbenchHardColumn:
+    benchmark: str
+    metric: str
+    col_name: str
 
+class HealthbenchHardColumns(Enum):
+    healthbench_hard_column0 = HealthbenchHardColumn("Overall Score", "score", "Overall Score")
+    healthbench_hard_column2 = HealthbenchHardColumn("Responding under uncertainty", "score", "Responding under uncertainty")
+    healthbench_hard_column3 = HealthbenchHardColumn("Health data tasks", "score", "Health data tasks")
+    healthbench_hard_column4 = HealthbenchHardColumn("Global health", "score", "Global health")
+    healthbench_hard_column5 = HealthbenchHardColumn("Expertise-tailored communication", "score", "Expertise-tailored communication")
+    healthbench_hard_column6 = HealthbenchHardColumn("Context seeking", "score", "Context seeking")
+    healthbench_hard_column7 = HealthbenchHardColumn("Emergency referrals", "score", "Emergency referrals")
+    healthbench_hard_column8 = HealthbenchHardColumn("Response depth", "score", "Response depth")
+    healthbench_hard_column9 = HealthbenchHardColumn("Axis: Completeness", "score", "Axis: Completeness")
+    healthbench_hard_column10 = HealthbenchHardColumn("Axis: Context awareness", "score", "Axis: Context awareness")
+    healthbench_hard_column11 = HealthbenchHardColumn("Axis: Accuracy", "score", "Axis: Accuracy")
+    healthbench_hard_column12 = HealthbenchHardColumn("Axis: Instruction following", "score", "Axis: Instruction following")
+    healthbench_hard_column13 = HealthbenchHardColumn("Axis: Communication quality", "score", "Axis: Communication quality")
 
 NUM_FEWSHOT = 0  # Change with your few shot
 # ---------------------------------------------------
 
-
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title" style="color: red;"> [DEV Mode] </h1>"""
 # LOGO = """<img src="https://equalengineers.com/wp-content/uploads/2024/04/dummy-logo-5b.png" alt="Clinical X HF" width="500" height="333">"""
@@ -240,6 +268,10 @@ CROSS_EVALUATION_METRICS = """
 - **Overall Score**: The average of coverage, conformity, consistency, and the harmonic mean of coverage and conciseness (if both are positive, otherwise 0).
 """
 
+HEALTHBENCH_METRICS = """
+OpenAI HealthBench
+"""
+
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
 @misc{kanithi2024mediccomprehensiveframeworkevaluating,

src/display/utils.py

@@ -4,7 +4,7 @@ from enum import Enum
 import pandas as pd
 
 # changes to be made here
-from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
+from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns
 from src.envs import PRIVATE_REPO
 import json
 import gradio as gr
@@ -32,6 +32,8 @@ class ColumnContent:
     aci_col: bool = False
     soap_col: bool = False
     closed_ended_arabic_col: bool = False
+    healthbench_col: bool = False
+    healthbench_hard_col: bool = False
 
 
 ## Leaderboard columns
@@ -59,9 +61,18 @@ for column in ACIColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, aci_col=True, invariant=False)])
 for column in SOAPColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, soap_col=True, invariant=False)])
-# if PRIVATE_REPO:
-for column in ClosedEndedArabicColumns:
-    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, closed_ended_arabic_col=True, invariant=False)])
+
+for column in HealthbenchColumns:
+    if column.value.col_name.startswith("Axis"):
+        auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", False, False, healthbench_col=True, invariant=False)])
+    else:
+        auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, healthbench_col=True, invariant=False)])
+for column in HealthbenchHardColumns:
+    if column.value.col_name.startswith("Axis"):
+        auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", False, False, healthbench_hard_col=True, invariant=False)])
+    else:
+        auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, healthbench_hard_col=True, invariant=False)])
+
 auto_eval_column_dict.append(["is_domain_specific", ColumnContent, ColumnContent("Is Domain Specific", "bool", False)])
 auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Uses Chat Template", "bool", False)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -94,9 +105,7 @@ class EvalQueueColumn:  # Queue column
     med_safety_status = ColumnContent("med_safety_status", "str", True)
     medical_summarization_status = ColumnContent("medical_summarization_status", "str", True)
     note_generation_status = ColumnContent("note_generation_status", "str", True)
-    if PRIVATE_REPO:
-        closed_ended_arabic_status = ColumnContent("closed_ended_arabic_status", "str", True)
-
+    
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
@@ -221,13 +230,9 @@ MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c
 MEDICAL_SUMMARIZATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medical_summarization_col or c.invariant)]
 ACI_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.aci_col or c.invariant)]
 SOAP_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.soap_col or c.invariant)]
-# if PRIVATE_REPO:
-CLOSED_ENDED_ARABIC_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.closed_ended_arabic_col or c.invariant)]
-# CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.cross_examination_col or c.invariant)]
-# DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.cross_examination_col]
-# OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col and not c.med_safety_col and not c.cross_examination_col]
-# MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.dataset_task_col and not c.cross_examination_col]
-# CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.dataset_task_col]
+HEALTHBENCH_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.healthbench_col or c.invariant)]
+HEALTHBENCH_HARD_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.healthbench_hard_col or c.invariant)]
+
 
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
@@ -243,9 +248,8 @@ MED_SAFETY_BENCHMARK_COLS = [t.value.col_name for t in MedSafetyColumns]
 MEDICAL_SUMMARIZATION_BENCHMARK_COLS = [t.value.col_name for t in MedicalSummarizationColumns]
 ACI_BENCHMARK_COLS = [t.value.col_name for t in ACIColumns]
 SOAP_BENCHMARK_COLS = [t.value.col_name for t in SOAPColumns]
-# if PRIVATE_REPO:
-CLOSED_ENDED_ARABIC_BENCHMARK_COLS = [t.value.col_name for t in ClosedEndedArabicColumns]
-# CROSS_EXAMINATION_BENCHMARK_COLS = [t.value.col_name for t in CrossExaminationTasks]
+HEALTHBENCH_BENCHMARK_COLS = [t.value.col_name for t in HealthbenchColumns]
+HEALTHBENCH_HARD_BENCHMARK_COLS = [t.value.col_name for t in HealthbenchHardColumns]
 
 NUMERIC_INTERVALS = {
     "?": pd.Interval(-100, 0, closed="right"),

src/leaderboard/read_evals.py

@@ -9,11 +9,10 @@ import numpy as np
 
 from src.display.formatting import make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns
 from src.submission.check_validity import is_model_on_hub
 from src.envs import PRIVATE_REPO
 
-
 @dataclass
 class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
@@ -30,7 +29,8 @@ class EvalResult:
     medical_summarization_results: dict
     aci_results: dict
     soap_results: dict
-    closed_ended_arabic_results: dict
+    healthbench_results: dict
+    healthbench_hard_results: dict
     is_domain_specific: bool
     use_chat_template: bool
     # clinical_type_results:dict
@@ -167,39 +167,35 @@ class EvalResult:
                     continue
                 mean_acc = np.mean(accs)  # * 100.0
                 soap_results[task.benchmark] = mean_acc
-        closed_ended_arabic_results = {}
-        if PRIVATE_REPO and "closed-ended-arabic" in data["results"]:
-            for task in ClosedEndedArabicColumns:
+        
+        healthbench_results = {}
+        if "healthbench" in data["results"]:
+            for task in HealthbenchColumns:
                 task = task.value
-                # We average all scores of a given metric (not all metrics are present in all files)
-                try:
-                    accs = np.array([v.get(task.metric, None) for k, v in data["results"]["closed-ended-arabic"].items() if task.benchmark == k])
-                except:
-                    # breakpoint()
-                    accs = np.array([])
-                if accs.size == 0 or any([acc is None for acc in accs]):
-                    continue
-                mean_acc = np.mean(accs)  # * 100.0
-                closed_ended_arabic_results[task.benchmark] = mean_acc
-        # if open_ended_results == {} or med_safety_results == {} or medical_summarization_results == {} or aci_results == {} or soap_results == {}:
-        #     open_ended_results = {}
-        #     med_safety_results = {}
-        #     medical_summarization_results = {}
-        #     aci_results = {}
-        #     soap_results = {}
-        # types_results = {}
-        # for clinical_type in ClinicalTypes:
-        #     clinical_type = clinical_type.value
-
-        #     # We average all scores of a given metric (not all metrics are present in all files)
-        #     accs = np.array([v.get(clinical_type.metric, None) for k, v in data[evaluation_metric]["clinical_type_results"].items() if clinical_type.benchmark == k])
-        #     if accs.size == 0 or any([acc is None for acc in accs]):
-        #         continue
+                if task.benchmark == "Overall Score":
+                    accs = data["results"]["healthbench"][task.benchmark]
+                    healthbench_results[task.benchmark] = accs
+                elif task.benchmark.startswith("Axis"):
+                    accs = data["results"]["healthbench"]["Axis Scores"][task.benchmark.replace("Axis: ", "")]
+                    healthbench_results[task.benchmark] = accs
+                else:
+                    accs = data["results"]["healthbench"]["Theme Scores"][task.benchmark]
+                    healthbench_results[task.benchmark] = accs
+                
+        healthbench_hard_results = {}
+        if "healthbench-hard" in data["results"]:
+            for task in HealthbenchHardColumns:
+                task = task.value
+                if task.benchmark == "Overall Score":
+                    accs = data["results"]["healthbench-hard"][task.benchmark]
+                    healthbench_hard_results[task.benchmark] = accs
+                elif task.benchmark.startswith("Axis"):
+                    accs = data["results"]["healthbench-hard"]["Axis Scores"][task.benchmark.replace("Axis: ", "")]
+                    healthbench_hard_results[task.benchmark] = accs
+                else:
+                    accs = data["results"]["healthbench-hard"]["Theme Scores"][task.benchmark]
+                    healthbench_hard_results[task.benchmark] = accs
 
-        #     mean_acc = np.mean(accs)  # * 100.0
-        #     types_results[clinical_type.benchmark] = mean_acc
-        # if "deepseek-ai/DeepSeek-R1-Distill-Llama-70B" in json_filepath:
-        #     breakpoint()
         return self(
             eval_name=result_key,
             full_model=full_model,
@@ -212,7 +208,8 @@ class EvalResult:
             medical_summarization_results=medical_summarization_results,
             aci_results=aci_results,
             soap_results=soap_results,
-            closed_ended_arabic_results=closed_ended_arabic_results,
+            healthbench_results=healthbench_results,
+            healthbench_hard_results=healthbench_hard_results,
             is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
             use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,
@@ -315,12 +312,15 @@ class EvalResult:
                 for task in SOAPColumns:
                     data_dict[task.value.col_name] = self.soap_results[task.value.benchmark]
             return data_dict
-        if PRIVATE_REPO and subset == "closed_ended_arabic":
-            average = sum([v for v in self.closed_ended_arabic_results.values() if v is not None]) / len(ClosedEndedArabicColumns)
-            data_dict[AutoEvalColumn.average.name] = average
-            if len(self.closed_ended_arabic_results) > 0:
-                for task in ClosedEndedArabicColumns:
-                    data_dict[task.value.col_name] = self.closed_ended_arabic_results[task.value.benchmark]
+        if subset == "healthbench":
+            if len(self.healthbench_results) > 0:
+                for task in HealthbenchColumns:
+                    data_dict[task.value.col_name] = self.healthbench_results[task.value.benchmark]
+            return data_dict
+        if subset == "healthbench_hard":
+            if len(self.healthbench_hard_results) > 0:
+                for task in HealthbenchHardColumns:
+                    data_dict[task.value.col_name] = self.healthbench_hard_results[task.value.benchmark]
             return data_dict
 
 def get_request_file_for_model(requests_path, model_name, precision):

src/populate.py

@@ -5,11 +5,10 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, HealthbenchColumns, HealthbenchHardColumns
 from src.leaderboard.read_evals import get_raw_eval_results
 from src.envs import PRIVATE_REPO
 
-
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, evaluation_metric:str, subset:str) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data =  get_raw_eval_results(results_path, requests_path, evaluation_metric)
@@ -33,6 +32,10 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         df = df.sort_values(by=[AutoEvalColumn.overall.name], ascending=False)
     elif subset == "closed_ended_arabic":
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    elif subset == "healthbench":
+        df = df.sort_values(by=["Overall Score"], ascending=False)
+    elif subset == "healthbench_hard":
+        df = df.sort_values(by=["Overall Score"], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)
     # filter out if any of the benchmarks have not been produced