Update space

.gitignore

@@ -6,8 +6,6 @@ __pycache__/
 *ipynb
 .vscode/
 
-eval-queue/
-eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/

app.py

@@ -27,29 +27,33 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
-
+import pdb
+import os
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 ### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
+# try:
+#     print("EVAL_REQUESTS_PATH:",EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+# try:
+#     print("EVAL_RESULTS_PATH:",EVAL_RESULTS_PATH)
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+
+# eval_results_path = os.path.join(EVAL_RESULTS_PATH, "svbench")
+# eval_requests_path = os.path.join(EVAL_REQUESTS_PATH, "svbench")
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+print("EVAL_RESULTS_PATH:",EVAL_RESULTS_PATH)
+# print("LEADERBOARD_DF:",LEADERBOARD_DF)
 
 (
     finished_eval_queue_df,
@@ -60,6 +64,12 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
+    
+    # Check for None in filter_columns
+    filter_columns = [
+        ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+    ]
+    
     return Leaderboard(
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
@@ -68,27 +78,26 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.model.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
+            # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
+            # ColumnFilter(
+            #     AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+            # ),
         ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
 
-
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)

eval-queue/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-queue/svbench/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-queue/svbench/Flash-VStream.json

@@ -0,0 +1,14 @@
+{
+    "model": "Flash-VStream",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/GPT-4V.json

@@ -0,0 +1,14 @@
+{
+    "model": "GPT-4V",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "",
+    "likes": 0,
+    "params": 0,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/GPT-4o.json

@@ -0,0 +1,14 @@
+{
+    "model": "GPT-4o",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "",
+    "likes": 0,
+    "params": 0,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/Gemini 1.5 Pro.json

@@ -0,0 +1,14 @@
+{
+    "model": "Gemini 1.5 Pro",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "",
+    "likes": 0,
+    "params": 0,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/InternLM-XC2.5.json

@@ -0,0 +1,14 @@
+{
+    "model": "InternLM-XC2.5",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/InternVL2.json

@@ -0,0 +1,14 @@
+{
+    "model": "InternVL2",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "ImageLLM",
+    "likes": 0,
+    "params": 8,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/LLaVA-NeXT-Video.json

@@ -0,0 +1,14 @@
+{
+    "model": "LLaVA-NeXT-Video",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/MiniCPM-V 2.6.json

@@ -0,0 +1,14 @@
+{
+    "model": "MiniCPM-V 2.6",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "ImageLLM",
+    "likes": 0,
+    "params": 8,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/MovieChat.json

@@ -0,0 +1,14 @@
+{
+    "model": "MovieChat",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/Qwen2-VL.json

@@ -0,0 +1,14 @@
+{
+    "model": "Qwen2-VL",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "ImageLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/ShareGPT4Video.json

@@ -0,0 +1,14 @@
+{
+    "model": "ShareGPT4Video",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 8,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/TimeChat.json

@@ -0,0 +1,14 @@
+{
+    "model": "TimeChat",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/VILA.json

@@ -0,0 +1,14 @@
+{
+    "model": "VILA",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "ImageLLM",
+    "likes": 0,
+    "params": 8,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/Video-ChatGPT.json

@@ -0,0 +1,14 @@
+{
+    "model": "Video-ChatGPT",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/Video-LLaVA.json

@@ -0,0 +1,14 @@
+{
+    "model": "Video-LLaVA",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-queue/svbench/VideoLLaMA2.json

@@ -0,0 +1,14 @@
+{
+    "model": "VideoLLaMA2",
+    "base_model": "",
+    "revision": "float16",
+    "precision": "",
+    "weight_type": "",
+    "status": "FINISHED",
+    "submitted_time": "",
+    "model_type": "VideoLLM",
+    "likes": 0,
+    "params": 7,
+    "license": "",
+    "private": false
+}

eval-results/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-results/svbench/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-results/svbench/Flash-VStream/results_Flash-VStream.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "Flash-VStream",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.3754
+        },
+        "Dialogue_CC": {
+            "acc": 0.4474
+        },
+        "Dialogue_LC": {
+            "acc": 0.5102
+        },
+        "Dialogue_TU": {
+            "acc": 0.4795
+        },
+        "Dialogue_IC": {
+            "acc": 0.3794
+        },
+        "Dialogue_OS": {
+            "acc": 0.4272
+        },
+        "Streaming_SA": {
+            "acc": 0.3571
+        },
+        "Streaming_CC": {
+            "acc": 0.4424
+        },
+        "Streaming_LC": {
+            "acc": 0.4849
+        },
+        "Streaming_TU": {
+            "acc": 0.3895
+        },
+        "Streaming_IC": {
+            "acc": 0.3900
+        },
+        "Streaming_OS": {
+            "acc": 0.3880
+        }
+    }
+}

eval-results/svbench/GPT-4V/results_GPT-4V.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "GPT-4V",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.5603
+        },
+        "Dialogue_CC": {
+            "acc": 0.6261
+        },
+        "Dialogue_LC": {
+            "acc": 0.6909
+        },
+        "Dialogue_TU": {
+            "acc": 0.6536
+        },
+        "Dialogue_IC": {
+            "acc": 0.5373
+        },
+        "Dialogue_OS": {
+            "acc": 0.6030
+        },
+        "Streaming_SA": {
+            "acc": 0.5637
+        },
+        "Streaming_CC": {
+            "acc": 0.6141
+        },
+        "Streaming_LC": {
+            "acc": 0.6580
+        },
+        "Streaming_TU": {
+            "acc": 0.5918
+        },
+        "Streaming_IC": {
+            "acc": 0.5716
+        },
+        "Streaming_OS": {
+            "acc": 0.5793
+        }
+    }
+}

eval-results/svbench/GPT-4o/results_GPT-4o.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "GPT-4o",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.5826
+        },
+        "Dialogue_CC": {
+            "acc": 0.6476
+        },
+        "Dialogue_LC": {
+            "acc": 0.7075
+        },
+        "Dialogue_TU": {
+            "acc": 0.6768
+        },
+        "Dialogue_IC": {
+            "acc": 0.5582
+        },
+        "Dialogue_OS": {
+            "acc": 0.6257
+        },
+        "Streaming_SA": {
+            "acc": 0.5799
+        },
+        "Streaming_CC": {
+            "acc": 0.6352
+        },
+        "Streaming_LC": {
+            "acc": 0.6772
+        },
+        "Streaming_TU": {
+            "acc": 0.6018
+        },
+        "Streaming_IC": {
+            "acc": 0.5925
+        },
+        "Streaming_OS": {
+            "acc": 0.5997
+        }
+    }
+}

eval-results/svbench/Gemini 1.5 Pro/results_Gemini 1.5 Pro.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "Gemini 1.5 Pro",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.4907
+        },
+        "Dialogue_CC": {
+            "acc": 0.5615
+        },
+        "Dialogue_LC": {
+            "acc": 0.6224
+        },
+        "Dialogue_TU": {
+            "acc": 0.5836
+        },
+        "Dialogue_IC": {
+            "acc": 0.4772
+        },
+        "Dialogue_OS": {
+            "acc": 0.5368
+        },
+        "Streaming_SA": {
+            "acc": 0.4935
+        },
+        "Streaming_CC": {
+            "acc": 0.5577
+        },
+        "Streaming_LC": {
+            "acc": 0.6041
+        },
+        "Streaming_TU": {
+            "acc": 0.5289
+        },
+        "Streaming_IC": {
+            "acc": 0.5111
+        },
+        "Streaming_OS": {
+            "acc": 0.5155
+        }
+    }
+}

eval-results/svbench/InternLM-XC2.5/results_InternLM-XC2.5.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "InternLM-XC2.5",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.4651
+        },
+        "Dialogue_CC": {
+            "acc": 0.5316
+        },
+        "Dialogue_LC": {
+            "acc": 0.5984
+        },
+        "Dialogue_TU": {
+            "acc": 0.5294
+        },
+        "Dialogue_IC": {
+            "acc": 0.4587
+        },
+        "Dialogue_OS": {
+            "acc": 0.5071
+        },
+        "Streaming_SA": {
+            "acc": 0.5262
+        },
+        "Streaming_CC": {
+            "acc": 0.5855
+        },
+        "Streaming_LC": {
+            "acc": 0.6289
+        },
+        "Streaming_TU": {
+            "acc": 0.5398
+        },
+        "Streaming_IC": {
+            "acc": 0.5439
+        },
+        "Streaming_OS": {
+            "acc": 0.5439
+        }
+    }
+}

eval-results/svbench/InternVL2/results_InternVL2.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "InternVL2",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.4053
+        },
+        "Dialogue_CC": {
+            "acc": 0.4677
+        },
+        "Dialogue_LC": {
+            "acc": 0.5238
+        },
+        "Dialogue_TU": {
+            "acc": 0.4697
+        },
+        "Dialogue_IC": {
+            "acc": 0.4035
+        },
+        "Dialogue_OS": {
+            "acc": 0.4448
+        },
+        "Streaming_SA": {
+            "acc": 0.3892
+        },
+        "Streaming_CC": {
+            "acc": 0.4542
+        },
+        "Streaming_LC": {
+            "acc": 0.5045
+        },
+        "Streaming_TU": {
+            "acc": 0.4153
+        },
+        "Streaming_IC": {
+            "acc": 0.4235
+        },
+        "Streaming_OS": {
+            "acc": 0.4162
+        }
+    }
+}

eval-results/svbench/LLaVA-NeXT-Video/results_LLaVA-NeXT-Video.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "LLaVA-NeXT-Video",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.3771
+        },
+        "Dialogue_CC": {
+            "acc": 0.4459
+        },
+        "Dialogue_LC": {
+            "acc": 0.5205
+        },
+        "Dialogue_TU": {
+            "acc": 0.4180
+        },
+        "Dialogue_IC": {
+            "acc": 0.3658
+        },
+        "Dialogue_OS": {
+            "acc": 0.4140
+        },
+        "Streaming_SA": {
+            "acc": 0.3429
+        },
+        "Streaming_CC": {
+            "acc": 0.3968
+        },
+        "Streaming_LC": {
+            "acc": 0.4765
+        },
+        "Streaming_TU": {
+            "acc": 0.3533
+        },
+        "Streaming_IC": {
+            "acc": 0.3668
+        },
+        "Streaming_OS": {
+            "acc": 0.3612
+        }
+    }
+}

eval-results/svbench/MiniCPM-V 2.6/results_MiniCPM-V 2.6.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "MiniCPM-V 2.6",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.5170
+        },
+        "Dialogue_CC": {
+            "acc": 0.5950
+        },
+        "Dialogue_LC": {
+            "acc": 0.6533
+        },
+        "Dialogue_TU": {
+            "acc": 0.6172
+        },
+        "Dialogue_IC": {
+            "acc": 0.5009
+        },
+        "Dialogue_OS": {
+            "acc": 0.5663
+        },
+        "Streaming_SA": {
+            "acc": 0.4644
+        },
+        "Streaming_CC": {
+            "acc": 0.5273
+        },
+        "Streaming_LC": {
+            "acc": 0.5835
+        },
+        "Streaming_TU": {
+            "acc": 0.5348
+        },
+        "Streaming_IC": {
+            "acc": 0.4832
+        },
+        "Streaming_OS": {
+            "acc": 0.4967
+        }
+    }
+}

eval-results/svbench/MovieChat/results_MovieChat.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "MovieChat",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.2036
+        },
+        "Dialogue_CC": {
+            "acc": 0.2374
+        },
+        "Dialogue_LC": {
+            "acc": 0.2897
+        },
+        "Dialogue_TU": {
+            "acc": 0.228
+        },
+        "Dialogue_IC": {
+            "acc": 0.2051
+        },
+        "Dialogue_OS": {
+            "acc": 0.2272
+        },
+        "Streaming_SA": {
+            "acc": 0.1892
+        },
+        "Streaming_CC": {
+            "acc": 0.2238
+        },
+        "Streaming_LC": {
+            "acc": 0.2677
+        },
+        "Streaming_TU": {
+            "acc": 0.2046
+        },
+        "Streaming_IC": {
+            "acc": 0.2098
+        },
+        "Streaming_OS": {
+            "acc": 0.1964
+        }
+    }
+}

eval-results/svbench/Qwen2-VL/results_Qwen2-VL.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "Qwen2-VL",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.5047
+        },
+        "Dialogue_CC": {
+            "acc": 0.5771
+        },
+        "Dialogue_LC": {
+            "acc": 0.6346
+        },
+        "Dialogue_TU": {
+            "acc": 0.6077
+        },
+        "Dialogue_IC": {
+            "acc": 0.4944
+        },
+        "Dialogue_OS": {
+            "acc": 0.5529
+        },
+        "Streaming_SA": {
+            "acc": 0.4838
+        },
+        "Streaming_CC": {
+            "acc": 0.5517
+        },
+        "Streaming_LC": {
+            "acc": 0.5991
+        },
+        "Streaming_TU": {
+            "acc": 0.5204
+        },
+        "Streaming_IC": {
+            "acc": 0.5142
+        },
+        "Streaming_OS": {
+            "acc": 0.5139
+        }
+    }
+}

eval-results/svbench/ShareGPT4Video/results_ShareGPT4Video.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "ShareGPT4Video",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.3626
+        },
+        "Dialogue_CC": {
+            "acc": 0.4368
+        },
+        "Dialogue_LC": {
+            "acc": 0.5012
+        },
+        "Dialogue_TU": {
+            "acc": 0.4733
+        },
+        "Dialogue_IC": {
+            "acc": 0.3725
+        },
+        "Dialogue_OS": {
+            "acc": 0.4176
+        },
+        "Streaming_SA": {
+            "acc": 0.3314
+        },
+        "Streaming_CC": {
+            "acc": 0.4048
+        },
+        "Streaming_LC": {
+            "acc": 0.4601
+        },
+        "Streaming_TU": {
+            "acc": 0.3815
+        },
+        "Streaming_IC": {
+            "acc": 0.3781
+        },
+        "Streaming_OS": {
+            "acc": 0.3710
+        }
+    }
+}

eval-results/svbench/TimeChat/results_TimeChat.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "TimeChat",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.3109
+        },
+        "Dialogue_CC": {
+            "acc": 0.3857
+        },
+        "Dialogue_LC": {
+            "acc": 0.4552
+        },
+        "Dialogue_TU": {
+            "acc": 0.4337
+        },
+        "Dialogue_IC": {
+            "acc": 0.3110
+        },
+        "Dialogue_OS": {
+            "acc": 0.3624
+        },
+        "Streaming_SA": {
+            "acc": 0.2714
+        },
+        "Streaming_CC": {
+            "acc": 0.3442
+        },
+        "Streaming_LC": {
+            "acc": 0.3978
+        },
+        "Streaming_TU": {
+            "acc": 0.3680
+        },
+        "Streaming_IC": {
+            "acc": 0.3171
+        },
+        "Streaming_OS": {
+            "acc": 0.3115
+        }
+    }
+}

eval-results/svbench/VILA/results_VILA.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "VILA",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.4323
+        },
+        "Dialogue_CC": {
+            "acc": 0.4930
+        },
+        "Dialogue_LC": {
+            "acc": 0.5559
+        },
+        "Dialogue_TU": {
+            "acc": 0.5247
+        },
+        "Dialogue_IC": {
+            "acc": 0.4127
+        },
+        "Dialogue_OS": {
+            "acc": 0.4707
+        },
+        "Streaming_SA": {
+            "acc": 0.3819
+        },
+        "Streaming_CC": {
+            "acc": 0.4427
+        },
+        "Streaming_LC": {
+            "acc": 0.4918
+        },
+        "Streaming_TU": {
+            "acc": 0.4129
+        },
+        "Streaming_IC": {
+            "acc": 0.4055
+        },
+        "Streaming_OS": {
+            "acc": 0.4038
+        }
+    }
+}

eval-results/svbench/Video-ChatGPT/results_Video-ChatGPT.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "Video-ChatGPT",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.2801
+        },
+        "Dialogue_CC": {
+            "acc": 0.3404
+        },
+        "Dialogue_LC": {
+            "acc": 0.4089
+        },
+        "Dialogue_TU": {
+            "acc": 0.3566
+        },
+        "Dialogue_IC": {
+            "acc": 0.2959
+        },
+        "Dialogue_OS": {
+            "acc": 0.3224
+        },
+        "Streaming_SA": {
+            "acc": 0.2284
+        },
+        "Streaming_CC": {
+            "acc": 0.2844
+        },
+        "Streaming_LC": {
+            "acc": 0.3393
+        },
+        "Streaming_TU": {
+            "acc": 0.2631
+        },
+        "Streaming_IC": {
+            "acc": 0.2643
+        },
+        "Streaming_OS": {
+            "acc": 0.2502
+        }
+    }
+}

eval-results/svbench/Video-LLaVA/results_Video-LLaVA.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "Video-LLaVA",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.3185
+        },
+        "Dialogue_CC": {
+            "acc": 0.3838
+        },
+        "Dialogue_LC": {
+            "acc": 0.4493
+        },
+        "Dialogue_TU": {
+            "acc": 0.4154
+        },
+        "Dialogue_IC": {
+            "acc": 0.3280
+        },
+        "Dialogue_OS": {
+            "acc": 0.3649
+        },
+        "Streaming_SA": {
+            "acc": 0.2695
+        },
+        "Streaming_CC": {
+            "acc": 0.3368
+        },
+        "Streaming_LC": {
+            "acc": 0.3900
+        },
+        "Streaming_TU": {
+            "acc": 0.3183
+        },
+        "Streaming_IC": {
+            "acc": 0.3153
+        },
+        "Streaming_OS": {
+            "acc": 0.2989
+        }
+    }
+}

eval-results/svbench/VideoLLaMA2/results_VideoLLaMA2.json

@@ -0,0 +1,45 @@
+{
+    "config": {
+        "model_dtype": "torch.float16", 
+        "model_name": "VideoLLaMA2",
+        "model_sha": ""
+    },
+    "results": {
+        "Dialogue_SA": {
+            "acc": 0.4250
+        },
+        "Dialogue_CC": {
+            "acc": 0.4988
+        },
+        "Dialogue_LC": {
+            "acc": 0.5596
+        },
+        "Dialogue_TU": {
+            "acc": 0.5223
+        },
+        "Dialogue_IC": {
+            "acc": 0.4140
+        },
+        "Dialogue_OS": {
+            "acc": 0.4710
+        },
+        "Streaming_SA": {
+            "acc": 0.3895
+        },
+        "Streaming_CC": {
+            "acc": 0.4611
+        },
+        "Streaming_LC": {
+            "acc": 0.5177
+        },
+        "Streaming_TU": {
+            "acc": 0.4369
+        },
+        "Streaming_IC": {
+            "acc": 0.4222
+        },
+        "Streaming_OS": {
+            "acc": 0.4277
+        }
+    }
+}

src/about.py

@@ -12,8 +12,18 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("Dialogue_SA", "acc", "Dialogue_SA")
+    task1 = Task("Dialogue_CC", "acc", "Dialogue_CC")
+    task2 = Task("Dialogue_LC", "acc", "Dialogue_LC")
+    task3 = Task("Dialogue_TU", "acc", "Dialogue_TU")
+    task4 = Task("Dialogue_IC", "acc", "Dialogue_IC")
+    task5 = Task("Dialogue_OS", "acc", "Dialogue_OS")
+    task6 = Task("Streaming_SA", "acc", "Streaming_SA")
+    task7 = Task("Streaming_CC", "acc", "Streaming_CC")
+    task8 = Task("Streaming_LC", "acc", "Streaming_LC")
+    task9 = Task("Streaming_TU", "acc", "Streaming_TU")
+    task10 = Task("Streaming_IC", "acc", "Streaming_IC")
+    task11 = Task("Streaming_OS", "acc", "Streaming_OS")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -25,7 +35,9 @@ TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+SVBench is a benchmark specifically designed to evaluate the performance of Large Vision-Language Models (LVLMs) in long-context streaming video understanding tasks. This benchmark comprehensively assesses the models' capabilities in handling streaming videos through its unique temporal multi-turn question-answering chains. To facilitate research and development, SVBench provides a detailed leaderboard showcasing the performance results of over a dozen models on this benchmark. By ranking the models based on their performance on SVBench, users can quickly identify models that excel in specific tasks, thereby guiding subsequent research and applications.
+Detailed information about SVBench and the leaderboard can be accessed via the following link: [SVBench Benchmark](https://yzy-bupt.github.io/SVBench). The paper is available at: [SVBench Paper](https://arxiv.org/abs/2502.10810). Leaderboard submissions can be made through the following link: [Leaderboard Submission](https://docs.google.com/forms/d/e/1FAIpQLSfz62pGaIdKjmDbOP0vw74dXSiG-2ILJI7gdugdx4pfWSc42Q/viewform). Additionally, the related dataset is hosted on the Hugging Face platform, and researchers can access it at [SVBench Dataset](https://huggingface.co/datasets/yzy666/SVBench) for further experiments and model development.
+This leaderboard not only provides a fair competitive environment for current models but also serves as an important reference standard for future model improvements and innovations.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
@@ -69,4 +81,10 @@ If everything is done, check you can launch the EleutherAIHarness on your model
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+@article{yang2025svbench,
+  title={SVBench: A Benchmark with Temporal Multi-Turn Dialogues for Streaming Video Understanding},
+  author={Yang, Zhenyu and Hu, Yuhang and Du, Zemin and Xue, Dizhan and Qian, Shengsheng and Wu, Jiahong and Yang, Fan and Dong, Weiming and Xu, Changsheng},
+  journal={arXiv preprint arXiv:2502.10810},
+  year={2025}
+}
 """

src/display/utils.py

@@ -23,7 +23,7 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
@@ -31,14 +31,14 @@ for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -47,11 +47,11 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
+    # revision = ColumnContent("revision", "str", True)
+    # private = ColumnContent("private", "bool", True)
+    # precision = ColumnContent("precision", "str", True)
+    # weight_type = ColumnContent("weight_type", "str", "Original")
+    # status = ColumnContent("status", "str", True)
 
 ## All the model information that we might need
 @dataclass
@@ -60,12 +60,9 @@ class ModelDetails:
     display_name: str = ""
     symbol: str = "" # emoji
 
-
 class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    VideoLLM = ModelDetails(name="VideoLLM", symbol="🎥")
+    ImageLLM = ModelDetails(name="ImageLLM", symbol="🖼️")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -73,16 +70,34 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
+        if "VideoLLM" in type or "🎥" in type:
+            return ModelType.VideoLLM
+        if "ImageLLM" in type or "🖼️" in type:
+            return ModelType.ImageLLM
         return ModelType.Unknown
 
+# class ModelType(Enum):
+#     PT = ModelDetails(name="pretrained", symbol="🟢")
+#     FT = ModelDetails(name="fine-tuned", symbol="🔶")
+#     IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+#     RL = ModelDetails(name="RL-tuned", symbol="🟦")
+#     Unknown = ModelDetails(name="", symbol="?")
+
+#     def to_str(self, separator=" "):
+#         return f"{self.value.symbol}{separator}{self.value.name}"
+
+#     @staticmethod
+#     def from_str(type):
+#         if "fine-tuned" in type or "🔶" in type:
+#             return ModelType.FT
+#         if "pretrained" in type or "🟢" in type:
+#             return ModelType.PT
+#         if "RL-tuned" in type or "🟦" in type:
+#             return ModelType.RL
+#         if "instruction-tuned" in type or "⭕" in type:
+#             return ModelType.IFT
+#         return ModelType.Unknown
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")

src/envs.py

@@ -17,8 +17,8 @@ RESULTS_REPO = f"{OWNER}/results"
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue/svbench")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results/svbench")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 

src/leaderboard/read_evals.py

@@ -48,7 +48,7 @@ class EvalResult:
         org_and_model = org_and_model.split("/", 1)
 
         if len(org_and_model) == 1:
-            org = None
+            org = "svbench"
             model = org_and_model[0]
             result_key = f"{model}_{precision.value.name}"
         else:
@@ -93,37 +93,42 @@ class EvalResult:
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
+        request_file = get_request_file_for_model(requests_path, self.model, self.precision.value.name)
+        print("requests_path:",requests_path)
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
+            print(f"Request file content: {request}")  # 调试输出
             self.model_type = ModelType.from_str(request.get("model_type", ""))
             self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
+            self.num_params = request.get("params", "")
             self.date = request.get("submitted_time", "")
-        except Exception:
+        except FileNotFoundError:
             print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+        except json.JSONDecodeError:
+            print(f"Error decoding JSON from request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+        except Exception as e:
+            print(f"An unexpected error occurred: {e}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
+            # AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
+            # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            # AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
+            # AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
+            # AutoEvalColumn.license.name: self.license,
+            # AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
         for task in Tasks:
@@ -134,11 +139,9 @@ class EvalResult:
 
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
+    request_files_pattern = os.path.join(requests_path, f"{model_name}.json")
+    print(f"Looking for request files with pattern: {request_files_pattern}")  # 调试输出
+    request_files = glob.glob(request_files_pattern)
 
     # Select correct request file (precision)
     request_file = ""
@@ -146,6 +149,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
+            print(f"Checking request file: {tmp_request_file}, Content: {req_content}")  # 调试输出
             if (
                 req_content["status"] in ["FINISHED"]
                 and req_content["precision"] == precision.split(".")[-1]
@@ -164,10 +168,10 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             continue
 
         # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
+        # try:
+        #     files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        # except dateutil.parser._parser.ParserError:
+        #     files = [files[-1]]
 
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
@@ -177,7 +181,6 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
         eval_result.update_with_request_file(requests_path)
-
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():

src/populate.py

@@ -1,6 +1,6 @@
 import json
 import os
-
+import pprint
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
@@ -34,7 +34,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data = json.load(fp)
 
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            # data[EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
         elif ".md" not in entry: