Upload 14 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -7,7 +7,7 @@ import tempfile
 import re
 from constants import *
 from src.auto_leaderboard.model_metadata_type import ModelType
-
+import dask.dataframe as dd
 
 global data_component, filter_component
 
@@ -26,15 +26,12 @@ def prediction_analyse(prediction_content):
     # pdb.set_trace()
     predictions = prediction_content.split("\n")
 
-    # 读取 ground_truth JSON 文件
-    with open("./file/SEED-Bench-1.json", "r") as file:
-        ground_truth_data = json.load(file)["questions"]
-
-    # 将 ground_truth 数据转换为以 question_id 为键的字典
-    ground_truth = {item["question_id"]: item for item in ground_truth_data}
+    # 读取 ground_truth 文件
+    df = dd.read_parquet("./file/av_odyssey.parquet")
+    ground_truth = {row[0]: row[6] for row in df.itertuples(index=False, name=None)}
 
     # 初始化结果统计字典
-    results = {i: {"correct": 0, "total": 0} for i in range(1, 13)}
+    results = {i: {"correct": 0, "total": 0} for i in range(1, 27)}
 
     # 遍历 predictions，计算每个 question_type_id 的正确预测数和总预测数
     for prediction in predictions:
@@ -48,15 +45,15 @@ def prediction_analyse(prediction_content):
             print(f"Warning: Skipping invalid JSON data in line: {prediction}")
             continue
         question_id = prediction["question_id"]
-        if question_id not in ground_truth:
+        if question_id not in ground_truth.keys():
             continue
         gt_item = ground_truth[question_id]
-        question_type_id = gt_item["question_type_id"]
+        question_type_id = question_id.split("_")[0]
 
-        if prediction["prediction"] == gt_item["answer"]:
-            results[question_type_id]["correct"] += 1
+        if prediction["prediction"] == gt_item:
+            results[int(question_type_id)]["correct"] += 1
 
-        results[question_type_id]["total"] += 1
+        results[int(question_type_id)]["total"] += 1
     
     return results
 
@@ -70,45 +67,23 @@ def add_new_eval(
     if input_file is None:
         return "Error! Empty file!"
     else:
-        model_size = validate_model_size(model_size)
         # v1 evaluation
         content = input_file.decode("utf-8")
         prediction = prediction_analyse(content)
         csv_data = pd.read_csv(CSV_DIR)
+        # pdb.set_trace()
 
-        Start_dimension, End_dimension = 1, 13
-        if Evaluation_dimension == 'Image':
-            End_dimension = 10
-        elif Evaluation_dimension == 'Video':
-            Start_dimension = 10
-        each_task_accuracy = {i: round(prediction[i]["correct"] / prediction[i]["total"] * 100, 1) if i >= Start_dimension and i < End_dimension else 0 for i in range(1, 13)}
+        each_task_accuracy = {i: round(prediction[i]["correct"] / prediction[i]["total"] * 100, 1) for i in range(1, 27)}
 
         # count for average image\video\all
-        total_correct_image = sum(prediction[i]["correct"] for i in range(1, 10))
-        total_correct_video = sum(prediction[i]["correct"] for i in range(10, 13))
-
-        total_image = sum(prediction[i]["total"] for i in range(1, 10))
-        total_video = sum(prediction[i]["total"] for i in range(10, 13))
-
-        if Evaluation_dimension != 'Video':
-            average_accuracy_image = round(total_correct_image / total_image * 100, 1)
-        else:
-            average_accuracy_image = 0
-        
-        if Evaluation_dimension != 'Image':
-            average_accuracy_video = round(total_correct_video / total_video * 100, 1)
-        else:
-            average_accuracy_video = 0
-        
-        if Evaluation_dimension == 'All':
-            overall_accuracy = round((total_correct_image + total_correct_video) / (total_image + total_video) * 100, 1)
-        else:
-            overall_accuracy = 0
-
-        if LLM_type == 'Other':
-            LLM_name = LLM_name_textbox
-        else:
-            LLM_name = LLM_type
+        total_correct_timbre = round(sum(prediction[i]["correct"] for i in range(timbre_task[0], timbre_task[1] + 1)) / sum(prediction[i]["total"] for i in range(timbre_task[0], timbre_task[1] + 1)) * 100, 1)
+        total_correct_tone = round(sum(prediction[i]["correct"] for i in range(tone_task[0], tone_task[1] + 1)) / sum(prediction[i]["total"] for i in range(tone_task[0], tone_task[1] + 1)) * 100, 1)
+        total_correct_melody = round(sum(prediction[i]["correct"] for i in range(melody_task[0], melody_task[1] + 1)) / sum(prediction[i]["total"] for i in range(melody_task[0], melody_task[1] + 1)) * 100, 1)
+        total_correct_space = round(sum(prediction[i]["correct"] for i in range(space_task[0], space_task[1] + 1)) / sum(prediction[i]["total"] for i in range(space_task[0], space_task[1] + 1)) * 100, 1)
+        total_correct_time = round(sum(prediction[i]["correct"] for i in range(time_task[0], time_task[1] + 1)) / sum(prediction[i]["total"] for i in range(time_task[0], time_task[1] + 1)) * 100, 1)
+        total_correct_hallucination = round(sum(prediction[i]["correct"] for i in range(hallucination_task[0], hallucination_task[1] + 1)) / sum(prediction[i]["total"] for i in range(hallucination_task[0], hallucination_task[1] + 1)) * 100, 1)
+        total_correct_intricay = round(sum(prediction[i]["correct"] for i in range(intricay_task[0], intricay_task[1] + 1)) / sum(prediction[i]["total"] for i in range(intricay_task[0], intricay_task[1] + 1)) * 100, 1)
+        all_average = round(sum(prediction[i]["correct"] for i in range(1, 27)) / sum(prediction[i]["total"] for i in range(1, 27)) * 100, 1)
         
         if revision_name_textbox == '':
             col = csv_data.shape[0]
@@ -130,11 +105,14 @@ def add_new_eval(
         # add new data
         new_data = [
             model_name, 
-            LLM_name,
-            model_size,
-            overall_accuracy,
-            average_accuracy_image,
-            average_accuracy_video,
+            all_average,
+            total_correct_timbre, 
+            total_correct_tone, 
+            total_correct_melody, 
+            total_correct_space, 
+            total_correct_time, 
+            total_correct_hallucination, 
+            total_correct_intricay, 
             each_task_accuracy[1],
             each_task_accuracy[2],
             each_task_accuracy[3],
@@ -146,13 +124,25 @@ def add_new_eval(
             each_task_accuracy[9],
             each_task_accuracy[10],
             each_task_accuracy[11],
-            each_task_accuracy[12], 
+            each_task_accuracy[12],  
+            each_task_accuracy[13],
+            each_task_accuracy[14],
+            each_task_accuracy[15],
+            each_task_accuracy[16],
+            each_task_accuracy[17],
+            each_task_accuracy[18],
+            each_task_accuracy[19],
+            each_task_accuracy[20],
+            each_task_accuracy[21],
+            each_task_accuracy[22], 
+            each_task_accuracy[23],
+            each_task_accuracy[24],
+            each_task_accuracy[25], 
+            each_task_accuracy[26],
             ]
         csv_data.loc[col] = new_data
         csv_data = csv_data.to_csv(CSV_DIR, index=False)
 
-        csv_task_data.loc[col] = new_data
-        csv_task_data = csv_task_data.to_csv(CSV_TASK_DIR, index=False)
     return 0
 
 def get_baseline_df():

constants.py

@@ -13,6 +13,14 @@ AVG_INFO = ["Avg. All", "Avg. Timbre", "Avg. Tone", "Avg. Melody", "Avg. Space",
 DATA_TITILE_TYPE = ["markdown"] * len(MODEL_INFO) + ["number"] * len(TASK_INFO)
 CSV_DIR = "./file/AV-Odyssey_performance.csv"
 
+timbre_task = [1, 11]
+tone_task = [12, 13]
+melody_task = [14, 18]
+space_task = [19, 20]
+time_task = [21, 23]
+hallucination_task = [24, 24]
+intricay_task = [25, 26]
+
 COLUMN_NAMES = MODEL_INFO + TASK_INFO
 
 DATA_NUM = [200, 200, 200, 200, 200, 200, 200, 200, 108, 196, 200, 200, 20, 97, 200, 200, 200, 200, 20, 20, 200, 200, 200, 200, 199, 195]

file/AV-Odyssey_performance.csv

@@ -1,4 +1,4 @@
-Model,Avg. All,Avg. Timbre,Avg. Tone,Avg. Melody,Avg. Space,Avg. Time,Avg. Hallucination,Avg. Intricacy,Instrument Recognition,Singer Recognition,Gunshot Recognition,Bird Recognition,Animal Recognition,Transportation Recognition,Material Recognition,Scene Recognition,Hazard Recognition,Action Recognition,Eating Sound Recognition,Speech Sentiment Analysis,Meme Understanding,Music Sentiment Analysis,Music Genre Classification,Dance and Music Matching,Film and Music Matching,Music Score Matching,Audio 3D Angle Estimation,Audio Distance Estimation,Audio Time Estimation,Audio-Visual Synchronization,Action Sequencing,Hallucination Evaluation,Action Prediction,Action Tracing
+Model,Avg. All,Avg. Timbre,Avg. Tone,Avg. Melody,Avg. Space,Avg. Time,Avg. Hallucination,Avg. Intricacy,Instrument Recognition,Singer Recognition,Gunshot Recognition,Bird Recognition,Animal Recognition,Transportation Recognition,Material Recognition,Scene Recognition,Hazard Recognition,Action Recognition,Eating Sound Recognition,Speech Sentiment Analysis,Meme Understanding,Music Sentiment Analysis,Music Genre Classification,Dance and Music Matching,Film and Music Matching,Music Score Matching,Audio 3D Angle Estimation,Audio Distance Estimation,Audio Time Estimation,Audio-Visual Synchronization,Action Sequencing,Hallucination Evaluation,Action Prediction,Action Tracing
 [Unified-IO-2 L](https://unified-io-2.allenai.org/),26.0,23.8,24.1,28.8,15.0,26.8,30.0,30.4,20.5,22.5,25.5,18.5,27.0,26.5,23.0,28.0,21.3,20.9,26.5,24.5,20.0,27.9,31.0,27.5,32.5,24.5,15.0,15.0,28.0,25.5,27.0,30.0,27.1,33.8
 [Unified-IO-2 XL](https://unified-io-2.allenai.org/),26.3,24.3,23.2,27.8,22.5,25.3,31.5,34.8,20.0,23.5,24.0,20.5,27.5,26.0,27.5,30.0,19.4,19.9,26.5,23.0,25.0,26.9,30.5,27.0,31.5,22.5,30.0,15.0,26.5,25.5,24.0,31.5,35.7,33.8
 [Unified-IO-2 XXL](https://unified-io-2.allenai.org/),27.2,26.3,22.7,26.4,32.5,26.8,24.5,33.8,29.5,24.0,23.5,29.0,23.5,25.5,30.5,26.5,23.1,27.0,25.5,23.0,20.0,23.9,31.5,27.5,24.5,23.5,50.0,15.0,28.0,25.0,27.5,24.5,33.2,34.4

file/av_odyssey.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c065933c9dff31e8d16c9684570fb4d2e90ddec621bef3a138bd1d44d56e82a0
+size 251176

requirements.txt

@@ -68,3 +68,5 @@ urllib3==1.26.15
 uvicorn==0.21.1
 websockets==11.0.1
 yarl==1.8.2
+fastparquet 
+dask