Spaces:
Running
Running
fix formatting
Browse files- app.py +2 -3
- src/about.py +1 -1
- src/display/formatting.py +5 -1
- src/leaderboard/read_evals.py +7 -3
- src/populate.py +2 -0
- src/utils.py +3 -0
app.py
CHANGED
@@ -97,7 +97,6 @@ print(f'Term length dataframe is {term_length_df}')
|
|
97 |
variate_type_df = pivot_existed_df(variate_type_df, tab_name='univariate')
|
98 |
print(f'Variate type dataframe is {variate_type_df}')
|
99 |
model_info_df = get_model_info_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
|
100 |
-
|
101 |
# (
|
102 |
# finished_eval_queue_df,
|
103 |
# running_eval_queue_df,
|
@@ -169,11 +168,11 @@ with demo:
|
|
169 |
leaderboard = init_leaderboard(freq_df, model_info_df)
|
170 |
print(f"FINAL Frequency LEADERBOARD 1 {freq_df}")
|
171 |
|
172 |
-
with gr.TabItem("π
By
|
173 |
leaderboard = init_leaderboard(term_length_df, model_info_df)
|
174 |
print(f"FINAL term length LEADERBOARD 1 {term_length_df}")
|
175 |
|
176 |
-
with gr.TabItem("π
By
|
177 |
leaderboard = init_leaderboard(variate_type_df, model_info_df)
|
178 |
print(f"FINAL LEADERBOARD 1 {variate_type_df}")
|
179 |
with gr.TabItem("π About", elem_id="llm-benchmark-tab-table", id=4):
|
|
|
97 |
variate_type_df = pivot_existed_df(variate_type_df, tab_name='univariate')
|
98 |
print(f'Variate type dataframe is {variate_type_df}')
|
99 |
model_info_df = get_model_info_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
|
|
|
100 |
# (
|
101 |
# finished_eval_queue_df,
|
102 |
# running_eval_queue_df,
|
|
|
168 |
leaderboard = init_leaderboard(freq_df, model_info_df)
|
169 |
print(f"FINAL Frequency LEADERBOARD 1 {freq_df}")
|
170 |
|
171 |
+
with gr.TabItem("π
By Term Length", elem_id="llm-benchmark-tab-table", id=2):
|
172 |
leaderboard = init_leaderboard(term_length_df, model_info_df)
|
173 |
print(f"FINAL term length LEADERBOARD 1 {term_length_df}")
|
174 |
|
175 |
+
with gr.TabItem("π
By Variate Type", elem_id="llm-benchmark-tab-table", id=3):
|
176 |
leaderboard = init_leaderboard(variate_type_df, model_info_df)
|
177 |
print(f"FINAL LEADERBOARD 1 {variate_type_df}")
|
178 |
with gr.TabItem("π About", elem_id="llm-benchmark-tab-table", id=4):
|
src/about.py
CHANGED
@@ -34,7 +34,7 @@ TITLE = """<h1 align="center" id="space-title">GIFT-Eval Time Series Forecasting
|
|
34 |
|
35 |
# What does your leaderboard evaluate?
|
36 |
INTRODUCTION_TEXT = """
|
37 |
-
|
38 |
a pioneering benchmark aimed at promoting evaluation across diverse datasets.
|
39 |
GIFT-Eval encompasses 28 datasets over 144,000 time series and 177 million data
|
40 |
points, spanning seven domains, 10 frequencies, multivariate inputs, and prediction lengths ranging from short to long-term forecasts.
|
|
|
34 |
|
35 |
# What does your leaderboard evaluate?
|
36 |
INTRODUCTION_TEXT = """
|
37 |
+
We introduce the **G**eneral T**I**me Series **F**orecas**T**ing Model Evaluation, GIFT-Eval,
|
38 |
a pioneering benchmark aimed at promoting evaluation across diverse datasets.
|
39 |
GIFT-Eval encompasses 28 datasets over 144,000 time series and 177 million data
|
40 |
points, spanning seven domains, 10 frequencies, multivariate inputs, and prediction lengths ranging from short to long-term forecasts.
|
src/display/formatting.py
CHANGED
@@ -1,5 +1,9 @@
|
|
1 |
def model_hyperlink(link, model_name):
|
2 |
-
|
|
|
|
|
|
|
|
|
3 |
|
4 |
|
5 |
def make_clickable_model(model_name):
|
|
|
1 |
def model_hyperlink(link, model_name):
|
2 |
+
if link == "":
|
3 |
+
return model_name
|
4 |
+
# return f'<a target="_blank">{model_name}</a>'
|
5 |
+
# return f'<a target="_blank" href="{link}" rel="noopener noreferrer">{model_name}</a>'
|
6 |
+
return f'<a target="_blank" href="{link}">{model_name}</a>'
|
7 |
|
8 |
|
9 |
def make_clickable_model(model_name):
|
src/leaderboard/read_evals.py
CHANGED
@@ -8,7 +8,7 @@ import dateutil
|
|
8 |
import ipdb
|
9 |
import numpy as np
|
10 |
|
11 |
-
from src.display.formatting import make_clickable_model
|
12 |
from src.display.utils import ModelType, Tasks, Precision, WeightType, ModelInfoColumn
|
13 |
from src.submission.check_validity import is_model_on_hub
|
14 |
|
@@ -17,6 +17,7 @@ from src.submission.check_validity import is_model_on_hub
|
|
17 |
class ModelConfig:
|
18 |
"""Represents the model configuration of a model"""
|
19 |
model: str
|
|
|
20 |
model_type: ModelType = ModelType.Unknown
|
21 |
precision: Precision = Precision.Unknown
|
22 |
license: str = "?"
|
@@ -35,15 +36,18 @@ class ModelConfig:
|
|
35 |
precision = Precision.from_str(data.get("model_dtype"))
|
36 |
model_type = ModelType.from_str(data.get("model_type", ""))
|
37 |
model = data.get("model", "")
|
38 |
-
|
|
|
39 |
|
40 |
def to_dict(self):
|
41 |
"""Converts the model info to a dict compatible with our dataframe display"""
|
42 |
data_dict = {
|
43 |
-
"model": self.model,
|
|
|
44 |
ModelInfoColumn.precision.name: self.precision.value.name,
|
45 |
ModelInfoColumn.model_type.name: self.model_type.value.name,
|
46 |
ModelInfoColumn.model_type_symbol.name: self.model_type.value.symbol,
|
|
|
47 |
ModelInfoColumn.license.name: self.license,
|
48 |
ModelInfoColumn.likes.name: self.likes,
|
49 |
ModelInfoColumn.params.name: self.num_params,
|
|
|
8 |
import ipdb
|
9 |
import numpy as np
|
10 |
|
11 |
+
from src.display.formatting import make_clickable_model, model_hyperlink
|
12 |
from src.display.utils import ModelType, Tasks, Precision, WeightType, ModelInfoColumn
|
13 |
from src.submission.check_validity import is_model_on_hub
|
14 |
|
|
|
17 |
class ModelConfig:
|
18 |
"""Represents the model configuration of a model"""
|
19 |
model: str
|
20 |
+
model_link: str = ""
|
21 |
model_type: ModelType = ModelType.Unknown
|
22 |
precision: Precision = Precision.Unknown
|
23 |
license: str = "?"
|
|
|
36 |
precision = Precision.from_str(data.get("model_dtype"))
|
37 |
model_type = ModelType.from_str(data.get("model_type", ""))
|
38 |
model = data.get("model", "")
|
39 |
+
model_link = data.get("model_link", "")
|
40 |
+
return cls(model=model, model_link=model_link, model_type=model_type, precision=precision)
|
41 |
|
42 |
def to_dict(self):
|
43 |
"""Converts the model info to a dict compatible with our dataframe display"""
|
44 |
data_dict = {
|
45 |
+
"model": self.model,
|
46 |
+
'model_w_link': model_hyperlink(self.model_link, self.model),
|
47 |
ModelInfoColumn.precision.name: self.precision.value.name,
|
48 |
ModelInfoColumn.model_type.name: self.model_type.value.name,
|
49 |
ModelInfoColumn.model_type_symbol.name: self.model_type.value.symbol,
|
50 |
+
# ModelInfoColumn.model.model_link: model_hyperlink(self.full_model),
|
51 |
ModelInfoColumn.license.name: self.license,
|
52 |
ModelInfoColumn.likes.name: self.likes,
|
53 |
ModelInfoColumn.params.name: self.num_params,
|
src/populate.py
CHANGED
@@ -20,6 +20,8 @@ def get_model_info_df(results_path: str, requests_path: str, cols: list=[], benc
|
|
20 |
def get_merged_df(result_df: pd.DataFrame, model_info_df: pd.DataFrame) -> pd.DataFrame:
|
21 |
"""Merges the model info dataframe with the results dataframe"""
|
22 |
merged_df = pd.merge(model_info_df, result_df, on='model', how='inner')
|
|
|
|
|
23 |
return merged_df
|
24 |
|
25 |
def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
|
|
|
20 |
def get_merged_df(result_df: pd.DataFrame, model_info_df: pd.DataFrame) -> pd.DataFrame:
|
21 |
"""Merges the model info dataframe with the results dataframe"""
|
22 |
merged_df = pd.merge(model_info_df, result_df, on='model', how='inner')
|
23 |
+
merged_df = merged_df.drop(columns=['model'])
|
24 |
+
merged_df = merged_df.rename(columns={'model_w_link': 'model'})
|
25 |
return merged_df
|
26 |
|
27 |
def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
|
src/utils.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
import pandas as pd
|
2 |
import os
|
3 |
import re
|
@@ -60,6 +61,8 @@ def pivot_existed_df(df, tab_name):
|
|
60 |
df_pivot = df_pivot.reset_index()
|
61 |
# df_pivot = df_pivot.round(3)
|
62 |
df_pivot = df_pivot.applymap(format_number)
|
|
|
|
|
63 |
return df_pivot
|
64 |
|
65 |
|
|
|
1 |
+
import ipdb
|
2 |
import pandas as pd
|
3 |
import os
|
4 |
import re
|
|
|
61 |
df_pivot = df_pivot.reset_index()
|
62 |
# df_pivot = df_pivot.round(3)
|
63 |
df_pivot = df_pivot.applymap(format_number)
|
64 |
+
# make sure the data type is float
|
65 |
+
df_pivot.iloc[:, 1:] = df_pivot.iloc[:, 1:].astype(float)
|
66 |
return df_pivot
|
67 |
|
68 |
|