Omartificial-Intelligence-Space
commited on
Commit
•
d73f924
1
Parent(s):
59696b8
update utils
Browse files- src/display/utils.py +31 -96
src/display/utils.py
CHANGED
@@ -39,16 +39,30 @@ COLUMNS.append(
|
|
39 |
)
|
40 |
|
41 |
# Include per-subject accuracy columns based on your subjects
|
|
|
42 |
for task in Tasks:
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
50 |
)
|
51 |
-
)
|
52 |
|
53 |
# Additional columns
|
54 |
COLUMNS.extend([
|
@@ -57,126 +71,47 @@ COLUMNS.extend([
|
|
57 |
type=str,
|
58 |
label="Model Type",
|
59 |
description="Type of the model (e.g., Transformer, RNN, etc.)",
|
60 |
-
displayed_by_default=
|
61 |
-
),
|
62 |
-
ColumnContent(
|
63 |
-
name="architecture",
|
64 |
-
type=str,
|
65 |
-
label="Architecture",
|
66 |
-
description="Model architecture",
|
67 |
-
displayed_by_default=False,
|
68 |
),
|
69 |
ColumnContent(
|
70 |
name="weight_type",
|
71 |
type=str,
|
72 |
label="Weight Type",
|
73 |
description="Type of model weights (e.g., Original, Delta, Adapter)",
|
74 |
-
displayed_by_default=
|
75 |
),
|
76 |
ColumnContent(
|
77 |
name="precision",
|
78 |
type=str,
|
79 |
label="Precision",
|
80 |
description="Precision of the model weights (e.g., float16)",
|
81 |
-
displayed_by_default=
|
82 |
),
|
83 |
ColumnContent(
|
84 |
name="license",
|
85 |
type=str,
|
86 |
label="License",
|
87 |
description="License of the model",
|
88 |
-
displayed_by_default=
|
89 |
-
),
|
90 |
-
ColumnContent(
|
91 |
-
name="params",
|
92 |
-
type=float,
|
93 |
-
label="Parameters (B)",
|
94 |
-
description="Number of model parameters in billions",
|
95 |
-
displayed_by_default=False,
|
96 |
),
|
97 |
ColumnContent(
|
98 |
name="likes",
|
99 |
type=int,
|
100 |
label="Likes",
|
101 |
description="Number of likes on the Hugging Face Hub",
|
102 |
-
displayed_by_default=
|
103 |
),
|
104 |
ColumnContent(
|
105 |
name="still_on_hub",
|
106 |
type=bool,
|
107 |
label="Available on the Hub",
|
108 |
description="Whether the model is still available on the Hugging Face Hub",
|
109 |
-
displayed_by_default=
|
110 |
-
),
|
111 |
-
ColumnContent(
|
112 |
-
name="revision",
|
113 |
-
type=str,
|
114 |
-
label="Model Revision",
|
115 |
-
description="Model revision or commit hash",
|
116 |
-
displayed_by_default=False,
|
117 |
),
|
118 |
])
|
119 |
|
120 |
# Now we can create lists of column names for use in the application
|
121 |
COLS = [col.name for col in COLUMNS]
|
122 |
-
BENCHMARK_COLS = [col.name for col in COLUMNS if col.name not in [
|
123 |
-
|
124 |
-
|
125 |
-
@dataclass(frozen=True)
|
126 |
-
class EvalQueueColumn:
|
127 |
-
model: str
|
128 |
-
revision: str
|
129 |
-
private: bool
|
130 |
-
precision: str
|
131 |
-
weight_type: str
|
132 |
-
status: str
|
133 |
-
|
134 |
-
EVAL_COLS = ["model", "revision", "private", "precision", "weight_type", "status"]
|
135 |
-
EVAL_TYPES = [str, str, bool, str, str, str]
|
136 |
-
|
137 |
-
## All the model information that we might need
|
138 |
-
@dataclass
|
139 |
-
class ModelDetails:
|
140 |
-
name: str
|
141 |
-
display_name: str = ""
|
142 |
-
symbol: str = "" # emoji
|
143 |
-
|
144 |
-
class ModelType(Enum):
|
145 |
-
PT = ModelDetails(name="pretrained", symbol="🟢")
|
146 |
-
FT = ModelDetails(name="fine-tuned", symbol="🔶")
|
147 |
-
IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
|
148 |
-
RL = ModelDetails(name="RL-tuned", symbol="🟦")
|
149 |
-
Unknown = ModelDetails(name="", symbol="?")
|
150 |
-
|
151 |
-
def to_str(self, separator=" "):
|
152 |
-
return f"{self.value.symbol}{separator}{self.value.name}"
|
153 |
-
|
154 |
-
@staticmethod
|
155 |
-
def from_str(type_str):
|
156 |
-
if "fine-tuned" in type_str or "🔶" in type_str:
|
157 |
-
return ModelType.FT
|
158 |
-
if "pretrained" in type_str or "🟢" in type_str:
|
159 |
-
return ModelType.PT
|
160 |
-
if "RL-tuned" in type_str or "🟦" in type_str:
|
161 |
-
return ModelType.RL
|
162 |
-
if "instruction-tuned" in type_str or "⭕" in type_str:
|
163 |
-
return ModelType.IFT
|
164 |
-
return ModelType.Unknown
|
165 |
-
|
166 |
-
class WeightType(Enum):
|
167 |
-
Adapter = "Adapter"
|
168 |
-
Original = "Original"
|
169 |
-
Delta = "Delta"
|
170 |
-
|
171 |
-
class Precision(Enum):
|
172 |
-
float16 = "float16"
|
173 |
-
bfloat16 = "bfloat16"
|
174 |
-
Unknown = "Unknown"
|
175 |
-
|
176 |
-
@staticmethod
|
177 |
-
def from_str(precision_str):
|
178 |
-
if precision_str in ["torch.float16", "float16"]:
|
179 |
-
return Precision.float16
|
180 |
-
if precision_str in ["torch.bfloat16", "bfloat16"]:
|
181 |
-
return Precision.bfloat16
|
182 |
-
return Precision.Unknown
|
|
|
39 |
)
|
40 |
|
41 |
# Include per-subject accuracy columns based on your subjects
|
42 |
+
# Remove unwanted subjects from here
|
43 |
for task in Tasks:
|
44 |
+
# Exclude unwanted tasks
|
45 |
+
if task.name not in [
|
46 |
+
"History",
|
47 |
+
"Mathematics",
|
48 |
+
"Science",
|
49 |
+
"Geography",
|
50 |
+
"Literature",
|
51 |
+
"Art",
|
52 |
+
"Physics",
|
53 |
+
"Chemistry",
|
54 |
+
"Biology",
|
55 |
+
"Computer_Science",
|
56 |
+
]:
|
57 |
+
COLUMNS.append(
|
58 |
+
ColumnContent(
|
59 |
+
name=task.value.benchmark,
|
60 |
+
type=float,
|
61 |
+
label=f"{task.value.col_name} (%)",
|
62 |
+
description=f"Accuracy on {task.value.col_name}",
|
63 |
+
displayed_by_default=True, # Set to True to display by default
|
64 |
+
)
|
65 |
)
|
|
|
66 |
|
67 |
# Additional columns
|
68 |
COLUMNS.extend([
|
|
|
71 |
type=str,
|
72 |
label="Model Type",
|
73 |
description="Type of the model (e.g., Transformer, RNN, etc.)",
|
74 |
+
displayed_by_default=True, # Set to True to display by default
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
75 |
),
|
76 |
ColumnContent(
|
77 |
name="weight_type",
|
78 |
type=str,
|
79 |
label="Weight Type",
|
80 |
description="Type of model weights (e.g., Original, Delta, Adapter)",
|
81 |
+
displayed_by_default=True, # Set to True to display by default
|
82 |
),
|
83 |
ColumnContent(
|
84 |
name="precision",
|
85 |
type=str,
|
86 |
label="Precision",
|
87 |
description="Precision of the model weights (e.g., float16)",
|
88 |
+
displayed_by_default=True, # Set to True to display by default
|
89 |
),
|
90 |
ColumnContent(
|
91 |
name="license",
|
92 |
type=str,
|
93 |
label="License",
|
94 |
description="License of the model",
|
95 |
+
displayed_by_default=True, # Set to True to display by default
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
96 |
),
|
97 |
ColumnContent(
|
98 |
name="likes",
|
99 |
type=int,
|
100 |
label="Likes",
|
101 |
description="Number of likes on the Hugging Face Hub",
|
102 |
+
displayed_by_default=True, # Set to True to display by default
|
103 |
),
|
104 |
ColumnContent(
|
105 |
name="still_on_hub",
|
106 |
type=bool,
|
107 |
label="Available on the Hub",
|
108 |
description="Whether the model is still available on the Hugging Face Hub",
|
109 |
+
displayed_by_default=True, # Set to True to display by default
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
110 |
),
|
111 |
])
|
112 |
|
113 |
# Now we can create lists of column names for use in the application
|
114 |
COLS = [col.name for col in COLUMNS]
|
115 |
+
BENCHMARK_COLS = [col.name for col in COLUMNS if col.name not in [
|
116 |
+
"model", "average", "model_type", "weight_type", "precision", "license", "likes", "still_on_hub"
|
117 |
+
]]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|