Commit
•
fae0e19
1
Parent(s):
d46be0d
Rename load file functions
Browse files- src/details.py +2 -2
- src/hub.py +2 -2
- src/results.py +2 -2
src/details.py
CHANGED
@@ -5,7 +5,7 @@ import pandas as pd
|
|
5 |
from huggingface_hub import HfFileSystem
|
6 |
|
7 |
import src.constants as constants
|
8 |
-
from src.hub import
|
9 |
|
10 |
|
11 |
def update_task_description_component(task):
|
@@ -53,7 +53,7 @@ async def load_details_dataframe(model_id, subtask):
|
|
53 |
if not paths:
|
54 |
return
|
55 |
path = max(paths)
|
56 |
-
data = await
|
57 |
df = pd.json_normalize(data)
|
58 |
df = df.sort_values(by=["doc_id"])
|
59 |
# df = df.rename_axis("Parameters", axis="columns")
|
|
|
5 |
from huggingface_hub import HfFileSystem
|
6 |
|
7 |
import src.constants as constants
|
8 |
+
from src.hub import load_jsonlines_file
|
9 |
|
10 |
|
11 |
def update_task_description_component(task):
|
|
|
53 |
if not paths:
|
54 |
return
|
55 |
path = max(paths)
|
56 |
+
data = await load_jsonlines_file(path)
|
57 |
df = pd.json_normalize(data)
|
58 |
df = df.sort_values(by=["doc_id"])
|
59 |
# df = df.rename_axis("Parameters", axis="columns")
|
src/hub.py
CHANGED
@@ -9,13 +9,13 @@ from huggingface_hub.utils import build_hf_headers
|
|
9 |
client = httpx.AsyncClient(follow_redirects=True)
|
10 |
|
11 |
|
12 |
-
async def
|
13 |
url = to_url(path)
|
14 |
r = await client.get(url)
|
15 |
return r.json()
|
16 |
|
17 |
|
18 |
-
async def
|
19 |
url = to_url(path)
|
20 |
r = await client.get(url, headers=build_hf_headers())
|
21 |
f = io.StringIO(r.text)
|
|
|
9 |
client = httpx.AsyncClient(follow_redirects=True)
|
10 |
|
11 |
|
12 |
+
async def load_json_file(path):
|
13 |
url = to_url(path)
|
14 |
r = await client.get(url)
|
15 |
return r.json()
|
16 |
|
17 |
|
18 |
+
async def load_jsonlines_file(path):
|
19 |
url = to_url(path)
|
20 |
r = await client.get(url, headers=build_hf_headers())
|
21 |
f = io.StringIO(r.text)
|
src/results.py
CHANGED
@@ -5,7 +5,7 @@ import pandas as pd
|
|
5 |
from huggingface_hub import HfFileSystem
|
6 |
|
7 |
import src.constants as constants
|
8 |
-
from src.hub import
|
9 |
|
10 |
|
11 |
def fetch_result_paths():
|
@@ -32,7 +32,7 @@ async def load_results_dataframe(model_id, result_paths_per_model=None):
|
|
32 |
if not model_id or not result_paths_per_model:
|
33 |
return
|
34 |
result_paths = result_paths_per_model[model_id]
|
35 |
-
results = await asyncio.gather(*[
|
36 |
data = {"results": {}, "configs": {}}
|
37 |
for result in results:
|
38 |
data["results"].update(result["results"])
|
|
|
5 |
from huggingface_hub import HfFileSystem
|
6 |
|
7 |
import src.constants as constants
|
8 |
+
from src.hub import load_json_file
|
9 |
|
10 |
|
11 |
def fetch_result_paths():
|
|
|
32 |
if not model_id or not result_paths_per_model:
|
33 |
return
|
34 |
result_paths = result_paths_per_model[model_id]
|
35 |
+
results = await asyncio.gather(*[load_json_file(path) for path in result_paths])
|
36 |
data = {"results": {}, "configs": {}}
|
37 |
for result in results:
|
38 |
data["results"].update(result["results"])
|