Spaces:
Runtime error
Runtime error
Commit
•
fe6396c
1
Parent(s):
41212e8
refactor: Remove unnecessary imports and update data_loader.py
Browse files- data_loader.py +2 -24
data_loader.py
CHANGED
@@ -1,31 +1,9 @@
|
|
1 |
import os
|
2 |
from datetime import datetime
|
3 |
-
|
4 |
-
from dotenv import load_dotenv
|
5 |
-
from httpx import Client, AsyncClient
|
6 |
-
from huggingface_hub import HfApi
|
7 |
-
from huggingface_hub.utils import logging
|
8 |
-
from tqdm.auto import tqdm
|
9 |
from typing import Any, Dict, List
|
10 |
-
import pandas as pd
|
11 |
|
12 |
-
|
13 |
-
|
14 |
-
HF_TOKEN = os.getenv("HF_TOKEN")
|
15 |
-
assert HF_TOKEN is not None, "You need to set HF_TOKEN in your environment variables"
|
16 |
-
USER_AGENT = os.getenv("USER_AGENT")
|
17 |
-
assert (
|
18 |
-
USER_AGENT is not None
|
19 |
-
), "You need to set USER_AGENT in your environment variables"
|
20 |
-
|
21 |
-
logger = logging.get_logger(__name__)
|
22 |
-
headers = {
|
23 |
-
"authorization": f"Bearer ${HF_TOKEN}",
|
24 |
-
"user-agent": USER_AGENT,
|
25 |
-
}
|
26 |
-
client = Client(headers=headers)
|
27 |
-
async_client = AsyncClient(headers=headers)
|
28 |
-
api = HfApi(token=HF_TOKEN)
|
29 |
|
30 |
|
31 |
def has_card_data(dataset):
|
|
|
1 |
import os
|
2 |
from datetime import datetime
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
from typing import Any, Dict, List
|
|
|
4 |
|
5 |
+
import pandas as pd
|
6 |
+
from tqdm.auto import tqdm
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
|
8 |
|
9 |
def has_card_data(dataset):
|