Spaces:
Runtime error
Runtime error
File size: 1,706 Bytes
1f64946 ed12cc6 1f64946 ed12cc6 1f64946 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
"""Global variables used in the space.
"""
from huggingface_hub import HfApi
import jsonlines
import gradio as gr
from src.constants import DATASET_NAME, HF_TOKEN, ASSETS_FOLDER
hf_api: HfApi
all_metadata: dict
def setup():
global hf_api
global all_metadata
hf_api = HfApi(token=HF_TOKEN)
hf_api.snapshot_download(
local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
repo_id=DATASET_NAME,
repo_type="dataset",
)
all_metadata = {}
for split in ["train", "validation", "test"]:
all_metadata[split] = []
with jsonlines.open(f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/metadata.jsonl") as reader:
for row in reader:
all_metadata[split].append(row)
def get_metadata(split):
global all_metadata
global hf_api
hf_api.hf_hub_download(
repo_id=DATASET_NAME,
filename="metadata.jsonl",
subfolder=f"data/{split}",
repo_type="dataset",
local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
)
all_metadata[split] = []
with jsonlines.open(f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/metadata.jsonl") as reader:
for row in reader:
all_metadata[split].append(row)
def save_metadata(split):
global all_metadata
values = all_metadata[split]
with jsonlines.open(f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/metadata.jsonl", mode='w') as writer:
writer.write_all(values)
hf_api.upload_file(
path_or_fileobj=f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/metadata.jsonl",
path_in_repo=f"data/{split}/metadata.jsonl",
repo_id=DATASET_NAME,
repo_type="dataset",
)
if gr.NO_RELOAD:
setup() |