pminervini's picture
update
e829671
raw
history blame
556 Bytes
#!/usr/bin/env python3
import glob
import os
import random
import requests
from datasets import load_dataset, Dataset, DatasetDict
folder_path = 'isp-data-json/' # Replace with your folder path
# Search for all .json files in the folder
json_files = glob.glob(os.path.join(folder_path, '*.jsonl'))
path = 'pminervini/inverse-scaling'
for json_path in json_files:
base_name = os.path.basename(json_path)
name = base_name.split("_")[0]
ds = load_dataset("json", data_files={'data': json_path})
ds.push_to_hub(path, config_name=name)