Spaces:
Runtime error
Runtime error
File size: 2,604 Bytes
6bec1f5 92efc51 6bec1f5 c26894e 6bec1f5 92efc51 6bec1f5 92efc51 6bec1f5 92efc51 6bec1f5 c26894e 6bec1f5 a9bd212 6bec1f5 967e6fd dbaa2bd a9bd212 6bec1f5 8aabc99 6bec1f5 a9bd212 6bec1f5 35db90a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
import os
import sys
import subprocess
from huggingface_hub import hf_hub_download
def run_command(command: str, cwd: str = None) -> tuple:
"""Run a shell command in the specified directory and return the output."""
process = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
)
stdout, stderr = process.communicate()
if process.returncode != 0:
print(f"Error: {stderr.decode()}")
else:
print(f"Output: {stdout.decode()}")
return stdout, stderr
def download_dataset():
"""Download the dataset."""
print("Downloading the autocast dataset...")
repo_id = "valory/autocast"
base_dir = os.getcwd()
output_dir = os.path.join(
base_dir, "olas-predict-benchmark", "benchmark", "data", "autocast"
)
if not os.path.exists(output_dir):
os.makedirs(output_dir, exist_ok=True)
filenames = [
"autocast_questions_filtered.json",
"autocast_questions_filtered.pkl",
]
for filename in filenames:
hf_hub_download(
repo_id=repo_id,
filename=filename,
local_dir=output_dir,
repo_type="dataset",
)
print("Dataset downloaded successfully.")
def start():
"""Start commands."""
print("Starting commands...")
base_dir = os.getcwd()
# olas_dir = os.path.join(base_dir, "olas-predict-benchmark")
# benchmark_dir = os.path.join(olas_dir, "benchmark")
# mech_dir = os.path.join(olas_dir, "benchmark", "mech")
commands = [
("pip install poetry", base_dir),
("git submodule init", base_dir),
("git submodule update --init --recursive", base_dir),
("git submodule update --remote --recursive", base_dir),
# (
# 'git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"',
# olas_dir,
# ),
# ("git remote update", olas_dir),
# ("git fetch --all", olas_dir),
# ("git checkout main", olas_dir),
# ("git pull origin main", olas_dir),
# ("git checkout main", mech_dir),
# ("git pull origin main", mech_dir),
# ("poetry install", benchmark_dir),
# ("pip install -e .", mech_dir),
("pip install lxml[html_clean]", base_dir),
("pip install --upgrade huggingface_hub", base_dir),
]
for command, cwd in commands:
run_command(command, cwd=cwd)
# add benchmark to the path
# sys.path.append(os.path.join(olas_dir, "benchmark"))
# Download the dataset
download_dataset()
start()
|