File size: 3,630 Bytes
ee9e25e 513e813 cc32c4f b58e1f0 0a77c60 b58e1f0 ee9e25e 513e813 83a34f0 ee9e25e 30fa96a ee9e25e f228d38 c32735e 513e813 b58e1f0 0a77c60 b58e1f0 0a77c60 b58e1f0 0a77c60 b58e1f0 0a77c60 b58e1f0 0a77c60 b58e1f0 9adae3c b58e1f0 9adae3c 0a77c60 9adae3c 0a77c60 b58e1f0 0a77c60 9adae3c 0a77c60 b58e1f0 0a77c60 b58e1f0 513e813 9adae3c 83a34f0 cc32c4f 0a77c60 cc32c4f c32735e 30fa96a c32735e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
import pandas as pd
import os
import fnmatch
import json
import re
import numpy as np
import requests
from urllib.parse import quote
from datetime import datetime
import uuid
class DetailsDataProcessor:
# Download
#url example https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/64bits/LexPodLM-13B/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-25T13%3A41%3A51.227672.json
def __init__(self, directory='results', pattern='results*.json'):
self.directory = directory
self.pattern = pattern
def _find_files(self, directory='results', pattern='results*.json'):
matching_files = [] # List to hold matching filenames
for root, dirs, files in os.walk(directory):
for basename in files:
if fnmatch.fnmatch(basename, pattern):
filename = os.path.join(root, basename)
matching_files.append(filename) # Append the matching filename to the list
return matching_files # Return the list of matching filenames
@staticmethod
def download_file(url, save_file_path):
#TODO: I may not need to save the file. I can just read it in and convert to a dataframe
# Get the current date and time
error_count = 0
success_count = 0
# timestamp = datetime.now()
# Format the timestamp as a string, suitable for use in a filename
# filename_timestamp = timestamp.strftime("%Y-%m-%dT%H-%M-%S")
# Generate a unique UUID
unique_id = uuid.uuid4()
# Append the UUID to the filename
save_file_path = save_file_path + "_" + str(unique_id) + ".json"
try:
# Sending a GET request
r = requests.get(url, allow_redirects=True)
r.raise_for_status() # Raises an HTTPError if the HTTP request returned an unsuccessful status code
# Writing the content to the specified file
with open(save_file_path, 'wb') as file:
file.write(r.content)
success_count += 1
except requests.ConnectionError as e:
error_count += 1
except requests.HTTPError as e:
error_count += 1
except FileNotFoundError as e:
error_count += 1
except Exception as e:
error_count += 1
return error_count, success_count
@staticmethod
def single_file_pipeline(url, filename):
DetailsDataProcessor.download_file(url, filename)
# read file
with open(filename) as f:
data = json.load(f)
# convert to dataframe
df = pd.DataFrame(data)
return df
@staticmethod
def build_url(file_path):
segments = file_path.split('/')
bits = segments[1]
model_name = segments[2]
try:
timestamp = segments[3].split('_')[1]
except IndexError:
print(f"Error: {file_path}")
return None
url = f'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/{bits}/{model_name}/details_harness%7ChendrycksTest-moral_scenarios%7C5_{quote(timestamp, safe="")}'
return url
def pipeline(self):
dataframes = []
file_paths = self._find_files(self.directory, self.pattern)
for file_path in file_paths:
print(file_path)
url = self.generate_url(file_path)
file_path = file_path.split('/')[-1]
df = self.single_file_pipeline(url, file_path)
dataframes.append(df)
return dataframes
|