Spaces:
Runtime error
Runtime error
File size: 1,557 Bytes
6bec1f5 7d57619 6bec1f5 f62fea6 6bec1f5 7d57619 f62fea6 c4e55cd f62fea6 c4e55cd f62fea6 b871994 6bec1f5 33a9520 6bec1f5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
import os
from benchmark.run_benchmark import run_benchmark
def run_benchmark_main(tool_name, model_name, num_questions, openai_api_key, anthropic_api_key, openrouter_api_key):
"""Run the benchmark using the provided function and API key."""
# Empyt the results directory
os.system("rm -rf results/*")
# Set the benchmark parameters
kwargs = {}
if not num_questions:
num_questions = 10
kwargs["num_questions"] = num_questions
kwargs["tools"] = [tool_name]
if model_name:
kwargs["model"] = [model_name]
kwargs["api_keys"] = {}
if openai_api_key:
kwargs["api_keys"]["openai"] = openai_api_key
if anthropic_api_key:
kwargs["api_keys"]["anthropic"] = anthropic_api_key
if openrouter_api_key:
kwargs["api_keys"]["openrouter"] = openrouter_api_key
if "gpt" in model_name:
kwargs["llm_provider"] = "openai"
elif "claude" in model_name:
kwargs["llm_provider"] = "anthropic"
else:
kwargs["llm_provider"] = "openrouter"
if tool_name == "prediction-request-reasoning" or tool_name == "prediction-request-rag":
if not openai_api_key:
return f"Error: Tools that use RAG also require an OpenAI API Key"
kwargs["num_urls"] = 3
kwargs["num_words"] = 300
kwargs["provide_source_links"] = True
print(f"Running benchmark")
# Run the benchmark
try:
run_benchmark(kwargs=kwargs)
return "completed"
except Exception as e:
return f"Error running benchmark: {e}"
|