Upload folder using huggingface_hub

app.py

@@ -22,11 +22,11 @@ prompt_dir = 'iclr2024'
 api_keys = {
     'openai_api_key': os.environ.get('openai_api_key'),
     'claude_api_key': os.environ.get('anthropic_api_key'),
-    'gemini_api_key': os.environ.get('google_api_key'),
+    'gemini_api_key': os.environ.get('gemini_api_key'),
     'commandr_api_key': os.environ.get('cohere_api_key')
 }
 
-use_real_api = False
+use_real_api = True
 
 # Function to generate a paper_id using SHA-512 hash
 def generate_paper_id(paper_content):
@@ -204,50 +204,49 @@ def setup_interface():
 
             with gr.TabItem("Leaderboard"):
                 gr.Markdown("## Leaderboard")
-
-                # Fetch the leaderboard data from the database
-                leaderboard_data = get_leaderboard()
-                # print(leaderboard_data)
-                
-                # Create the leaderboard HTML dynamically
-                leaderboard_html = """
-                    <table style="width:100%; border: 1px solid #444; border-collapse: collapse; font-family: Arial, sans-serif; background-color: #2b2b2b;">
-                        <thead>
-                            <tr style="border: 1px solid #444; padding: 12px; background-color: #1a1a1a;">
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Rank</th>
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Model</th>
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Arena Elo</th>
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">95% CI</th>
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Votes</th>
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Organization</th>
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">License</th>
-                                <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Knowledge Cutoff</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                """
                 
-                for rank, model in enumerate(leaderboard_data, start=1):
-                    leaderboard_html += f"""
-                        <tr style="border: 1px solid #444; padding: 12px;">
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{rank}</td>
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['ModelID']}</td>
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['EloScore']}</td>
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['CI_Lower']} - {model['CI_Upper']}</td>
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['Votes']}</td>
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">Organization</td>
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">License</td>
-                            <td style="border: 1px solid #444; padding: 12px; color: #ddd;">Knowledge Cutoff</td>
-                        </tr>
+                def refresh_leaderboard():
+                    leaderboard_data = get_leaderboard()
+                    leaderboard_html = """
+                        <table style="width:100%; border: 1px solid #444; border-collapse: collapse; font-family: Arial, sans-serif; background-color: #2b2b2b;">
+                            <thead>
+                                <tr style="border: 1px solid #444; padding: 12px; background-color: #1a1a1a;">
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Rank</th>
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Model</th>
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Arena Elo</th>
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">95% CI</th>
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Votes</th>
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Organization</th>
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">License</th>
+                                    <th style="border: 1px solid #444; padding: 12px; color: #ddd;">Knowledge Cutoff</th>
+                                </tr>
+                            </thead>
+                            <tbody>
                     """
+                    
+                    for rank, model in enumerate(leaderboard_data, start=1):
+                        leaderboard_html += f"""
+                            <tr style="border: 1px solid #444; padding: 12px;">
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{rank}</td>
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['ModelID']}</td>
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['EloScore']}</td>
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['CI_Lower']} - {model['CI_Upper']}</td>
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['Votes']}</td>
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['Organization']}</td>
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['License']}</td>
+                                <td style="border: 1px solid #444; padding: 12px; color: #ddd;">{model['KnowledgeCutoff']}</td>
+                            </tr>
+                        """
 
-                leaderboard_html += """
-                        </tbody>
-                    </table>
-                """
-                
-                gr.HTML(leaderboard_html)
-
+                    leaderboard_html += """
+                            </tbody>
+                        </table>
+                    """
+                    return gr.update(value=leaderboard_html)
+                new_html = get_leaderboard()
+                leaderboard_html = gr.HTML(new_html)
+                refresh_button = gr.Button("Refresh Leaderboard")
+                refresh_button.click(fn=refresh_leaderboard, inputs=[], outputs=[leaderboard_html])
 
     logging.debug("Gradio interface setup complete.")
     return demo
@@ -257,3 +256,4 @@ if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     demo = setup_interface()
     demo.launch()
+

logging_config.py

@@ -3,7 +3,7 @@ import logging
 def setup_logging():
     logging.basicConfig(
         filename="arena.log",
-        level=logging.DEBUG,  # Change to DEBUG level
+        level=logging.INFO,  # Change to INFO level
         format='%(asctime)s - %(levelname)s - %(message)s'
     )
     logging.info("Logging setup complete.")

models.py

@@ -22,7 +22,7 @@ class PaperProcessor:
     def __init__(self, prompt_dir, model, openai_api_key, claude_api_key, gemini_api_key, commandr_api_key):
         self.prompt_dir = prompt_dir
         self.model = model
-        self.openai_api_key = openai_api_key
+        self.openai_api_key = openai_api_key    
         self.claude_api_key = claude_api_key
         self.gemini_api_key = gemini_api_key
         self.commandr_api_key = commandr_api_key
@@ -55,7 +55,7 @@ class PaperProcessor:
         logging.info(f"Sending the following prompt to {model_type}: {prompt}")
 
         try:
-            if model_type == 'gpt':
+            if model_type == 'gpt-4-turbo-2024-04-09':
                 client = OpenAI(api_key=self.openai_api_key)
                 messages = [{"role": "system", "content": system_role}, {"role": "user", "content": prompt}]
                 completion = client.chat.completions.create(
@@ -63,9 +63,21 @@ class PaperProcessor:
                     messages=messages,
                     temperature=1
                 )
+                print(completion)
+                return completion.choices[0].message.content.strip()
+            
+            elif model_type == 'gpt-4o':
+                client = OpenAI(api_key=self.openai_api_key)
+                messages = [{"role": "system", "content": system_role}, {"role": "user", "content": prompt}]
+                completion = client.chat.completions.create(
+                    model="gpt-4o",
+                    messages=messages,
+                    temperature=1
+                )
+                print(completion)
                 return completion.choices[0].message.content.strip()
 
-            elif model_type == 'claude':
+            elif model_type == 'claude-3-opus-20240229':
                 client = anthropic.Anthropic(api_key=self.claude_api_key)
                 response = client.messages.create(
                     model='claude-3-opus-20240229',
@@ -74,25 +86,29 @@ class PaperProcessor:
                     temperature=0.5, 
                     messages=[{"role": "user", "content": prompt}]
                 )
+                print(response)
                 return response.content[0].text
 
-            elif model_type == 'commandr':
+            elif model_type == 'command-r-plus':
                 co = cohere.Client(self.commandr_api_key)
                 response = co.chat(
                     model="command-r-plus",
                     message=prompt,
                     preamble=system_role
                 )
+                print(response)
                 return response.text
 
-            elif model_type == 'gemini':
+            elif model_type == 'gemini-pro':
                 genai.configure(api_key=self.gemini_api_key)
                 model = genai.GenerativeModel('gemini-pro')
                 response = model.generate_content(prompt)
+                print(response)
                 return response.candidates[0].content.parts[0].text
 
         except Exception as e:
             logging.error(f"Exception occurred: {e}")
+            print(e)
             return None
 
     def is_content_appropriate(self, content):

utils.py

@@ -3,7 +3,8 @@ import os
 import logging
 import random
 from models import Paper, PaperProcessor
-
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
 
 def extract_text_from_pdf(filename):
     with fitz.open(filename) as pdf_document:
@@ -33,13 +34,24 @@ def process_paper(pdf_file, paper_dir, prompt_dir, api_keys):
     paper = Paper(pdf_file.name if hasattr(pdf_file, 'name')
                   else os.path.basename(pdf_path), extracted_text)
 
-    models = ['gpt', 'claude', 'gemini', 'commandr']
+    models = ['gpt-4-turbo-2024-04-09', 'gpt-4o', 'claude-3-opus-20240229', 'gemini-pro', 'command-r-plus']
     selected_models = random.sample(models, 2)
 
     reviews = []
-    for model in selected_models:
+
+    def process_with_model(model):
         processor = PaperProcessor(prompt_dir, model, **api_keys)
-        review_text = processor.process_paper(paper)
-        reviews.append(review_text)
+        return processor.process_paper(paper)
+
+    with ThreadPoolExecutor() as executor:
+        future_to_model = {executor.submit(process_with_model, model): model for model in selected_models}
+        for future in concurrent.futures.as_completed(future_to_model):
+            model = future_to_model[future]
+            try:
+                review_text = future.result()
+                reviews.append(review_text)
+            except Exception as exc:
+                logging.error(f"Model {model} generated an exception: {exc}")
+
     logging.debug(f"Reviews generated: {reviews}")
     return reviews, selected_models