Upload folder using huggingface_hub

app.py

@@ -4,6 +4,10 @@ import os
 import logging
 import html
 from logging_config import setup_logging
+from aws_utils import update_leaderboard, get_leaderboard, write_request
+from flask import request
+import hashlib
+import uuid
 
 setup_logging()
 paper_dir = 'path_to_temp_storage'
@@ -17,9 +21,17 @@ api_keys = {
 
 use_real_api = False
 
+# Function to generate a paper_id using SHA-512 hash
+def generate_paper_id(paper_content):
+    return hashlib.sha512(paper_content).hexdigest()
+
+# Function to get user IP address
+def get_user_ip():
+    return request.remote_addr
 
 def review_papers(pdf_file):
     logging.info(f"Received file type: {type(pdf_file)}")
+    paper_content = pdf_file.read()  # Read the content of the uploaded PDF file
     if use_real_api:
         reviews, selected_models = process_paper(
             pdf_file, paper_dir, prompt_dir, api_keys)
@@ -77,7 +89,24 @@ def review_papers(pdf_file):
     model_b = selected_models[1]
 
     logging.debug(f"Final formatted reviews: {review_texts}")
-    return review_texts[0], review_texts[1], gr.update(visible=True), gr.update(visible=True), model_a, model_b
+    return review_texts[0], review_texts[1], gr.update(visible=True), gr.update(visible=True), model_a, model_b, paper_content
+
+
+def handle_vote(vote, model_a, model_b, paper_content):
+    user_id = get_user_ip()  # Get the user IP address as user_id
+    paper_id = generate_paper_id(paper_content)  # Generate paper_id from paper content
+    
+    # Write the request
+    write_request(user_id, paper_id, model_a, model_b, vote)
+    
+    # Update the leaderboard
+    update_leaderboard(model_a, model_b, vote)
+    
+    # Fetch the updated leaderboard (optional, if you want to display it immediately)
+    leaderboard = get_leaderboard()
+    
+    message = f"<p>Thank you for your vote!</p><p>Model A: {model_a}</p><p>Model B: {model_b}</p>"
+    return gr.update(value=message, visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
 
 
 def setup_interface():
@@ -155,20 +184,14 @@ def setup_interface():
 
                 model_identity_message = gr.HTML("", visible=False)
 
-                def handle_vote(vote, model_a, model_b):
-                    print(f"Vote received: {vote}")
-                    message = f"<p>Thank you for your vote!</p><p>Model A: {model_a}</p><p>Model B: {model_b}</p>"
-                    return gr.update(value=message, visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
+                def handle_vote_interface(vote, model_identity_message_a, model_identity_message_b, paper_content):
+                    return handle_vote(vote, model_identity_message_a, model_identity_message_b, paper_content)
 
-                vote_button.click(fn=handle_vote, inputs=[vote, model_identity_message, model_identity_message], outputs=[
-                                  vote_message, vote, vote_button, another_paper_button])
+                vote_button.click(fn=handle_vote_interface, inputs=[vote, model_identity_message, model_identity_message, paper_content],
+                                  outputs=[vote_message, vote, vote_button, another_paper_button])
 
-                submit_button.click(
-                    fn=review_papers,
-                    inputs=[file_input],
-                    outputs=[review1, review2, vote, vote_button,
-                             model_identity_message, model_identity_message] 
-                )
+                submit_button.click(fn=review_papers, inputs=[file_input],
+                                    outputs=[review1, review2, vote, vote_button, model_identity_message, model_identity_message, paper_content])
 
                 another_paper_button.click(
                     fn=lambda: None, inputs=None, outputs=None, js="() => { location.reload(); }")
@@ -242,6 +265,7 @@ def setup_interface():
                         </tbody>
                     </table>
                 """
+                leaderboard = get_leaderboard()
                 gr.HTML(leaderboard_html)
 
     logging.debug("Gradio interface setup complete.")

aws_utils.py

@@ -0,0 +1,99 @@
+import boto3
+import uuid
+import datetime
+import os
+
+# Load AWS credentials from environment variables
+aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID')
+aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+aws_region = os.environ.get('AWS_REGION')
+
+# Initialize the DynamoDB client
+dynamodb = boto3.resource('dynamodb',
+                          region_name=aws_region,
+                          aws_access_key_id=aws_access_key_id,
+                          aws_secret_access_key=aws_secret_access_key)
+
+# Define the tables
+requests_table = dynamodb.Table('reviewer_arena_requests')
+leaderboards_table = dynamodb.Table('reviewer_arena_leaderboard')
+
+# Function to write a request to the Requests table
+def write_request(user_id, paper_id, model_a, model_b, vote):
+    request_id = str(uuid.uuid4())
+    timestamp = datetime.datetime.now().isoformat()
+    
+    response = requests_table.put_item(
+        Item={
+            'RequestID': request_id,
+            'Timestamp': timestamp,
+            'UserID': user_id,
+            'PaperID': paper_id,
+            'ModelA': model_a,
+            'ModelB': model_b,
+            'Vote': vote
+        }
+    )
+    return response
+
+# Function to update leaderboard after a vote
+def update_leaderboard(model_a, model_b, vote):
+    # Retrieve current stats for ModelA and ModelB
+    model_a_stats = leaderboards_table.get_item(Key={'ModelID': model_a}).get('Item', {})
+    model_b_stats = leaderboards_table.get_item(Key={'ModelID': model_b}).get('Item', {})
+    
+    # Initialize stats if they don't exist
+    if not model_a_stats:
+        model_a_stats = {'ModelID': model_a, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': 1200, 'Votes': 0}
+    if not model_b_stats:
+        model_b_stats = {'ModelID': model_b, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': 1200, 'Votes': 0}
+    
+    # Update stats based on the vote
+    if vote == "A is better":
+        model_a_stats['Wins'] += 1
+        model_b_stats['Losses'] += 1
+    elif vote == "B is better":
+        model_a_stats['Losses'] += 1
+        model_b_stats['Wins'] += 1
+    elif vote == "Tie":
+        model_a_stats['Ties'] += 1
+        model_b_stats['Ties'] += 1
+    model_a_stats['Votes'] += 1
+    model_b_stats['Votes'] += 1
+    
+    # Calculate new Elo scores (simple Elo calculation for illustration)
+    model_a_stats['EloScore'], model_b_stats['EloScore'] = calculate_elo(model_a_stats['EloScore'], model_b_stats['EloScore'], vote)
+    
+    # Write updated stats back to the Leaderboards table
+    leaderboards_table.put_item(Item=model_a_stats)
+    leaderboards_table.put_item(Item=model_b_stats)
+
+# Function to calculate new Elo scores
+def calculate_elo(elo_a, elo_b, vote, k=32):
+    expected_a = 1 / (1 + 10 ** ((elo_b - elo_a) / 400))
+    expected_b = 1 / (1 + 10 ** ((elo_a - elo_b) / 400))
+    
+    if vote == "A is better":
+        actual_a = 1
+        actual_b = 0
+    elif vote == "B is better":
+        actual_a = 0
+        actual_b = 1
+    else:  # Tie
+        actual_a = 0.5
+        actual_b = 0.5
+    
+    new_elo_a = elo_a + k * (actual_a - expected_a)
+    new_elo_b = elo_b + k * (actual_b - expected_b)
+    
+    return round(new_elo_a), round(new_elo_b)
+
+# Function to query leaderboard
+def get_leaderboard():
+    response = leaderboards_table.scan()
+    leaderboard = response.get('Items', [])
+    
+    # Sort by EloScore in descending order
+    leaderboard.sort(key=lambda x: x['EloScore'], reverse=True)
+    
+    return leaderboard