Spaces:
Running
Running
Commit
•
9e62f85
1
Parent(s):
25f01d1
Upload folder using huggingface_hub
Browse files- app.py +37 -13
- aws_utils.py +99 -0
- requirements.txt +0 -0
app.py
CHANGED
@@ -4,6 +4,10 @@ import os
|
|
4 |
import logging
|
5 |
import html
|
6 |
from logging_config import setup_logging
|
|
|
|
|
|
|
|
|
7 |
|
8 |
setup_logging()
|
9 |
paper_dir = 'path_to_temp_storage'
|
@@ -17,9 +21,17 @@ api_keys = {
|
|
17 |
|
18 |
use_real_api = False
|
19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
def review_papers(pdf_file):
|
22 |
logging.info(f"Received file type: {type(pdf_file)}")
|
|
|
23 |
if use_real_api:
|
24 |
reviews, selected_models = process_paper(
|
25 |
pdf_file, paper_dir, prompt_dir, api_keys)
|
@@ -77,7 +89,24 @@ def review_papers(pdf_file):
|
|
77 |
model_b = selected_models[1]
|
78 |
|
79 |
logging.debug(f"Final formatted reviews: {review_texts}")
|
80 |
-
return review_texts[0], review_texts[1], gr.update(visible=True), gr.update(visible=True), model_a, model_b
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
81 |
|
82 |
|
83 |
def setup_interface():
|
@@ -155,20 +184,14 @@ def setup_interface():
|
|
155 |
|
156 |
model_identity_message = gr.HTML("", visible=False)
|
157 |
|
158 |
-
def
|
159 |
-
|
160 |
-
message = f"<p>Thank you for your vote!</p><p>Model A: {model_a}</p><p>Model B: {model_b}</p>"
|
161 |
-
return gr.update(value=message, visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
|
162 |
|
163 |
-
vote_button.click(fn=
|
164 |
-
vote_message, vote, vote_button, another_paper_button])
|
165 |
|
166 |
-
submit_button.click(
|
167 |
-
|
168 |
-
inputs=[file_input],
|
169 |
-
outputs=[review1, review2, vote, vote_button,
|
170 |
-
model_identity_message, model_identity_message]
|
171 |
-
)
|
172 |
|
173 |
another_paper_button.click(
|
174 |
fn=lambda: None, inputs=None, outputs=None, js="() => { location.reload(); }")
|
@@ -242,6 +265,7 @@ def setup_interface():
|
|
242 |
</tbody>
|
243 |
</table>
|
244 |
"""
|
|
|
245 |
gr.HTML(leaderboard_html)
|
246 |
|
247 |
logging.debug("Gradio interface setup complete.")
|
|
|
4 |
import logging
|
5 |
import html
|
6 |
from logging_config import setup_logging
|
7 |
+
from aws_utils import update_leaderboard, get_leaderboard, write_request
|
8 |
+
from flask import request
|
9 |
+
import hashlib
|
10 |
+
import uuid
|
11 |
|
12 |
setup_logging()
|
13 |
paper_dir = 'path_to_temp_storage'
|
|
|
21 |
|
22 |
use_real_api = False
|
23 |
|
24 |
+
# Function to generate a paper_id using SHA-512 hash
|
25 |
+
def generate_paper_id(paper_content):
|
26 |
+
return hashlib.sha512(paper_content).hexdigest()
|
27 |
+
|
28 |
+
# Function to get user IP address
|
29 |
+
def get_user_ip():
|
30 |
+
return request.remote_addr
|
31 |
|
32 |
def review_papers(pdf_file):
|
33 |
logging.info(f"Received file type: {type(pdf_file)}")
|
34 |
+
paper_content = pdf_file.read() # Read the content of the uploaded PDF file
|
35 |
if use_real_api:
|
36 |
reviews, selected_models = process_paper(
|
37 |
pdf_file, paper_dir, prompt_dir, api_keys)
|
|
|
89 |
model_b = selected_models[1]
|
90 |
|
91 |
logging.debug(f"Final formatted reviews: {review_texts}")
|
92 |
+
return review_texts[0], review_texts[1], gr.update(visible=True), gr.update(visible=True), model_a, model_b, paper_content
|
93 |
+
|
94 |
+
|
95 |
+
def handle_vote(vote, model_a, model_b, paper_content):
|
96 |
+
user_id = get_user_ip() # Get the user IP address as user_id
|
97 |
+
paper_id = generate_paper_id(paper_content) # Generate paper_id from paper content
|
98 |
+
|
99 |
+
# Write the request
|
100 |
+
write_request(user_id, paper_id, model_a, model_b, vote)
|
101 |
+
|
102 |
+
# Update the leaderboard
|
103 |
+
update_leaderboard(model_a, model_b, vote)
|
104 |
+
|
105 |
+
# Fetch the updated leaderboard (optional, if you want to display it immediately)
|
106 |
+
leaderboard = get_leaderboard()
|
107 |
+
|
108 |
+
message = f"<p>Thank you for your vote!</p><p>Model A: {model_a}</p><p>Model B: {model_b}</p>"
|
109 |
+
return gr.update(value=message, visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
|
110 |
|
111 |
|
112 |
def setup_interface():
|
|
|
184 |
|
185 |
model_identity_message = gr.HTML("", visible=False)
|
186 |
|
187 |
+
def handle_vote_interface(vote, model_identity_message_a, model_identity_message_b, paper_content):
|
188 |
+
return handle_vote(vote, model_identity_message_a, model_identity_message_b, paper_content)
|
|
|
|
|
189 |
|
190 |
+
vote_button.click(fn=handle_vote_interface, inputs=[vote, model_identity_message, model_identity_message, paper_content],
|
191 |
+
outputs=[vote_message, vote, vote_button, another_paper_button])
|
192 |
|
193 |
+
submit_button.click(fn=review_papers, inputs=[file_input],
|
194 |
+
outputs=[review1, review2, vote, vote_button, model_identity_message, model_identity_message, paper_content])
|
|
|
|
|
|
|
|
|
195 |
|
196 |
another_paper_button.click(
|
197 |
fn=lambda: None, inputs=None, outputs=None, js="() => { location.reload(); }")
|
|
|
265 |
</tbody>
|
266 |
</table>
|
267 |
"""
|
268 |
+
leaderboard = get_leaderboard()
|
269 |
gr.HTML(leaderboard_html)
|
270 |
|
271 |
logging.debug("Gradio interface setup complete.")
|
aws_utils.py
ADDED
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import boto3
|
2 |
+
import uuid
|
3 |
+
import datetime
|
4 |
+
import os
|
5 |
+
|
6 |
+
# Load AWS credentials from environment variables
|
7 |
+
aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID')
|
8 |
+
aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
9 |
+
aws_region = os.environ.get('AWS_REGION')
|
10 |
+
|
11 |
+
# Initialize the DynamoDB client
|
12 |
+
dynamodb = boto3.resource('dynamodb',
|
13 |
+
region_name=aws_region,
|
14 |
+
aws_access_key_id=aws_access_key_id,
|
15 |
+
aws_secret_access_key=aws_secret_access_key)
|
16 |
+
|
17 |
+
# Define the tables
|
18 |
+
requests_table = dynamodb.Table('reviewer_arena_requests')
|
19 |
+
leaderboards_table = dynamodb.Table('reviewer_arena_leaderboard')
|
20 |
+
|
21 |
+
# Function to write a request to the Requests table
|
22 |
+
def write_request(user_id, paper_id, model_a, model_b, vote):
|
23 |
+
request_id = str(uuid.uuid4())
|
24 |
+
timestamp = datetime.datetime.now().isoformat()
|
25 |
+
|
26 |
+
response = requests_table.put_item(
|
27 |
+
Item={
|
28 |
+
'RequestID': request_id,
|
29 |
+
'Timestamp': timestamp,
|
30 |
+
'UserID': user_id,
|
31 |
+
'PaperID': paper_id,
|
32 |
+
'ModelA': model_a,
|
33 |
+
'ModelB': model_b,
|
34 |
+
'Vote': vote
|
35 |
+
}
|
36 |
+
)
|
37 |
+
return response
|
38 |
+
|
39 |
+
# Function to update leaderboard after a vote
|
40 |
+
def update_leaderboard(model_a, model_b, vote):
|
41 |
+
# Retrieve current stats for ModelA and ModelB
|
42 |
+
model_a_stats = leaderboards_table.get_item(Key={'ModelID': model_a}).get('Item', {})
|
43 |
+
model_b_stats = leaderboards_table.get_item(Key={'ModelID': model_b}).get('Item', {})
|
44 |
+
|
45 |
+
# Initialize stats if they don't exist
|
46 |
+
if not model_a_stats:
|
47 |
+
model_a_stats = {'ModelID': model_a, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': 1200, 'Votes': 0}
|
48 |
+
if not model_b_stats:
|
49 |
+
model_b_stats = {'ModelID': model_b, 'Wins': 0, 'Losses': 0, 'Ties': 0, 'EloScore': 1200, 'Votes': 0}
|
50 |
+
|
51 |
+
# Update stats based on the vote
|
52 |
+
if vote == "A is better":
|
53 |
+
model_a_stats['Wins'] += 1
|
54 |
+
model_b_stats['Losses'] += 1
|
55 |
+
elif vote == "B is better":
|
56 |
+
model_a_stats['Losses'] += 1
|
57 |
+
model_b_stats['Wins'] += 1
|
58 |
+
elif vote == "Tie":
|
59 |
+
model_a_stats['Ties'] += 1
|
60 |
+
model_b_stats['Ties'] += 1
|
61 |
+
model_a_stats['Votes'] += 1
|
62 |
+
model_b_stats['Votes'] += 1
|
63 |
+
|
64 |
+
# Calculate new Elo scores (simple Elo calculation for illustration)
|
65 |
+
model_a_stats['EloScore'], model_b_stats['EloScore'] = calculate_elo(model_a_stats['EloScore'], model_b_stats['EloScore'], vote)
|
66 |
+
|
67 |
+
# Write updated stats back to the Leaderboards table
|
68 |
+
leaderboards_table.put_item(Item=model_a_stats)
|
69 |
+
leaderboards_table.put_item(Item=model_b_stats)
|
70 |
+
|
71 |
+
# Function to calculate new Elo scores
|
72 |
+
def calculate_elo(elo_a, elo_b, vote, k=32):
|
73 |
+
expected_a = 1 / (1 + 10 ** ((elo_b - elo_a) / 400))
|
74 |
+
expected_b = 1 / (1 + 10 ** ((elo_a - elo_b) / 400))
|
75 |
+
|
76 |
+
if vote == "A is better":
|
77 |
+
actual_a = 1
|
78 |
+
actual_b = 0
|
79 |
+
elif vote == "B is better":
|
80 |
+
actual_a = 0
|
81 |
+
actual_b = 1
|
82 |
+
else: # Tie
|
83 |
+
actual_a = 0.5
|
84 |
+
actual_b = 0.5
|
85 |
+
|
86 |
+
new_elo_a = elo_a + k * (actual_a - expected_a)
|
87 |
+
new_elo_b = elo_b + k * (actual_b - expected_b)
|
88 |
+
|
89 |
+
return round(new_elo_a), round(new_elo_b)
|
90 |
+
|
91 |
+
# Function to query leaderboard
|
92 |
+
def get_leaderboard():
|
93 |
+
response = leaderboards_table.scan()
|
94 |
+
leaderboard = response.get('Items', [])
|
95 |
+
|
96 |
+
# Sort by EloScore in descending order
|
97 |
+
leaderboard.sort(key=lambda x: x['EloScore'], reverse=True)
|
98 |
+
|
99 |
+
return leaderboard
|
requirements.txt
CHANGED
Binary files a/requirements.txt and b/requirements.txt differ
|
|