Spaces:
AIR-Bench
/
Running on CPU Upgrade

nan commited on
Commit
240d9ce
1 Parent(s): 606d718

feat-add-reranker-url-validator-0515 (#4)

Browse files

- feat: add validation for the reranking models (6709543893db4707357e86d7d68a1812331e30a0)

Files changed (2) hide show
  1. app.py +16 -6
  2. src/utils.py +27 -8
app.py CHANGED
@@ -399,13 +399,23 @@ with demo:
399
  gr.Markdown("## ✉️Submit your model here!", elem_classes="markdown-text")
400
  with gr.Row():
401
  with gr.Column():
402
- model_name = gr.Textbox(label="Model name")
403
  with gr.Column():
404
- model_url = gr.Textbox(label="Model URL")
 
405
  with gr.Column():
406
- reranker_name = gr.Textbox(label="Reranker name")
 
 
 
 
407
  with gr.Column():
408
- reranker_url = gr.Textbox(label="Reranker URL")
 
 
 
 
 
409
  with gr.Column():
410
  benchmark_version = gr.Dropdown(
411
  ["AIR-Bench_24.04", ],
@@ -437,8 +447,8 @@ with demo:
437
  file_output,
438
  model_name,
439
  model_url,
440
- reranker_name,
441
- reranker_url,
442
  benchmark_version,
443
  is_anonymous
444
  ],
 
399
  gr.Markdown("## ✉️Submit your model here!", elem_classes="markdown-text")
400
  with gr.Row():
401
  with gr.Column():
402
+ model_name = gr.Textbox(label="Retrieval Model name")
403
  with gr.Column():
404
+ model_url = gr.Textbox(label="Retrieval Model URL")
405
+ with gr.Row():
406
  with gr.Column():
407
+ reranking_model_name = gr.Textbox(
408
+ label="Reranking Model name",
409
+ info="Optional",
410
+ value="NoReranker"
411
+ )
412
  with gr.Column():
413
+ reranking_model_url = gr.Textbox(
414
+ label="Reranking Model URL",
415
+ info="Optional",
416
+ value=""
417
+ )
418
+ with gr.Row():
419
  with gr.Column():
420
  benchmark_version = gr.Dropdown(
421
  ["AIR-Bench_24.04", ],
 
447
  file_output,
448
  model_name,
449
  model_url,
450
+ reranking_model_name,
451
+ reranking_model_url,
452
  benchmark_version,
453
  is_anonymous
454
  ],
src/utils.py CHANGED
@@ -244,7 +244,14 @@ def calculate_file_md5(file_path):
244
  return md5.hexdigest()
245
 
246
 
247
- def submit_results(filepath: str, model: str, model_url: str, reranker: str, reranker_url: str, version: str = "AIR-Bench_24.04", is_anonymous=False):
 
 
 
 
 
 
 
248
  if not filepath.endswith(".zip"):
249
  return styled_error(f"file uploading aborted. wrong file type: {filepath}")
250
 
@@ -266,6 +273,18 @@ def submit_results(filepath: str, model: str, model_url: str, reranker: str, rer
266
  except EntryNotFoundError as e:
267
  return styled_error(
268
  f"failed to submit. Model url must be a link to a valid HuggingFace model on HuggingFace space. Could not get model {repo_id}")
 
 
 
 
 
 
 
 
 
 
 
 
269
 
270
  # rename the uploaded file
271
  input_fp = Path(filepath)
@@ -274,12 +293,12 @@ def submit_results(filepath: str, model: str, model_url: str, reranker: str, rer
274
  output_fn = f"{timestamp_fn}-{revision}.zip"
275
  input_folder_path = input_fp.parent
276
 
277
- if not reranker:
278
- reranker = 'NoReranker'
279
 
280
  API.upload_file(
281
  path_or_fileobj=filepath,
282
- path_in_repo=f"{version}/{model}/{reranker}/{output_fn}",
283
  repo_id=SEARCH_RESULTS_REPO,
284
  repo_type="dataset",
285
  commit_message=f"feat: submit {model} to evaluate")
@@ -288,8 +307,8 @@ def submit_results(filepath: str, model: str, model_url: str, reranker: str, rer
288
  output_config = {
289
  "model_name": f"{model}",
290
  "model_url": f"{model_url}",
291
- "reranker_name": f"{reranker}",
292
- "reranker_url": f"{reranker_url}",
293
  "version": f"{version}",
294
  "is_anonymous": is_anonymous,
295
  "revision": f"{revision}",
@@ -299,10 +318,10 @@ def submit_results(filepath: str, model: str, model_url: str, reranker: str, rer
299
  json.dump(output_config, f, indent=4, ensure_ascii=False)
300
  API.upload_file(
301
  path_or_fileobj=input_folder_path / output_config_fn,
302
- path_in_repo=f"{version}/{model}/{reranker}/{output_config_fn}",
303
  repo_id=SEARCH_RESULTS_REPO,
304
  repo_type="dataset",
305
- commit_message=f"feat: submit {model} + {reranker} config")
306
  return styled_message(
307
  f"Thanks for submission!\nSubmission revision: {revision}"
308
  )
 
244
  return md5.hexdigest()
245
 
246
 
247
+ def submit_results(
248
+ filepath: str,
249
+ model: str,
250
+ model_url: str,
251
+ reranking_model: str="",
252
+ reranking_model_url: str="",
253
+ version: str="AIR-Bench_24.04",
254
+ is_anonymous=False):
255
  if not filepath.endswith(".zip"):
256
  return styled_error(f"file uploading aborted. wrong file type: {filepath}")
257
 
 
273
  except EntryNotFoundError as e:
274
  return styled_error(
275
  f"failed to submit. Model url must be a link to a valid HuggingFace model on HuggingFace space. Could not get model {repo_id}")
276
+ if reranking_model != "NoReranker":
277
+ if not reranking_model_url.startswith("https://") and not reranking_model_url.startswith("http://"):
278
+ return styled_error(
279
+ f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
280
+ if reranking_model_url.startswith("https://huggingface.co/"):
281
+ # validate model card
282
+ repo_id = reranking_model_url.removeprefix("https://huggingface.co/")
283
+ try:
284
+ card = ModelCard.load(repo_id)
285
+ except EntryNotFoundError as e:
286
+ return styled_error(
287
+ f"failed to submit. Model url must be a link to a valid HuggingFace model on HuggingFace space. Could not get model {repo_id}")
288
 
289
  # rename the uploaded file
290
  input_fp = Path(filepath)
 
293
  output_fn = f"{timestamp_fn}-{revision}.zip"
294
  input_folder_path = input_fp.parent
295
 
296
+ if not reranking_model:
297
+ reranking_model = 'NoReranker'
298
 
299
  API.upload_file(
300
  path_or_fileobj=filepath,
301
+ path_in_repo=f"{version}/{model}/{reranking_model}/{output_fn}",
302
  repo_id=SEARCH_RESULTS_REPO,
303
  repo_type="dataset",
304
  commit_message=f"feat: submit {model} to evaluate")
 
307
  output_config = {
308
  "model_name": f"{model}",
309
  "model_url": f"{model_url}",
310
+ "reranker_name": f"{reranking_model}",
311
+ "reranker_url": f"{reranking_model_url}",
312
  "version": f"{version}",
313
  "is_anonymous": is_anonymous,
314
  "revision": f"{revision}",
 
318
  json.dump(output_config, f, indent=4, ensure_ascii=False)
319
  API.upload_file(
320
  path_or_fileobj=input_folder_path / output_config_fn,
321
+ path_in_repo=f"{version}/{model}/{reranking_model}/{output_config_fn}",
322
  repo_id=SEARCH_RESULTS_REPO,
323
  repo_type="dataset",
324
+ commit_message=f"feat: submit {model} + {reranking_model} config")
325
  return styled_message(
326
  f"Thanks for submission!\nSubmission revision: {revision}"
327
  )