Add a simple model validation

app.py

@@ -1,11 +1,26 @@
 import gradio as gr
 import datasets
+import huggingface_hub
 
 
 theme = gr.themes.Soft(
     primary_hue="green",
 )
 
+def check_model(model_id):
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+    except Exception:
+        return None, None
+
+    try:
+        from transformers import pipeline
+        ppl = pipeline(task=task, model=model_id)
+        
+        return model_id, ppl
+    except Exception as e:
+        return model_id, e
+
 
 def check_dataset(dataset_id, dataset_config="default", dataset_split="test"):
     try:
@@ -30,16 +45,25 @@ def check_dataset(dataset_id, dataset_config="default", dataset_split="test"):
     return dataset_id, dataset_config, dataset_split
 
 
-def try_submit(dataset_id, dataset_config, dataset_split):
+def try_submit(model_id, dataset_id, dataset_config, dataset_split):
+    # Validate model
+    m_id, ppl = check_model(model_id=model_id)
+    if m_id is None:
+        gr.Warning(f'Model "{model_id}" is not accessible. Please set your HF_TOKEN if it is a private model.')
+        return dataset_config, dataset_split
+    if isinstance(ppl, Exception):
+        gr.Warning(f'Failed to load "{model_id} model": {ppl}')
+        return dataset_config, dataset_split
+
     # Validate dataset
     d_id, config, split = check_dataset(dataset_id=dataset_id, dataset_config=dataset_config, dataset_split=dataset_split)
 
     if d_id is None:
         gr.Warning(f'Dataset "{dataset_id}" is not accessible. Please set your HF_TOKEN if it is a private dataset.')
-    if isinstance(config, list):
+    elif isinstance(config, list):
         gr.Warning(f'Dataset "{dataset_id}" does have "{dataset_config}" config. Please choose a valid config.')
         config = gr.Dropdown.update(choices=config, value=config[0])
-    if isinstance(split, list):
+    elif isinstance(split, list):
         gr.Warning(f'Dataset "{dataset_id}" does have "{dataset_split}" split. Please choose a valid split.')
         split = gr.Dropdown.update(choices=split, value=split[0])
 
@@ -53,6 +77,7 @@ with gr.Blocks(theme=theme) as iface:
                 placeholder="cardiffnlp/twitter-roberta-base-sentiment-latest",
             )
 
+            # TODO: Add supported model pairs: Text Classification - text-classification
             model_type = gr.Dropdown(
                 label="Hugging Face model type",
                 choices=[
@@ -87,10 +112,11 @@ with gr.Blocks(theme=theme) as iface:
             )
 
     with gr.Row():
-        run_btn = gr.Button("Validate and submit", variant="primary")
+        run_btn = gr.Button("Validate and submit evaluation task", variant="primary")
         run_btn.click(
             try_submit,
             inputs=[
+                model_id_input,
                 dataset_id_input,
                 dataset_config_input,
                 dataset_split_input