GSK-2852-hide-unused-config-files (#131)

- only show the folder where submitted config files are (79d13cdbf5e4d75e5bee945cb98fcd9ac69a4c8c)
- check if config submitted path exist (2f7dd9a64c5b3f35e2fe4e52e6931f7e5bd159ff)

app_debug.py

@@ -3,12 +3,12 @@ from os.path import isfile, join
 import html
 
 import gradio as gr
-
+import os
 import pipe
 from io_utils import get_logs_file
 
 LOG_PATH = "./tmp"
-CONFIG_PATH = "./cicd/configs/"
+CONFIG_PATH = "./cicd/configs/submitted/"
 MAX_FILES_NUM = 20
 
 
@@ -69,6 +69,8 @@ def get_queue_status():
 
 
 def get_demo():
+    if not os.path.exists(CONFIG_PATH):
+        os.makedirs(CONFIG_PATH)
     with gr.Row():
         gr.HTML(        
             value=get_queue_status,

io_utils.py

@@ -1,15 +1,25 @@
 import os
-
+import logging
 import yaml
 
 YAML_PATH = "./cicd/configs"
 LOG_FILE = "temp_log"
 
+logger = logging.getLogger(__name__)
 
 class Dumper(yaml.Dumper):
     def increase_indent(self, flow=False, *args, **kwargs):
         return super().increase_indent(flow=flow, indentless=False)
 
+def get_submitted_yaml_path(uid):
+    if not os.path.exists(f"{YAML_PATH}/submitted"):
+        os.makedirs(f"{YAML_PATH}/submitted")
+    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
+        logger.error(f"config.yaml does not exist for {uid}")
+        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
+    if not os.path.exists(f"{YAML_PATH}/submitted/{uid}_config.yaml"):
+        os.system(f"cp {YAML_PATH}/{uid}_config.yaml {YAML_PATH}/submitted/{uid}_config.yaml")
+    return f"{YAML_PATH}/submitted/{uid}_config.yaml"
 
 def get_yaml_path(uid):
     if not os.path.exists(YAML_PATH):

run_jobs.py

@@ -17,7 +17,7 @@ from app_env import (
     HF_SPACE_ID,
     HF_WRITE_TOKEN,
 )
-from io_utils import LOG_FILE, get_yaml_path, write_log_to_user_file
+from io_utils import LOG_FILE, get_submitted_yaml_path, write_log_to_user_file
 from isolated_env import prepare_venv
 from leaderboard import LEADERBOARD
 
@@ -98,7 +98,7 @@ def prepare_env_and_get_command(
         "--label_mapping",
         json.dumps(label_mapping),
         "--scan_config",
-        get_yaml_path(uid),
+        get_submitted_yaml_path(uid),
         "--inference_type",
         inference_type,
         "--inference_api_token",

text_classification.py

@@ -7,9 +7,9 @@ import pandas as pd
 from transformers import pipeline
 import requests
 import os
+from app_env import HF_WRITE_TOKEN
 
 logger = logging.getLogger(__name__)
-HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
 AUTH_CHECK_URL = "https://huggingface.co/api/whoami-v2"
 
 logger = logging.getLogger(__file__)

text_classification_ui_helpers.py

@@ -35,6 +35,7 @@ from wordings import (
     get_dataset_fetch_error_raw,
 )
 import os
+from app_env import HF_WRITE_TOKEN
 
 MAX_LABELS = 40
 MAX_FEATURES = 20
@@ -268,7 +269,7 @@ def align_columns_and_show_prediction(
         gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
     ]
     
-    hf_token = os.environ.get("HF_WRITE_TOKEN", default="")
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
 
     prediction_input, prediction_response = get_example_prediction(
         model_id, dataset_id, dataset_config, dataset_split, hf_token