Use another process to run scan

app.py

@@ -1,10 +1,10 @@
 import gradio as gr
 import datasets
 import huggingface_hub
-import sys
 import os
 import time
-from pathlib import Path
+import subprocess
+import logging
 
 import json
 
@@ -154,48 +154,32 @@ def try_validate(model_id, dataset_id, dataset_config, dataset_split, column_map
 
 def try_submit(m_id, d_id, config, split, local):
     if local:
-        if "cicd" not in sys.path:
-            sys.path.append("cicd")
-        from giskard_cicd.loaders import HuggingFaceLoader
-        from giskard_cicd.pipeline.runner import PipelineRunner
-
-        from automation import create_discussion_detailed
-        supported_loaders = {
-            "huggingface": HuggingFaceLoader(),
-        }
-
-        runner = PipelineRunner(loaders=supported_loaders)
-
-        runner_kwargs = {
-            "loader_id": "huggingface",
-            "model": m_id,
-            "dataset": d_id,
-            "scan_config": None,
-            "dataset_split": split,
-            "dataset_config": config,
-        }
+        command = [
+            "python",
+            "cli.py",
+            "--loader", "huggingface",
+            "--model", m_id,
+            "--dataset", d_id,
+            "--dataset_config", config,
+            "--dataset_split", split,
+            "--hf_token", os.environ.get(HF_WRITE_TOKEN),
+            "--discussion_repo", os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
+            "--output_format", "markdown",
+            "--output_portal", "huggingface",
+        ]
 
         eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
         start = time.time()
-        print(f"Start local evaluation on {eval_str}")
+        logging.info(f"Start local evaluation on {eval_str}")
 
-        report = runner.run(**runner_kwargs)
-
-        # TODO: Publish it with given repo id/model id
-        if os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID) and os.environ.get(HF_WRITE_TOKEN):
-            rendered_report = report.to_markdown(template="github")
-            repo = os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID)
-            create_discussion_detailed(repo, m_id, d_id, config, split, os.environ.get(HF_WRITE_TOKEN), rendered_report)
-
-        # Cache locally
-        rendered_report = report.to_html()
-        output_dir = Path(f"output/{m_id}/{d_id}/{config}/{split}/")
-        output_dir.mkdir(parents=True, exist_ok=True)
-        with open(output_dir / "report.html", "w") as f:
-            print(f'Writing to {output_dir / "report.html"}')
-            f.write(rendered_report)
+        evaluator = subprocess.Popen(
+            command,
+            cwd=os.path.join(os.path.dirname(os.path.realpath(__file__)), "cicd"),
+            stderr=subprocess.STDOUT,
+        )
+        result = evaluator.wait()
 
-        print(f"Finished local evaluation on {eval_str}: {time.time() - start:.2f}s")
+        logging.info(f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s")
 
 
 with gr.Blocks(theme=theme) as iface:

cicd/cli.py

@@ -3,7 +3,7 @@ import argparse
 from giskard_cicd.loaders import GithubLoader, HuggingFaceLoader
 from giskard_cicd.pipeline.runner import PipelineRunner
 
-from automation import create_discussion
+from automation import create_discussion_detailed
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -55,7 +55,7 @@ if __name__ == "__main__":
     
     if args.output_portal == "huggingface":
         # Push to discussion
-        create_discussion(args.discussion_repo, args.model, args.hf_token, rendered_report)
+        create_discussion_detailed(args.discussion_repo, args.model, args.dataset, args.dataset_config, args.dataset_split, args.hf_token, rendered_report)
 
 
     if args.output: