new UI

app.py

@@ -1,181 +1,197 @@
 import os
-import subprocess
-import importlib.util
-
-NVIDIA_AVAILABLE = (
-    subprocess.run(
-        "nvidia-smi",
-        shell=True,
-        stdout=subprocess.DEVNULL,
-        stderr=subprocess.DEVNULL,
-    ).returncode
-    == 0
-)
-
-if NVIDIA_AVAILABLE:
-    DEVICES = ["cpu", "cuda"]
-    if importlib.util.find_spec("optimum_benchmark") is None:
-        os.system(
-            "pip install optimum-benchmark[onnxruntime-gpu,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git"
-        )
-        os.system("pip uninstall onnxruntime onnxruntime-gpu -y")
-        os.system("pip install onnxruntime-gpu")
-else:
-    DEVICES = ["cpu"]
-    if importlib.util.find_spec("optimum_benchmark") is None:
-        os.system(
-            "pip install optimum-benchmark[onnxruntime,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git"
-        )
-
-
-BACKENDS = ["pytorch", "onnxruntime", "openvino", "neural-compressor"]
-BENCHMARKS = ["inference", "training"]
-
-
-import random
+import time
+from huggingface_hub import create_repo, whoami
 import gradio as gr
-from optimum_benchmark.task_utils import (
-    TASKS_TO_AUTOMODELS,
-    infer_task_from_model_name_or_path,
-)
-from run import run_benchmark
 from config_store import (
-    get_training_config,
     get_inference_config,
-    get_neural_compressor_config,
     get_onnxruntime_config,
     get_openvino_config,
     get_pytorch_config,
+    get_process_config,
+)
+from optimum_benchmark.backends.openvino.utils import TASKS_TO_OVMODEL
+from optimum_benchmark.backends.transformers_utils import TASKS_TO_MODEL_LOADERS
+from optimum_benchmark.backends.onnxruntime.utils import TASKS_TO_ORTMODELS
+from optimum_benchmark.backends.ipex.utils import TASKS_TO_IPEXMODEL
+from optimum_benchmark import (
+    BenchmarkConfig,
+    PyTorchConfig,
+    OVConfig,
+    ORTConfig,
+    IPEXConfig,
+    ProcessConfig,
+    InferenceConfig,
+    Benchmark,
+)
+from optimum_benchmark.logging_utils import setup_logging
+
+os.environ["LOG_TO_FILE"] = "0"
+os.environ["LOG_LEVEL"] = "INFO"
+setup_logging(level="INFO", prefix="MAIN-PROCESS")
+
+DEVICE = "cpu"
+BACKENDS = ["pytorch", "onnxruntime", "openvino", "ipex"]
+
+CHOSEN_MODELS = ["bert-base-uncased", "gpt2"]
+CHOSEN_TASKS = (
+    set(TASKS_TO_OVMODEL.keys())
+    & set(TASKS_TO_ORTMODELS.keys())
+    & set(TASKS_TO_IPEXMODEL.keys())
+    & set(TASKS_TO_MODEL_LOADERS.keys())
 )
 
+
+def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
+    if oauth_token.token is None:
+        return "You must be logged in to use this space"
+
+    username = whoami(oauth_token.token)["name"]
+    create_repo(
+        f"{username}/benchmarks",
+        token=oauth_token.token,
+        repo_type="dataset",
+        exist_ok=True,
+    )
+
+    configs = {
+        "process": {},
+        "inference": {},
+        "onnxruntime": {},
+        "openvino": {},
+        "pytorch": {},
+        "ipex": {},
+    }
+
+    for key, value in kwargs.items():
+        if key.label == "model":
+            model = value
+        elif key.label == "task":
+            task = value
+        elif "." in key.label:
+            backend, argument = key.label.split(".")
+            configs[backend][argument] = value
+        else:
+            continue
+
+    process_config = ProcessConfig(**configs.pop("process"))
+    inference_config = InferenceConfig(**configs.pop("inference"))
+
+    configs["onnxruntime"] = ORTConfig(
+        task=task,
+        model=model,
+        device=DEVICE,
+        **configs["onnxruntime"],
+    )
+    configs["openvino"] = OVConfig(
+        task=task,
+        model=model,
+        device=DEVICE,
+        **configs["openvino"],
+    )
+    configs["pytorch"] = PyTorchConfig(
+        task=task,
+        model=model,
+        device=DEVICE,
+        **configs["pytorch"],
+    )
+    configs["ipex"] = IPEXConfig(
+        task=task,
+        model=model,
+        device=DEVICE,
+        **configs["ipex"],
+    )
+
+    for backend in configs:
+        benchmark_name = (
+            f"{model}-{task}-{backend}-{time.strftime('%Y-%m-%d-%H-%M-%S')}"
+        )
+        benchmark_config = BenchmarkConfig(
+            name=benchmark_name,
+            launcher=process_config,
+            scenario=inference_config,
+            backend=configs[backend],
+        )
+        benchmark_report = Benchmark.run(benchmark_config)
+        benchmark = Benchmark(config=benchmark_config, report=benchmark_report)
+        benchmark.push_to_hub(
+            repo_id=f"{username}/benchmarks",
+            subfolder=benchmark_name,
+            token=oauth_token.token,
+        )
+
+    return f"🚀 Benchmark {benchmark_name} has been pushed to {username}/benchmarks"
+
+
 with gr.Blocks() as demo:
+    # add login button
+    gr.LoginButton(min_width=250)
+
     # add image
     gr.Markdown(
         """<img src="https://huggingface.co/spaces/optimum/optimum-benchmark-ui/resolve/main/huggy_bench.png" style="display: block; margin-left: auto; margin-right: auto; width: 30%;">"""
     )
+
     # title text
-    gr.Markdown("<h1 style='text-align: center'>🤗 Optimum-Benchmark UI 🏋️</h1>")
+    gr.Markdown("<h1 style='text-align: center'>🤗 Optimum-Benchmark Interface 🏋️</h1>")
 
     # explanation text
     gr.HTML(
         "<h3 style='text-align: center'>"
-        "Zero code Gradio interface of <a href='https://github.com/huggingface/optimum-benchmark.git'>Optimum-Benchmark</a><br>"
+        "Zero code Gradio interface of "
+        "<a href='https://github.com/huggingface/optimum-benchmark.git'>"
+        "Optimum-Benchmark"
+        "</a>"
+        "<br>"
         "</h3>"
-        "<p style='text-align: center'>"
-        "Note: <a href='https://huggingface.co/spaces/optimum/optimum-benchmark-ui?duplicate=true'>Duplicate this space</a> and change its hardware to enable CUDA device<br>"
-        "or <a href='https://huggingface.co/spaces/optimum/optimum-benchmark-ui?docker=true'>Run with Docker</a> locally to target your own hardware."
-        "</p>"
     )
 
-    model = gr.Textbox(
+    model = gr.Dropdown(
         label="model",
-        value="distilbert-base-uncased-finetuned-sst-2-english",
-        info="Model to run the benchmark on. Press enter to infer the task automatically.",
+        choices=CHOSEN_MODELS,
+        value="bert-base-uncased",
+        info="Model to run the benchmark on.",
     )
     task = gr.Dropdown(
         label="task",
-        value="text-classification",
-        choices=list(TASKS_TO_AUTOMODELS.keys()),
-        info="Task to run the benchmark on. Can be infered automatically by submitting a model.",
-    )
-    device = gr.Dropdown(
-        value="cpu",
-        label="device",
-        choices=DEVICES,
-        info="Device to run the benchmark on. make sure to duplicate the space if you wanna run on CUDA devices.",
-    )
-    experiment = gr.Textbox(
-        label="experiment_name",
-        value=f"awesome-experiment-{random.randint(0, 100000)}",
-        info="Name of the experiment. Will be used to create a folder where results are stored.",
+        choices=CHOSEN_TASKS,
+        value="feature-extraction",
+        info="Task to run the benchmark on.",
     )
-    model.submit(fn=infer_task_from_model_name_or_path, inputs=model, outputs=task)
 
     with gr.Row():
-        with gr.Column():
-            with gr.Row():
-                backend = gr.Dropdown(
-                    label="backend",
-                    choices=BACKENDS,
-                    value=BACKENDS[0],
-                    info="Backend to run the benchmark on.",
-                )
-
-            with gr.Row() as backend_configs:
-                with gr.Accordion(label="backend options", open=False, visible=True):
-                    pytorch_config = get_pytorch_config()
-                with gr.Accordion(label="backend config", open=False, visible=False):
-                    onnxruntime_config = get_onnxruntime_config()
-                with gr.Accordion(label="backend config", open=False, visible=False):
-                    openvino_config = get_openvino_config()
-                with gr.Accordion(label="backend config", open=False, visible=False):
-                    neural_compressor_config = get_neural_compressor_config()
-
-        # hide backend configs based on backend
-        backend.change(
-            inputs=backend,
-            outputs=backend_configs.children,
-            fn=lambda value: [gr.update(visible=value == key) for key in BACKENDS],
-        )
+        with gr.Accordion(label="Process Config", open=False, visible=True):
+            process_config = get_process_config()
 
-        with gr.Column():
-            with gr.Row():
-                benchmark = gr.Dropdown(
-                    label="benchmark",
-                    choices=BENCHMARKS,
-                    value=BENCHMARKS[0],
-                    info="Type of benchmark to run.",
-                )
-
-            with gr.Row() as benchmark_configs:
-                with gr.Accordion(label="benchmark Config", open=False, visible=True):
-                    inference_config = get_inference_config()
-                with gr.Accordion(label="benchmark Config", open=False, visible=False):
-                    training_config = get_training_config()
-
-        # hide benchmark configs based on benchmark
-        benchmark.change(
-            inputs=benchmark,
-            outputs=benchmark_configs.children,
-            fn=lambda value: [gr.update(visible=value == key) for key in BENCHMARKS],
-        )
+    with gr.Row():
+        with gr.Accordion(label="PyTorch Config", open=True, visible=True):
+            pytorch_config = get_pytorch_config()
+        with gr.Accordion(label="OpenVINO Config", open=True, visible=True):
+            openvino_config = get_openvino_config()
+        with gr.Accordion(label="OnnxRuntime Config", open=True, visible=True):
+            onnxruntime_config = get_onnxruntime_config()
 
-    baseline = gr.Checkbox(
-        value=False,
-        label="compare_to_baseline",
-        info="Check this box to compare your chosen configuration to the baseline configuration.",
-    )
+    with gr.Row():
+        with gr.Accordion(label="Scenario Config", open=False, visible=True):
+            inference_config = get_inference_config()
 
     button = gr.Button(value="Run Benchmark", variant="primary")
-    with gr.Accordion(label="", open=True):
-        html_output = gr.HTML()
-        table_output = gr.Dataframe(visible=False)
+
+    html_output = gr.HTML()
 
     button.click(
         fn=run_benchmark,
         inputs={
-            experiment,
-            baseline,
-            model,
             task,
-            device,
-            backend,
-            benchmark,
-            *pytorch_config,
-            *openvino_config,
-            *onnxruntime_config,
-            *neural_compressor_config,
-            *inference_config,
-            *training_config,
+            model,
+            *process_config.values(),
+            *inference_config.values(),
+            *onnxruntime_config.values(),
+            *openvino_config.values(),
+            *pytorch_config.values(),
         },
-        outputs=[html_output, button, table_output],
-        queue=True,
-    )
-    button.click(
-        fn=lambda: f"awesome-experiment-{random.randint(0, 100000)}",
-        inputs=[],
-        outputs=experiment,
-        queue=True,
+        outputs=[html_output],
+        concurrency_limit=1,
     )
 
-demo.queue().launch()
+
+demo.queue(max_size=10).launch()

config_store.py

@@ -1,401 +1,127 @@
 import gradio as gr
 
 
-def get_base_backend_config(backend_name="pytorch"):
-    return [
-        # seed
-        gr.Textbox(
-            value=42,
-            label=f"{backend_name}.seed",
-            info="Sets seed for reproducibility",
-        ),
-        # inter_op_num_threads
-        gr.Textbox(
-            value="null",
-            label=f"{backend_name}.inter_op_num_threads",
-            info="Use null for default and -1 for cpu_count()",
-        ),
-        # intra_op_num_threads
-        gr.Textbox(
-            value="null",
-            label=f"{backend_name}.intra_op_num_threads",
-            info="Use null for default and -1 for cpu_count()",
-        ),
-        # initial_isolation_check
-        gr.Checkbox(
-            value=True,
-            label=f"{backend_name}.initial_isolation_check",
-            info="Makes sure that initially, no other process is running on the target device",
-        ),
-        # continous_isolation_check
-        gr.Checkbox(
-            value=True,
-            label=f"{backend_name}.continous_isolation_check",
-            info="Makes sure that throughout the benchmark, no other process is running on the target device",
-        ),
-        # delete_cache
-        gr.Checkbox(
+def get_process_config():
+    return {
+        "process.numactl": gr.Checkbox(
             value=False,
-            label=f"{backend_name}.delete_cache",
-            info="Deletes model cache (weights & configs) after benchmark is done",
+            label="process.numactl",
+            info="Runs the model with numactl",
         ),
-    ]
+        "process.numactl_kwargs": gr.Textbox(
+            value="",
+            label="process.numactl_kwargs",
+            info="Additional python dict of kwargs to pass to numactl",
+        ),
+    }
 
 
 def get_pytorch_config():
-    return get_base_backend_config(backend_name="pytorch") + [
-        # no_weights
-        gr.Checkbox(
-            value=False,
-            label="pytorch.no_weights",
-            info="Generates random weights instead of downloading pretrained ones",
-        ),
-        # # device_map
-        # gr.Dropdown(
-        #     value="null",
-        #
-        #     label="pytorch.device_map",
-        #     choices=["null", "auto", "sequential"],
-        #     info="Use null for default and `auto` or `sequential` the same way as in `from_pretrained`",
-        # ),
-        # torch_dtype
-        gr.Dropdown(
-            value="null",
+    return {
+        "pytorch.torch_dtype": gr.Dropdown(
+            value="float32",
             label="pytorch.torch_dtype",
-            choices=["null", "bfloat16", "float16", "float32", "auto"],
-            info="Use null for default and `auto` for automatic dtype selection",
-        ),
-        # amp_autocast
-        gr.Checkbox(
-            value=False,
-            label="pytorch.amp_autocast",
-            info="Enables Pytorch's native Automatic Mixed Precision",
-        ),
-        # amp_dtype
-        gr.Dropdown(
-            value="null",
-            label="pytorch.amp_dtype",
-            info="Use null for default",
-            choices=["null", "bfloat16", "float16"],
+            choices=["bfloat16", "float16", "float32", "auto"],
+            info="The dtype to use for the model",
         ),
-        # torch_compile
-        gr.Checkbox(
+        "pytorch.torch_compile": gr.Checkbox(
             value=False,
             label="pytorch.torch_compile",
             info="Compiles the model with torch.compile",
         ),
-        # bettertransformer
-        gr.Checkbox(
-            value=False,
-            label="pytorch.bettertransformer",
-            info="Applies optimum.BetterTransformer for fastpath anf optimized attention",
-        ),
-        # quantization_scheme
-        gr.Dropdown(
-            value="null",
-            choices=["null", "gptq", "bnb"],
-            label="pytorch.quantization_scheme",
-            info="Use null for no quantization",
-        ),
-        # # use_ddp
-        # gr.Checkbox(
-        #     value=False,
-        #
-        #     label="pytorch.use_ddp",
-        #     info="Uses DistributedDataParallel for multi-gpu training",
-        # ),
-        # peft_strategy
-        gr.Dropdown(
-            value="null",
-            choices=["null", "lora", "ada_lora", "prompt_tuning", "prefix_tuning", "p_tuning", "ia3"],
-            label="pytorch.peft_strategy",
-            info="Use null for no PEFT",
-        ),
-    ]
+    }
 
 
 def get_onnxruntime_config():
-    return get_base_backend_config(backend_name="onnxruntime") + [
-        # no_weights
-        gr.Checkbox(
-            value=False,
-            label="pytorch.no_weights",
-            info="Generates random weights instead of downloading pretrained ones",
-        ),
-        # export
-        gr.Checkbox(
+    return {
+        "onnxruntime.export": gr.Checkbox(
             value=True,
             label="onnxruntime.export",
             info="Exports the model to ONNX",
         ),
-        # use_cache
-        gr.Checkbox(
+        "onnxruntime.use_cache": gr.Checkbox(
             value=True,
             label="onnxruntime.use_cache",
             info="Uses cached ONNX model if available",
         ),
-        # use_merged
-        gr.Checkbox(
-            value=False,
+        "onnxruntime.use_merged": gr.Checkbox(
+            value=True,
             label="onnxruntime.use_merged",
             info="Uses merged ONNX model if available",
         ),
-        # torch_dtype
-        gr.Dropdown(
-            value="null",
+        "onnxruntime.torch_dtype": gr.Dropdown(
+            value="float32",
             label="onnxruntime.torch_dtype",
-            choices=["null", "bfloat16", "float16", "float32", "auto"],
-            info="Use null for default and `auto` for automatic dtype selection",
-        ),
-        # use_io_binding
-        gr.Checkbox(
-            value=True,
-            label="onnxruntime.use_io_binding",
-            info="Uses IO binding for inference",
-        ),
-        # auto_optimization
-        gr.Dropdown(
-            value="null",
-            label="onnxruntime.auto_optimization",
-            choices=["null", "O1", "O2", "O3", "O4"],
-            info="Use null for default",
-        ),
-        # auto_quantization
-        gr.Dropdown(
-            value="null",
-            label="onnxruntime.auto_quantization",
-            choices=["null", "arm64", "avx2", "avx512", "avx512_vnni", "tensorrt"],
-            info="Use null for default",
-        ),
-        # optimization
-        gr.Checkbox(
-            value=False,
-            label="onnxruntime.optimization",
-            info="Enables manual optimization",
-        ),
-        # optimization_config
-        gr.Dataframe(
-            type="array",
-            value=[["optimization_level"]],
-            headers=["1"],
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="onnxruntime.optimization_config",
-        ),
-        # quantization
-        gr.Checkbox(
-            value=False,
-            label="onnxruntime.quantization",
-            info="Enables manual quantization",
-        ),
-        # quantization_config
-        gr.Dataframe(
-            type="array",
-            value=[["is_static"]],
-            headers=[False],
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="onnxruntime.quantization_config",
-            info="Use null for default",
-        ),
-        # calibration
-        gr.Checkbox(
-            value=False,
-            label="onnxruntime.calibration",
-            info="Enables calibration",
+            choices=["bfloat16", "float16", "float32", "auto"],
+            info="The dtype to use for the model",
         ),
-        # calibration_config
-        gr.Dataframe(
-            type="array",
-            value=[["glue"]],
-            headers=["dataset_name"],
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="onnxruntime.calibration_config",
-            info="Use null for default",
-        ),
-        # peft_strategy
-        gr.Dropdown(
-            value="null",
-            label="onnxruntime.peft_strategy",
-            choices=["null", "lora", "ada_lora", "prompt_tuning", "prefix_tuning", "p_tuning", "ia3"],
-            info="Use null for full parameters fine-tuning",
-        ),
-    ]
+    }
 
 
 def get_openvino_config():
-    return get_base_backend_config(backend_name="openvino") + [
-        # export
-        gr.Checkbox(
+    return {
+        "openvino.export": gr.Checkbox(
             value=True,
             label="openvino.export",
             info="Exports the model to ONNX",
         ),
-        # use_cache
-        gr.Checkbox(
+        "openvino.use_cache": gr.Checkbox(
             value=True,
             label="openvino.use_cache",
             info="Uses cached ONNX model if available",
         ),
-        # use_merged
-        gr.Checkbox(
-            value=False,
+        "openvino.use_merged": gr.Checkbox(
+            value=True,
             label="openvino.use_merged",
             info="Uses merged ONNX model if available",
         ),
-        # reshape
-        gr.Checkbox(
+        "openvino.reshape": gr.Checkbox(
             value=False,
             label="openvino.reshape",
             info="Reshapes the model to the input shape",
         ),
-        # half
-        gr.Checkbox(
+        "openvino.half": gr.Checkbox(
             value=False,
             label="openvino.half",
             info="Converts model to half precision",
         ),
-        # quantization
-        gr.Checkbox(
-            value=False,
-            label="openvino.quantization",
-            info="Enables quantization",
-        ),
-        # quantization_config
-        gr.Dataframe(
-            type="array",
-            headers=["compression", "input_info", "save_onnx_model"],
-            value=[[None, None, None]],
-            row_count=(1, "static"),
-            col_count=(3, "dynamic"),
-            label="openvino.quantization_config",
-        ),
-        # calibration
-        gr.Checkbox(
-            value=False,
-            label="openvino.calibration",
-            info="Enables calibration",
-        ),
-        # calibration_config
-        gr.Dataframe(
-            type="array",
-            headers=["dataset_name"],
-            value=[["glue"]],
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="openvino.calibration_config",
-        ),
-    ]
-
-
-def get_neural_compressor_config():
-    return get_base_backend_config(backend_name="neural-compressor") + [
-        # ptq_quantization
-        gr.Checkbox(
-            value=False,
-            label="neural-compressor.ptq_quantization",
-            info="Enables post-training quantization",
-        ),
-        # ptq_quantization_config
-        gr.Dataframe(
-            type="array",
-            headers=["device"],
-            value=[["cpu"]],
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="neural-compressor.ptq_quantization_config",
-        ),
-        # calibration
-        gr.Checkbox(
-            value=False,
-            label="neural-compressor.calibration",
-            info="Enables calibration",
-        ),
-        # calibration_config
-        gr.Dataframe(
-            type="array",
-            headers=["dataset_name"],
-            value=[["glue"]],
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="neural-compressor.calibration_config",
-        ),
-    ]
+    }
 
 
 def get_inference_config():
-    return [
-        # duration
-        gr.Textbox(
+    return {
+        "inference.warmup_runs": gr.Slider(
+            step=1,
+            value=10,
+            minimum=0,
+            maximum=10,
+            label="inference.warmup_runs",
+            info="Number of warmup runs",
+        ),
+        "inference.duration": gr.Slider(
+            step=1,
             value=10,
+            minimum=0,
+            maximum=10,
             label="inference.duration",
-            info="Minimum duration of benchmark in seconds",
+            info="Minimum duration of the benchmark in seconds",
         ),
-        # warmup runs
-        gr.Textbox(
+        "inference.iterations": gr.Slider(
+            step=1,
             value=10,
-            label="inference.warmup_runs",
-            info="Number of warmup runs before measurements",
+            minimum=0,
+            maximum=10,
+            label="inference.iterations",
+            info="Minimum number of iterations of the benchmark",
         ),
-        # memory
-        gr.Checkbox(
-            value=False,
-            label="inference.memory",
-            info="Measures the peak memory footprint",
+        "inference.latency": gr.Checkbox(
+            value=True,
+            label="inference.latency",
+            info="Measures the latency of the model",
         ),
-        # energy
-        gr.Checkbox(
+        "inference.memory": gr.Checkbox(
             value=False,
-            label="inference.energy",
-            info="Measures energy consumption and carbon emissions",
-        ),
-        # input_shapes
-        gr.Dataframe(
-            type="array",
-            value=[[2, 16]],
-            row_count=(1, "static"),
-            col_count=(2, "dynamic"),
-            label="inference.input_shapes",
-            headers=["batch_size", "sequence_length"],
-            info="Controllable input shapes, add more columns for more inputs",
-        ),
-        # forward kwargs
-        gr.Dataframe(
-            type="array",
-            value=[[False]],
-            headers=["return_dict"],
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="inference.forward_kwargs",
-            info="Keyword arguments for the forward pass, add more columns for more arguments",
-        ),
-    ]
-
-
-def get_training_config():
-    return [
-        # warmup steps
-        gr.Textbox(
-            value=40,
-            label="training.warmup_steps",
-        ),
-        # dataset_shapes
-        gr.Dataframe(
-            type="array",
-            value=[[500, 16]],
-            headers=["dataset_size", "sequence_length"],
-            row_count=(1, "static"),
-            col_count=(2, "dynamic"),
-            label="training.dataset_shapes",
-        ),
-        # training_arguments
-        gr.Dataframe(
-            value=[[2]],
-            type="array",
-            row_count=(1, "static"),
-            col_count=(1, "dynamic"),
-            label="training.training_arguments",
-            headers=["per_device_train_batch_size"],
+            label="inference.memory",
+            info="Measures the peak memory consumption",
         ),
-    ]
+    }

configs/base_config.yaml

@@ -1,15 +0,0 @@
-defaults:
-  - backend: null # default backend
-  - benchmark: null # default benchmark
-  - experiment # inheriting experiment schema
-  - _self_ # for hydra 1.1 compatibility
-  - override hydra/job_logging: colorlog # colorful logging
-  - override hydra/hydra_logging: colorlog # colorful logging
-
-hydra:
-  run:
-    dir: runs/${experiment_name}
-  job:
-    chdir: true
-    env_set:
-      CUDA_VISIBLE_DEVICES: 0

requirements.txt

@@ -1 +1 @@
-ansi2html==1.8.0
+optimum-benchmark[oprnvino,onnxruntime,ipex]@git+https://github.com/huggingface/optimum-benchmark.git

run.py

@@ -1,189 +0,0 @@
-import subprocess
-import gradio as gr
-import pandas as pd
-from ansi2html import Ansi2HTMLConverter
-
-ansi2html_converter = Ansi2HTMLConverter(inline=True)
-
-
-def run_benchmark(kwargs):
-    for key, value in kwargs.copy().items():
-        if key.label == "compare_to_baseline":
-            baseline = value
-            kwargs.pop(key)
-        elif key.label == "experiment_name":
-            experiment_name = value
-            kwargs.pop(key)
-        elif key.label == "model":
-            model = value
-            kwargs.pop(key)
-        elif key.label == "task":
-            task = value
-            kwargs.pop(key)
-        elif key.label == "device":
-            device = value
-            kwargs.pop(key)
-        elif key.label == "backend":
-            backend = value
-            kwargs.pop(key)
-        elif key.label == "benchmark":
-            benchmark = value
-            kwargs.pop(key)
-        else:
-            continue
-
-    if baseline:
-        baseline_arguments = [
-            "optimum-benchmark",
-            "--config-dir",
-            "./configs",
-            "--config-name",
-            "base_config",
-            f"backend=pytorch",
-            f"task={task}",
-            f"model={model}",
-            f"device={device}",
-            f"benchmark={benchmark}",
-            f"experiment_name=baseline_{experiment_name}",
-        ]
-        for component, value in kwargs.items():
-            if f"{benchmark}." in component.label:
-                label = component.label.replace(f"{benchmark}.", "benchmark.")
-                if isinstance(component, gr.Dataframe):
-                    for sub_key, sub_value in zip(component.headers, value[0]):
-                        baseline_arguments.append(f"++{label}.{sub_key}={sub_value}")
-                else:
-                    baseline_arguments.append(f"{label}={value}")
-
-        # yield from run_experiment(baseline_arguments) but get the return code
-        baseline_return_code, html_text = yield from run_experiment(baseline_arguments, "")
-        if baseline_return_code is not None and baseline_return_code != 0:
-            yield gr.update(value=html_text), gr.update(interactive=True), gr.update(visible=False)
-            return
-    else:
-        html_text = ""
-
-    arguments = [
-        "optimum-benchmark",
-        "--config-dir",
-        "./configs",
-        "--config-name",
-        "base_config",
-        f"task={task}",
-        f"model={model}",
-        f"device={device}",
-        f"backend={backend}",
-        f"benchmark={benchmark}",
-        f"experiment_name={experiment_name}",
-    ]
-    for component, value in kwargs.items():
-        if f"{backend}." in component.label or f"{benchmark}." in component.label:
-            label = component.label.replace(f"{backend}.", "backend.").replace(f"{benchmark}.", "benchmark.")
-
-            if isinstance(component, gr.Dataframe):
-                for sub_key, sub_value in zip(component.headers, value[0]):
-                    arguments.append(f"++{label}.{sub_key}={sub_value}")
-            else:
-                arguments.append(f"{label}={value}")
-
-    return_code, html_text = yield from run_experiment(arguments, html_text)
-    if return_code is not None and return_code != 0:
-        yield gr.update(value=html_text), gr.update(interactive=True), gr.update(visible=False)
-        return
-
-    if baseline:
-        baseline_table = pd.read_csv(f"runs/baseline_{experiment_name}/{benchmark}_results.csv", index_col=0)
-        table = pd.read_csv(f"runs/{experiment_name}/{benchmark}_results.csv", index_col=0)
-        # concat tables
-        table = pd.concat([baseline_table, table], axis=0)
-        table = postprocess_table(table, experiment_name)
-    else:
-        table = pd.read_csv(f"runs/{experiment_name}/{benchmark}_results.csv", index_col=0)
-
-    table_update = gr.update(visible=True, value={"headers": list(table.columns), "data": table.values.tolist()})
-    yield gr.update(value=html_text), gr.update(interactive=True), table_update
-    return
-
-
-def run_experiment(args, html_text=""):
-    command = "<br>".join(args)
-    html_text += f"<h3>Running command:</h3>{command}"
-    yield gr.update(value=html_text), gr.update(interactive=False), gr.update(visible=False)
-
-    # stream subprocess output
-    process = subprocess.Popen(
-        args,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        universal_newlines=True,
-    )
-
-    curr_ansi_text = ""
-    for ansi_line in iter(process.stdout.readline, ""):
-        if process.returncode is not None and process.returncode != 0:
-            break
-
-        # stream process output to stdout
-        print(ansi_line, end="")
-        # skip torch.distributed.nn.jit.instantiator messages
-        if "torch.distributed.nn.jit.instantiator" in ansi_line:
-            continue
-        # process download messages
-        if "Downloading " in curr_ansi_text and "Downloading " in ansi_line:
-            curr_ansi_text = curr_ansi_text.split("\n")[:-2]
-            print(curr_ansi_text)
-            curr_ansi_text.append(ansi_line)
-            curr_ansi_text = "\n".join(curr_ansi_text)
-        else:
-            # append line to ansi text
-            curr_ansi_text += ansi_line
-        # convert ansi to html
-        curr_html_text = ansi2html_converter.convert(curr_ansi_text)
-        # stream html output to gradio
-        cumul_html_text = html_text + "<br><h3>Streaming logs:</h3>" + curr_html_text
-        yield gr.update(value=cumul_html_text), gr.update(interactive=False), gr.update(visible=False)
-
-    return process.returncode, cumul_html_text
-
-
-def postprocess_table(table, experiment_name):
-    table["experiment_name"] = ["baseline", experiment_name]
-    table = table.set_index("experiment_name")
-    table.reset_index(inplace=True)
-    if "forward.latency(s)" in table.columns:
-        table["forward.latency.reduction(%)"] = (
-            table["forward.latency(s)"] / table["forward.latency(s)"].iloc[0] - 1
-        ) * 100
-        table["forward.latency.reduction(%)"] = table["forward.latency.reduction(%)"].round(2)
-
-    if "forward.throughput(samples/s)" in table.columns:
-        table["forward.throughput.speedup(%)"] = (
-            table["forward.throughput(samples/s)"] / table["forward.throughput(samples/s)"].iloc[0] - 1
-        ) * 100
-        table["forward.throughput.speedup(%)"] = table["forward.throughput.speedup(%)"].round(2)
-
-    if "forward.peak_memory(MB)" in table.columns:
-        table["forward.peak_memory.reduction(%)"] = (
-            table["forward.peak_memory(MB)"] / table["forward.peak_memory(MB)"].iloc[0] - 1
-        ) * 100
-        table["forward.peak_memory.reduction(%)"] = table["forward.peak_memory.reduction(%)"].round(2)
-
-    if "generate.latency(s)" in table.columns:
-        table["generate.latency.reduction(%)"] = (
-            table["generate.latency(s)"] / table["generate.latency(s)"].iloc[0] - 1
-        ) * 100
-        table["generate.latency.reduction(%)"] = table["generate.latency.reduction(%)"].round(2)
-
-    if "generate.throughput(tokens/s)" in table.columns:
-        table["generate.throughput.speedup(%)"] = (
-            table["generate.throughput(tokens/s)"] / table["generate.throughput(tokens/s)"].iloc[0] - 1
-        ) * 100
-        table["generate.throughput.speedup(%)"] = table["generate.throughput.speedup(%)"].round(2)
-
-    if "generate.peak_memory(MB)" in table.columns:
-        table["generate.peak_memory.reduction(%)"] = (
-            table["generate.peak_memory(MB)"] / table["generate.peak_memory(MB)"].iloc[0] - 1
-        ) * 100
-        table["generate.peak_memory.reduction(%)"] = table["generate.peak_memory.reduction(%)"].round(2)
-
-    return table