Spaces:
Running
Running
| import subprocess | |
| import gradio as gr | |
| import pandas as pd | |
| from ansi2html import Ansi2HTMLConverter | |
| ansi2html_converter = Ansi2HTMLConverter(inline=True) | |
| def run_benchmark(kwargs): | |
| for key, value in kwargs.copy().items(): | |
| if key.label == "experiment_name": | |
| experiment_name = value | |
| kwargs.pop(key) | |
| elif key.label == "model": | |
| model = value | |
| kwargs.pop(key) | |
| elif key.label == "task": | |
| task = value | |
| kwargs.pop(key) | |
| elif key.label == "device": | |
| device = value | |
| kwargs.pop(key) | |
| elif key.label == "backend": | |
| backend = value | |
| kwargs.pop(key) | |
| elif key.label == "benchmark": | |
| benchmark = value | |
| kwargs.pop(key) | |
| else: | |
| continue | |
| arguments = [ | |
| "optimum-benchmark", | |
| "--config-dir", | |
| "./", | |
| "--config-name", | |
| "base_config", | |
| f"task={task}", | |
| f"model={model}", | |
| f"device={device}", | |
| f"backend={backend}", | |
| f"benchmark={benchmark}", | |
| f"experiment_name={experiment_name}", | |
| ] | |
| for component, value in kwargs.items(): | |
| if f"{backend}." in component.label or f"{benchmark}." in component.label: | |
| label = component.label.replace(f"{backend}.", "backend.").replace(f"{benchmark}.", "benchmark.") | |
| if isinstance(component, gr.Dataframe): | |
| for sub_key, sub_value in zip(component.headers, value[0]): | |
| arguments.append(f"++{label}.{sub_key}={sub_value}") | |
| else: | |
| arguments.append(f"{label}={value}") | |
| command = " ".join(arguments) | |
| yield gr.update(value=command), gr.update(interactive=False), gr.update(visible=False) | |
| # stream subprocess output | |
| process = subprocess.Popen( | |
| arguments, | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT, | |
| universal_newlines=True, | |
| ) | |
| ansi_text = "" | |
| for ansi_line in iter(process.stdout.readline, ""): | |
| # stream process output to stdout | |
| print(ansi_line, end="") | |
| # skip torch.distributed.nn.jit.instantiator messages | |
| if "torch.distributed.nn.jit.instantiator" in ansi_line: | |
| continue | |
| # if the last message is a download message (contains "Downloading ") then remove it and replace it with a new one | |
| if "Downloading " in ansi_text and "Downloading " in ansi_line: | |
| ansi_text = ansi_text.split("\n")[:-2] | |
| print(ansi_text) | |
| ansi_text.append(ansi_line) | |
| ansi_text = "\n".join(ansi_text) | |
| else: | |
| # append line to ansi text | |
| ansi_text += ansi_line | |
| # convert ansi to html | |
| html_text = ansi2html_converter.convert(ansi_text) | |
| # stream html output to gradio | |
| yield gr.update(value=html_text), gr.update(interactive=False), gr.update(visible=False) | |
| # read runs/{experiment_name}/{benchmark}_results.csv | |
| table = pd.read_csv(f"runs/{experiment_name}/{benchmark}_results.csv", index_col=0) | |
| print(table.to_dict("records")) | |
| yield gr.update(value=html_text), gr.update(interactive=True), gr.Dataframe.update( | |
| visible=True, value={"headers": list(table.columns), "data": table.values.tolist()} | |
| ) | |