Add MFU and MBU

app.py

@@ -2,6 +2,7 @@
 import os
 import datetime
 import socket
+import base64
 from threading import Thread
 
 import gradio as gr
@@ -20,6 +21,7 @@ from src.display.about import (
     LLM_BENCHMARKS_DETAILS,
     FAQ_TEXT,
     TITLE,
+    ACKNOWLEDGEMENT_TEXT,
 )
 
 from src.display.css_html_js import custom_css
@@ -89,6 +91,17 @@ def init_space():
         EVAL_REQUESTS_PATH, EVAL_COLS
     )
     return dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+    
+    
+def add_benchmark_columns(shown_columns):
+    benchmark_columns = []
+    for benchmark in BENCHMARK_COLS:
+        if benchmark in shown_columns:
+            for c in COLS:
+                if benchmark in c and benchmark != c:
+                    benchmark_columns.append(c)
+    return benchmark_columns
+
 
 # Searching and filtering
 def update_table(
@@ -96,7 +109,8 @@ def update_table(
 ):
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query)
     filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
+    benchmark_columns = add_benchmark_columns(columns)
+    df = select_columns(filtered_df, columns + benchmark_columns)
     return df
 
 
@@ -204,10 +218,21 @@ def load_query(request: gr.Request):
     return query
 
 
+def get_image_html(url, image_path):
+    with open(image_path, "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read()).decode()
+    return f'<a href="{url}" target="_blank"><img src="data:image/jpg;base64,{encoded_string}" alt="NetMind.AI Logo" style="width:100pt;"></a>'
+
+
+# Prepare the HTML content with the image
+image_html = get_image_html("https://netmind.ai/home", "./src/display/imgs/Netmind.AI_LOGO.jpg")
+
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    gr.HTML(ACKNOWLEDGEMENT_TEXT.format(image_html=image_html))
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("open-moe-llm-leaderboard", elem_id="llm-benchmark-tab-table", id=0):
@@ -270,18 +295,19 @@ with demo:
                     # )
 
             # breakpoint()
-
+            benchmark_columns = add_benchmark_columns(shown_columns.value)
             leaderboard_table = gr.components.Dataframe(
                 value=(
                     leaderboard_df[
                         [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                         + shown_columns.value
+                        + benchmark_columns
                         + [AutoEvalColumn.dummy.name]
                     ]
                     if leaderboard_df.empty is False
                     else leaderboard_df
                 ),
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value + benchmark_columns,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
@@ -313,7 +339,7 @@ with demo:
             demo.load(load_query, inputs=[], outputs=[search_bar])
 
             for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
-                selector.select(
+                selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,

backend-cli.py

@@ -17,7 +17,7 @@ from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
 from src.envs import QUEUE_REPO, RESULTS_REPO, API, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
-from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus
+from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus, get_gpu_details
 
 from src.leaderboard.read_evals import get_raw_eval_results
 
@@ -28,6 +28,8 @@ import time
 import pprint
 import logging
 
+from lm_eval.filters.extraction import RegexFilter
+
 
 # Configure the root logger
 logging.basicConfig(
@@ -42,6 +44,20 @@ eval_logger = logging.getLogger("lm-eval")
 # Explicitly set the level for 'lm-eval' logger to WARNING
 eval_logger.setLevel(logging.WARNING)
 
+def tuple_input_decorator(func):
+    def wrapper(self, resps, docs):
+        stripped_resps = [[resp_data[0] for resp_data in group] for group in resps]
+
+        filtered_resps = func(self, stripped_resps, docs)
+        
+        combined_resps = []
+        for original_group, new_group in zip(resps, filtered_resps):
+            combined_group = [(new_resp,) + rest_of_data[1:] for new_resp, rest_of_data in zip(new_group, original_group)]
+            combined_resps.append(combined_group)
+
+        return combined_resps
+    return wrapper
+
 
 def my_set_eval_request(api, eval_request, set_to_status, hf_repo, local_dir):
     for i in range(10):
@@ -126,9 +142,6 @@ def request_to_result_name(request: EvalRequest) -> str:
 def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[int] = None) -> dict:
     batch_size = 1
     batch_size = eval_request.batch_size
-    
-    if args.debug:
-        RESULTS_REPO = DEBUG_RESULTS_REPO
 
     init_gpu_info = analyze_gpu_stats(parse_nvidia_smi())
     # if init_gpu_info['Mem(M)'] > 500:
@@ -137,6 +150,12 @@ def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[in
     stop_event = threading.Event()
     monitor_thread = threading.Thread(target=monitor_gpus, args=(stop_event, 5, gpu_stats_list))
     monitor_thread.start()
+    
+    original_apply = RegexFilter.apply
+    if task.benchmark in ["gsm8k", "gsm8k_cot", "gsm8k_cot_self_consistency", "gsm8k_custom"]:
+        RegexFilter.apply = tuple_input_decorator(RegexFilter.apply)
+    else:
+        RegexFilter.apply = original_apply
 
     try:
         results = run_evaluation(
@@ -198,6 +217,8 @@ def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[in
         repo_id=RESULTS_REPO,
         repo_type="dataset",
     )
+    
+    RegexFilter.apply = original_apply
     return results
 
 
@@ -366,21 +387,7 @@ def maybe_refresh_results(thr: int, hard_task_lst: Optional[list[str]] = None) -
 
     return False
 
-
-def get_gpu_details():
-    gpus = GPUtil.getGPUs()
-    gpu = gpus[0]
-    name = gpu.name.replace(" ", "-")
-    # Convert memory from MB to GB and round to nearest whole number
-    memory_gb = round(gpu.memoryTotal / 1024)
-    memory = f"{memory_gb}GB"
-    formatted_name = f"{name}-{memory}"
-    return formatted_name
-
 def process_pending_requests() -> bool:
-    if args.debug:
-        QUEUE_REPO = DEBUG_QUEUE_REPO
-        
     sanity_checks()
     print("Processing pending requests")
     current_pending_status = [PENDING_STATUS]
@@ -443,13 +450,14 @@ def get_args():
     parser = argparse.ArgumentParser(description="Run the backend")
     parser.add_argument("--debug", action="store_true", help="Run in debug mode")
     # debug parameters
-    parser.add_argument("--task", type=str, default="selfcheckgpt,mmlu", help="Task to debug")
+    parser.add_argument("--task", type=str, default="selfcheckgpt,mmlu, gsm8k", help="Task to debug")
     parser.add_argument("--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1,mistralai/Mixtral-8x7B-v0.1", help="Model to debug")
     parser.add_argument("--precision", type=str, default="float32,float16,8bit,4bit", help="Precision to debug")
     parser.add_argument("--inference-framework", type=str, default="hf-chat", help="Inference framework to debug")
     parser.add_argument("--limit", type=int, default=None, help="Limit for the number of samples")
     parser.add_argument("--gpu-type", type=str, default="NVIDIA-A100-PCIe-80GB", 
                         help="GPU type. NVIDIA-A100-PCIe-80GB; NVIDIA-RTX-A5000-24GB; NVIDIA-H100-PCIe-80GB")
+    parser.add_argument("--debug_repo", action="store_true", help="Use debug repo")
     return parser.parse_args()
 
 
@@ -457,7 +465,7 @@ if __name__ == "__main__":
     args = get_args()
     local_debug = args.debug
     # debug specific task by ping
-    if local_debug:
+    if local_debug and not args.debug_repo:
         # debug_model_names = [args.model]  # Use model from arguments
         # debug_task_name = [args.task]  # Use task from arguments
         debug_model_names = args.model.split(",")
@@ -465,48 +473,67 @@ if __name__ == "__main__":
         precisions = args.precision.split(",")
         print(f"debug_model_names: {debug_model_names}, debug_task_name: {debug_task_name}, precisions: {precisions}")
         task_lst = TASKS_HARNESS.copy()
+        RESULTS_REPO = DEBUG_RESULTS_REPO
         for precision in precisions:
             for debug_model_name in debug_model_names:
                 for task in task_lst:
                     task_name = task.benchmark
                     if task_name not in debug_task_name:
                         continue
-                    try:
-                        eval_request = EvalRequest(
-                            model=debug_model_name,
-                            private=False,
-                            status="",
-                            json_filepath="",
-                            precision=precision,  # Use precision from arguments
-                            inference_framework=args.inference_framework,  # Use inference framework from arguments
-                            gpu_type=args.gpu_type
-                        )
-                        curr_gpu_type = get_gpu_details()
-                        if eval_request.gpu_type != curr_gpu_type:
-                            print(f"GPU type mismatch: {eval_request.gpu_type} vs {curr_gpu_type}")
-                            raise Exception("GPU type mismatch")
-                        results = process_evaluation(task, eval_request, limit=args.limit)
-                    except Exception as e:
-                        print(f"debug running error: {e}")
-    else:
+                    # try:
+                    eval_request = EvalRequest(
+                        model=debug_model_name,
+                        private=False,
+                        status="",
+                        json_filepath="",
+                        precision=precision,  # Use precision from arguments
+                        inference_framework=args.inference_framework,  # Use inference framework from arguments
+                        gpu_type=args.gpu_type
+                    )
+                    curr_gpu_type = get_gpu_details()
+                    if eval_request.gpu_type != curr_gpu_type:
+                        print(f"GPU type mismatch: {eval_request.gpu_type} vs {curr_gpu_type}")
+                        raise Exception("GPU type mismatch")
+                    results = process_evaluation(task, eval_request, limit=args.limit)
+                    # except Exception as e:
+                    #     print(f"debug running error: {e}")
+    elif local_debug and args.debug_repo:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+        RESULTS_REPO = DEBUG_RESULTS_REPO
         while True:
             res = False
-
             # if random.randint(0, 10) == 0:
             res = process_pending_requests()
             print(f"waiting for 60 seconds")
             time.sleep(60)
-
             # if res is False:
             #     if random.randint(0, 5) == 0:
             #         res = maybe_refresh_results(100)
             #     else:
             #         res = process_finished_requests(100)
-
             # time.sleep(60)
-
             # if res is False:
             #     if random.randint(0, 5) == 0:
             #         res = maybe_refresh_results(0)
             #     else:
             #         res = process_finished_requests(0)
+    elif not local_debug and not args.debug_repo:
+        while True:
+           res = False
+           # if random.randint(0, 10) == 0:
+           res = process_pending_requests()
+           print(f"waiting for 60 seconds")
+           time.sleep(60)
+           # if res is False:
+           #     if random.randint(0, 5) == 0:
+           #         res = maybe_refresh_results(100)
+           #     else:
+           #         res = process_finished_requests(100)
+           # time.sleep(60)
+           # if res is False:
+           #     if random.randint(0, 5) == 0:
+           #         res = maybe_refresh_results(0)
+           #     else:
+           #         res = process_finished_requests(0)
+    else:
+        raise Exception("Cannot use debug_repo without local debug flag")

requirements.txt

@@ -30,4 +30,5 @@ evaluate
 spacy==3.7.4
 selfcheckgpt
 immutabledict
-gputil
+gputil
+bitsandbytes

src/backend/envs.py

@@ -57,6 +57,7 @@ class Tasks(Enum):
 
     # task20 = Task("race", "acc", "RACE", 0)
     task21 = Task("mmlu", "acc", "MMLU", 5)
+    task22 = Task("gsm8k_custom", "em", "GSM8K", 5)
 
 
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")

src/backend/hflm_with_measurement.py

@@ -37,6 +37,9 @@ from lm_eval.models.utils import (
     stop_sequences_criteria,
 )
 from lm_eval.models.huggingface import HFLM
+from src.utils import get_gpu_number, get_gpu_details, get_peak_bw, transfer_precision2bytes, get_peak_flops
+from src.submission.check_validity import get_model_size
+from src.envs import API
 
 
 class StopWatch(TextStreamer):
@@ -67,6 +70,9 @@ class StopWatch(TextStreamer):
 class HFLMWithMeasurement(HFLM):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self.pretrained = kwargs.get("pretrained", None)
+        self.revision = kwargs.get("revision", None)
+        self.precision = kwargs.get("dtype", None)
 
     def _loglikelihood_tokens(
         self,
@@ -288,13 +294,15 @@ class HFLMWithMeasurement(HFLM):
 
         return re_ord.get_original(res)
 
-    def _model_generate(self, context, max_length, stop, **generation_kwargs):
+    def _model_generate(self, context, max_tokens, stop, **generation_kwargs):
         # temperature = 0.0 if not set
         # if do_sample is false and temp==0.0:
         # remove temperature, as do_sample=False takes care of this
         # and we don't want a warning from HF
         generation_kwargs["temperature"] = generation_kwargs.get("temperature", 0.0)
         do_sample = generation_kwargs.get("do_sample", None)
+        
+        # is_gsm8k = generation_kwargs.get("is_gsm8k", False)
 
         # The temperature has to be a strictly positive float -- if it is 0.0, use greedy decoding strategies
         if generation_kwargs.get("temperature") == 0.0 and do_sample is None:
@@ -302,7 +310,21 @@ class HFLMWithMeasurement(HFLM):
 
         if do_sample is False and generation_kwargs.get("temperature") == 0.0:
             generation_kwargs.pop("temperature")
-        # build stopping criteria
+        
+        # if is_gsm8k:
+        #     generation_kwargs.pop("is_gsm8k")
+            
+        context_length = context.shape[1]
+        model_config = self.model.config
+        
+        if not self.precision:
+            if model_config.quantization_config._load_in_4bit:
+                self.precision = "4bit"
+            elif model_config.quantization_config._load_in_8bit:
+                self.precision = "8bit"
+            else:
+                raise ValueError("Unknown precision")
+
         stopping_criteria = stop_sequences_criteria(
             self.tokenizer, stop, context.shape[1], context.shape[0]
         )
@@ -310,7 +332,7 @@ class HFLMWithMeasurement(HFLM):
         start = time()
         res = self.model.generate(
             input_ids=context,
-            max_length=max_length,
+            max_new_tokens=max_tokens,
             stopping_criteria=stopping_criteria,
             pad_token_id=self.tokenizer.pad_token_id,
             use_cache=True,
@@ -321,12 +343,68 @@ class HFLMWithMeasurement(HFLM):
 
         batch_size = context.shape[0]
         output_length = stop_watch.decoding_iterations
+        
+        precision_bytes = transfer_precision2bytes(self.precision)
+        
+        model_info = API.model_info(repo_id=self.pretrained, revision=self.revision)
+        model_size_param = get_model_size(model_info=model_info, precision=self.precision)
+
+        n_layers = model_config.num_hidden_layers if hasattr(model_config, "num_hidden_layers") else model_config.num_layers
+        d_model = model_config.hidden_size if hasattr(model_config, "hidden_size") else model_config.d_model
+        
+        if hasattr(model_config, "num_experts_per_tok"):
+            n_experts_per_tok = model_config.num_experts_per_tok
+        elif hasattr(model_config, "num_selected_experts"):
+            n_experts_per_tok = model_config.num_selected_experts
+        else:
+            n_experts_per_tok = 1
+        
+        if hasattr(model_config, "ffn_dim"):
+            d_ff = model_config.ffn_dim
+        elif hasattr(model_config, "intermediate_size"):
+            d_ff = model_config.intermediate_size
+        elif hasattr(model_config, "d_ff"):
+            d_ff = model_config.d_ff
+        else:
+            raise ValueError("Unknown ffn dim model configuration")
+        
+        if hasattr(model_config, "num_local_experts"):
+            num_experts = model_config.num_local_experts
+        elif hasattr(model_config, "num_experts"):
+            num_experts = model_config.num_experts
+        else:
+            num_experts = 1
+            
+        ffn_params = n_layers * d_ff * 2 * d_model
+        
+        shared_params = model_size_param * 1e9 - num_experts * ffn_params
+
+        model_size = shared_params + n_experts_per_tok * ffn_params
+
+        per_token_kv_size = 2 * n_layers * d_model * precision_bytes
+        
+        peak_bw_single = get_peak_bw(get_gpu_details())
+        peak_bw = peak_bw_single * get_gpu_number()
+        
+        kv_size = (output_length - 1) * per_token_kv_size / 1e9
 
         end_to_end_time = (end - start) / batch_size
         prefilling_time = stop_watch.prefilling_time / batch_size
         decoding_time = stop_watch.decoding_time / batch_size
         token_per_sec = output_length / decoding_time
-        return res, end_to_end_time, prefilling_time, token_per_sec
+        ach_mem_bw = (model_size * precision_bytes / 1e9 + kv_size) * token_per_sec
+        
+        flops_per_token = 2 * model_size + 2 * n_layers * context_length * d_model
+        peak_flops_single = get_peak_flops(get_gpu_details(), self.precision)
+        peak_flops = peak_flops_single * get_gpu_number()
+        
+        ## TODO only support llama-type decoder only models and moe models of switch transformer and mixtrial
+        mfu = token_per_sec * flops_per_token / peak_flops
+        mbu = ach_mem_bw / peak_bw
+        
+        # print(f"mfu: {mfu}, mbu: {mbu}")
+        
+        return res, end_to_end_time, prefilling_time, token_per_sec, mfu, mbu
 
     def generate_until(
         self, requests: List[Instance], disable_tqdm: bool = False
@@ -403,11 +481,19 @@ class HFLMWithMeasurement(HFLM):
                     f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}"
                 )
             # add EOS token to stop sequences
-            eos = self.tok_decode(self.eot_token_id)
+            eos = "<|eot_id|>"
             if not until:
                 until = [eos]
             else:
                 until.append(eos)
+            
+            # is_gsm8k = kwargs.get("is_gsm8k", False)
+            # if is_gsm8k:
+            #     until = ["Question:", "Question", "</s>"]
+            #     eos_ids = [self.tokenizer.eos_token_id, 
+            #              self.tokenizer.convert_tokens_to_ids("<|eot_id|>")]
+                
+                    
             if "max_gen_toks" in kwargs.keys():
                 max_gen_toks = kwargs.pop("max_gen_toks")
             else:
@@ -427,14 +513,16 @@ class HFLMWithMeasurement(HFLM):
                 left_truncate_len=max_ctx_len,
                 truncation=self.truncation,
             )
+            
+            # print("context: ", self.tok_decode(context_enc[0]))
             context_enc = context_enc.to(self.device)
             attn_masks = attn_masks.to(self.device)
 
-            if "max_length" not in kwargs:
-                kwargs["max_length"] = context_enc.shape[1] + max_gen_toks
+            if "max_tokens" not in kwargs:
+                kwargs["max_tokens"] = max_gen_toks
 
             # perform batched generation
-            cont, end_to_end_time, prefilling_time, token_per_sec = self._model_generate(
+            cont, end_to_end_time, prefilling_time, token_per_sec, mfu, mbu = self._model_generate(
                 context=context_enc,
                 attention_mask=attn_masks,
                 stop=until,
@@ -445,18 +533,21 @@ class HFLMWithMeasurement(HFLM):
             for cont_toks, context in zip(cont_toks_list, contexts):
                 # discard context + left-padding toks if using causal decoder-only LM
                 if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
+                    # print("After Generation: ", self.tok_decode(cont_toks))
                     cont_toks = cont_toks[context_enc.shape[1] :]
-
+                
                 s = self.tok_decode(cont_toks)
 
-                # use secondary stop seqs to cut off should-have-been-stopped content post-hoc
+                # # use secondary stop seqs to cut off should-have-been-stopped content post-hoc
+                # if not is_gsm8k:
                 for term in until:
                     if len(term) > 0:
                         # ignore '' separator,
                         # for seq2seq case where self.tok_decode(self.eot_token_id) = ''
                         s = s.split(term)[0]
-
-                res.append((s, end_to_end_time, prefilling_time, token_per_sec))
+                
+                # print(s)
+                res.append((s, end_to_end_time, prefilling_time, token_per_sec, mfu, mbu))
 
                 self.cache_hook.add_partial("generate_until", (context, gen_kwargs), s)
                 pbar.update(1)

src/backend/run_eval_suite.py

@@ -17,12 +17,16 @@ def process_results_decorator(func):
         end_to_end_time = sum([r[1] for r in results]) / len(results)
         prefilling_time = sum([r[2] for r in results]) / len(results)
         decoding_throughput = sum([r[3] for r in results]) / len(results)
+        mfu = sum([r[4] for r in results]) / len(results)
+        mbu = sum([r[5] for r in results]) / len(results)
         # print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
 
         result_dict = func(self, doc, processed_results, *args, **kwargs)
         result_dict["end_to_end_time"] = end_to_end_time
         result_dict["prefilling_time"] = prefilling_time
         result_dict["decoding_throughput"] = decoding_throughput
+        result_dict["mfu"] = mfu * 100
+        result_dict["mbu"] = mbu * 100
         return result_dict
     return wrapper
 ConfigurableTask.process_results = process_results_decorator(orig_process_results)
@@ -33,6 +37,8 @@ def aggregation_decorator(func):
         aggregation_list["end_to_end_time"] = mean
         aggregation_list["prefilling_time"] = mean
         aggregation_list["decoding_throughput"] = mean
+        aggregation_list["mfu"] = mean
+        aggregation_list["mbu"] = mean
         return aggregation_list
     return wrapper
 ConfigurableTask.aggregation = aggregation_decorator(orig_aggregation)
@@ -43,6 +49,8 @@ def higher_is_better_decorator(func):
         higher_is_better_dict["end_to_end_time"] = False
         higher_is_better_dict["prefilling_time"] = False
         higher_is_better_dict["decoding_throughput"] = True
+        higher_is_better_dict["mfu"] = True
+        higher_is_better_dict["mbu"] = True
         return higher_is_better_dict
     return wrapper
 ConfigurableTask.higher_is_better = higher_is_better_decorator(orig_higher_is_better)

src/backend/tasks/gsm8k/gsm8k-custom.yaml

@@ -0,0 +1,47 @@
+group:
+  - math_word_problems
+task: gsm8k_custom
+dataset_path: gsm8k
+dataset_name: main
+output_type: generate_until
+training_split: train
+fewshot_split: train
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{answer}}" #" {{answer.split('### ')[-1].rstrip()}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: false
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+      - "(?s).*#### "
+      - "\\.$"
+generation_kwargs:
+  until:
+    - "Question:"
+    - "Question"
+    - "</s>"
+    - "<|im_end|>"
+  do_sample: false
+  temperature: 0.0
+  # is_gsm8k: true
+repeats: 1
+num_fewshot: 5
+filter_list:
+  - name: "strict-match"
+    filter:
+      - function: "regex"
+        regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"
+  - name: "flexible-extract"
+    filter:
+      - function: "regex"
+        group_select: -1
+        regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+      - function: "take_first"
+metadata:
+  version: 3.0

src/backend/tasks/measurement_task_utils.py

@@ -12,6 +12,9 @@ def process_results_decorator(func):
         end_to_end_time = sum([r[1] for r in results]) / len(results)
         prefilling_time = sum([r[2] for r in results]) / len(results)
         decoding_throughput = sum([r[3] for r in results]) / len(results)
+        mfu = sum([r[4] for r in results]) / len(results)
+        mbu = sum([r[5] for r in results]) / len(results)
+        
         # print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
 
         # Now call the original process_results with the processed results
@@ -19,6 +22,8 @@ def process_results_decorator(func):
         result_dict["end_to_end_time"] = end_to_end_time
         result_dict["prefilling_time"] = prefilling_time
         result_dict["decoding_throughput"] = decoding_throughput
+        result_dict["mfu"] = mfu
+        result_dict["mbu"] = mbu
         return result_dict
     return wrapper
 
@@ -30,6 +35,8 @@ def aggregation_decorator(func):
         aggregation_list["end_to_end_time"] = mean
         aggregation_list["prefilling_time"] = mean
         aggregation_list["decoding_throughput"] = mean
+        aggregation_list["mfu"] = mean
+        aggregation_list["mbu"] = mean
         return aggregation_list
     return wrapper
 
@@ -41,6 +48,8 @@ def higher_is_better_decorator(func):
         higher_is_better_dict["end_to_end_time"] = False
         higher_is_better_dict["prefilling_time"] = False
         higher_is_better_dict["decoding_throughput"] = True
+        higher_is_better_dict["mfu"] = True
+        higher_is_better_dict["mbu"] = True
         return higher_is_better_dict
     return wrapper
 

src/display/about.py

@@ -3,7 +3,8 @@ from src.display.utils import ModelType
 TITLE = """<h1 align="center" id="space-title">OPEN-MOE-LLM-LEADERBOARD</h1>"""
 
 INTRODUCTION_TEXT = """
-The OPEN-MOE-LLM-LEADERBOARD is specifically designed to assess the performance and efficiency of various Mixture of Experts (MoE) Large Language Models (LLMs). This initiative, driven by the open-source community, aims to comprehensively evaluate these advanced MoE LLMs. We extend our gratitude to the Huggingface for the GPU community grant that supported the initial debugging process, and to [NetMind.AI](https://netmind.ai/home) for their generous GPU donation, which ensures the continuous operation of the Leaderboard.
+The OPEN-MOE-LLM-LEADERBOARD is specifically designed to assess the performance and efficiency of various Mixture of Experts (MoE) Large Language Models (LLMs). 
+This initiative, driven by the open-source community, aims to comprehensively evaluate these advanced MoE LLMs. 
 
 The OPEN-MOE-LLM-LEADERBOARD includes generation and multiple choice tasks to measure the performance and efficiency of MOE LLMs.
 
@@ -20,6 +21,15 @@ Columns and Metrics:
 - Precision: The precison of used model.
 
 """
+
+ACKNOWLEDGEMENT_TEXT = """
+<div>
+    <h4>Acknowledgements</h4>
+    {image_html}
+    <p>We express our sincere gratitude to <a href="https://netmind.ai/home">NetMind.AI</a> for their generous donation of GPUs, which plays a crucial role in ensuring the continuous operation of our Leaderboard.</p>
+</div>
+"""
+
 LLM_BENCHMARKS_TEXT = f"""
 
 """

src/display/utils.py

@@ -18,12 +18,16 @@ GPU_Power = 'Power(W)'
 GPU_Mem = 'Mem(G)'
 GPU_Name = "GPU"
 GPU_Util = 'Util(%)'
+MFU = 'MFU(%)'
+MBU = 'MBU(%)'
 BATCH_SIZE = 'bs'
 PRECISION = "Precision"
 system_metrics_to_name_map = {
     "end_to_end_time": f"{E2Es}",
     "prefilling_time": f"{PREs}",
     "decoding_throughput": f"{TS}",
+    "mfu": f"{MFU}",
+    "mbu": f"{MBU}"
 }
 
 gpu_metrics_to_name_map = {
@@ -75,6 +79,7 @@ class Tasks(Enum):
     # # XXX include me back at some point
     selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
     mmlu = Task("mmlu", "acc", "MMLU") #MMLU/Acc (5-shot)
+    gsm8k = Task("gsm8k_custom", "em", "GSM8K") #GSM8K/EM (8-shot)
 
 
 # These classes are for user facing column names,
@@ -104,16 +109,16 @@ auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnConten
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
     # System performance metrics
-    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_batch_size", ColumnContent, ColumnContent(f"{task.value.col_name} {BATCH_SIZE}", "number", True)])
-    # auto_eval_column_dict.append([f"{task.name}_precision", ColumnContent, ColumnContent(f"{task.value.col_name} {PRECISION}", "str", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Name}", "str", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_batch_size", ColumnContent, ColumnContent(f"{task.value.col_name} {BATCH_SIZE}", "number", True, hidden=True)])
+    # auto_eval_column_dict.append([f"{task.name}_precision", ColumnContent, ColumnContent(f"{task.value.col_name} {PRECISION}", "str", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Name}", "str", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True, hidden=True)])
     if task.value.benchmark in MULTIPLE_CHOICEs:
         continue
-    # auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", False)])
-    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} {TS}", "number", True)])
+    # auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", False, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} {TS}", "number", True, hidden=True)])
 
 
 # Model information
@@ -242,8 +247,8 @@ class Precision(Enum):
 
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 

src/submission/check_validity.py

@@ -74,7 +74,7 @@ def is_model_on_hub(
 
 
 def get_model_size(model_info: ModelInfo, precision: str):
-    size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
+    size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
@@ -130,7 +130,8 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}_{info['inference_framework']}_{info['gpu_type']}")
+                    if not info["status"] == "FINISHED" and not info["status"] == "RUNNING":
+                        file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}_{info['inference_framework']}_{info['gpu_type']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/utils.py

@@ -3,12 +3,54 @@ from huggingface_hub import snapshot_download
 import subprocess
 import re
 import os
+import GPUtil
 
 try:
     from src.display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util, GPU_Name
 except:
     print("local debug: from display.utils")
     from display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util, GPU_Name
+    
+MEM_BW_DICT ={
+    "NVIDIA-A100-PCIe-80GB": 1935,
+    "NVIDIA-A100-SXM-80GB": 2039,
+    "NVIDIA-H100-PCIe-80GB": 2039,
+    "NVIDIA-RTX-A5000-24GB": 768
+}
+
+PEAK_FLOPS_DICT = {
+    "float32":{
+        "NVIDIA-A100-PCIe-80GB": 312e12,
+        "NVIDIA-A100-SXM-80GB": 312e12,
+        "NVIDIA-H100-PCIe-80GB": 756e12,
+        "NVIDIA-RTX-A5000-24GB": 222.2e12
+    },
+    "float16":{
+        "NVIDIA-A100-PCIe-80GB": 624e12,
+        "NVIDIA-A100-SXM-80GB": 624e12,
+        "NVIDIA-H100-PCIe-80GB": 1513e12,
+        "NVIDIA-RTX-A5000-24GB": 444.4e12
+    },
+    "bfloat16":{
+        "NVIDIA-A100-PCIe-80GB": 624e12,
+        "NVIDIA-A100-SXM-80GB": 624e12,
+        "NVIDIA-H100-PCIe-80GB": 1513e12,
+        "NVIDIA-RTX-A5000-24GB": 444.4e12
+    },
+    "8bit":{
+        "NVIDIA-A100-PCIe-80GB": 1248e12,
+        "NVIDIA-A100-SXM-80GB": 1248e12,
+        "NVIDIA-H100-PCIe-80GB": 3026e12,
+        "NVIDIA-RTX-A5000-24GB": 889e12
+    },
+    "4bit": {
+        "NVIDIA-A100-PCIe-80GB": 2496e12,
+        "NVIDIA-A100-SXM-80GB": 2496e12,
+        "NVIDIA-H100-PCIe-80GB": 6052e12,
+        "NVIDIA-RTX-A5000-24GB": 1778e12
+    }
+
+}
 
 def my_snapshot_download(repo_id, revision, local_dir, repo_type, max_workers):
     for i in range(10):
@@ -52,11 +94,12 @@ def parse_nvidia_smi():
             print("Failed to query GPU indices.")
             return []
         gpu_indices = result.stdout.strip().split('\n')
-    print(f"gpu_indices: {gpu_indices}")
+    # print(f"gpu_indices: {gpu_indices}")
     gpu_stats = []
 
     gpu_info_pattern = re.compile(r'(\d+)C\s+P\d+\s+(\d+)W / \d+W\s+\|\s+(\d+)MiB / \d+MiB\s+\|\s+(\d+)%')
-    gpu_name_pattern = re.compile(r'NVIDIA\s+([\w\s]+?\d+GB)')
+    # gpu_name_pattern = re.compile(r'NVIDIA\s+([\w\s]+\d+(?:\s*GB)?)')
+    gpu_name_pattern = re.compile(r'NVIDIA\s+(RTX\s+)?([A-Z0-9]+)')
 
     gpu_name = ""
     for index in gpu_indices:
@@ -68,7 +111,7 @@ def parse_nvidia_smi():
             name_match = gpu_name_pattern.search(line)
             gpu_info = {}
             if name_match:
-                gpu_name = name_match.group(1).strip()
+                gpu_name = ''.join(filter(None, name_match.groups())).strip()
             if match:
                 temp, power_usage, mem_usage, gpu_util = map(int, match.groups())
                 gpu_info.update({
@@ -80,7 +123,7 @@ def parse_nvidia_smi():
 
             if len(gpu_info) >= 4:
                 gpu_stats.append(gpu_info)
-    print(f"gpu_stats: {gpu_stats}")
+    # print(f"gpu_stats: {gpu_stats}")
     gpu_name = f"{len(gpu_stats)}x{gpu_name}"
     gpu_stats_total = {
                         GPU_TEMP: 0,
@@ -131,5 +174,75 @@ def analyze_gpu_stats(stats_list):
 
     return avg_stats
 
+def get_gpu_number():
+    visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', None)
+    if visible_devices is not None:
+        gpu_indices = visible_devices.split(',')
+    else:
+        # Query all GPU indices if CUDA_VISIBLE_DEVICES is not set
+        result = subprocess.run(['nvidia-smi', '--query-gpu=index', '--format=csv,noheader'], capture_output=True, text=True)
+        if result.returncode != 0:
+            print("Failed to query GPU indices.")
+            return []
+        gpu_indices = result.stdout.strip().split('\n')
+    # print(f"gpu_indices: {gpu_indices}")
+    gpu_stats = []
+
+    gpu_info_pattern = re.compile(r'(\d+)C\s+P\d+\s+(\d+)W / \d+W\s+\|\s+(\d+)MiB / \d+MiB\s+\|\s+(\d+)%')
+
+    for index in gpu_indices:
+        result = subprocess.run(['nvidia-smi', '-i', index], capture_output=True, text=True)
+        output = result.stdout.strip()
+        lines = output.split("\n")
+        for line in lines:
+            match = gpu_info_pattern.search(line)
+            gpu_info = {}
+            if match:
+                temp, power_usage, mem_usage, gpu_util = map(int, match.groups())
+                gpu_info.update({
+                    GPU_TEMP: temp,
+                    GPU_Power: power_usage,
+                    GPU_Mem: round(mem_usage / 1024, 2),
+                    GPU_Util: gpu_util
+                })
+
+            if len(gpu_info) >= 4:
+                gpu_stats.append(gpu_info)
+                
+    return len(gpu_stats)
+
+def get_gpu_details():
+    gpus = GPUtil.getGPUs()
+    gpu = gpus[0]
+    name = gpu.name.replace(" ", "-")
+    memory_gb = round(gpu.memoryTotal / 1024)
+    memory = f"{memory_gb}GB"
+
+    for part in name.split('-'):
+        if part.endswith("GB") and part[:-2].isdigit():
+            name = name.replace(f"-{part}", "").replace(part, "")
+
+    formatted_name = f"{name}-{memory}"
+    
+    return formatted_name
+
+def get_peak_bw(gpu_name):
+    return MEM_BW_DICT[gpu_name]
+
+def get_peak_flops(gpu_name, precision):
+    return PEAK_FLOPS_DICT[precision][gpu_name]
+
+def transfer_precision2bytes(precision):
+    if precision == "float32":
+        return 4
+    elif precision in ["float16", "bfloat16"]:
+        return 2
+    elif precision == "8bit":
+        return 1
+    elif precision == "4bit":
+        return 0.5
+    else:
+        raise ValueError(f"Unsupported precision: {precision}")
+
 if __name__ == "__main__":
     print(analyze_gpu_stats(parse_nvidia_smi()))