alignment fix and desription change

README.md

@@ -1,5 +1,5 @@
 ---
-title: Edge Llm Leaderboard
+title: Edge LLM Leaderboard
 emoji: 🌖
 colorFrom: red
 colorTo: blue
@@ -11,10 +11,10 @@ license: apache-2.0
 tags: [edge llm leaderboard, llm edge leaderboard, llm, edge, leaderboard]
 ---
 
-# LLM-perf leaderboard
+# Edge LLM leaderboard
 
 ## 📝 About
-The Edge-LLM Leaderboard is a leaderboard to gauge practical performance and quality of edge LLMs.
+The Edge LLM Leaderboard is a leaderboard to gauge practical performance and quality of edge LLMs.
 Its aim is to benchmark the performance (throughput and memory) 
 of Large Language Models (LLMs) on Edge hardware - starting with a Raspberry Pi 5 (8GB) based on the ARM Cortex A76 CPU.
 
@@ -32,9 +32,21 @@ All of our throughput benchmarks are ran by this single tool
 [llama-bench](https://github.com/ggerganov/llama.cpp/tree/master/examples/llama-bench)
 using the power of [llama.cpp](https://github.com/ggerganov/llama.cpp) to guarantee reproducibility and consistency.
 
+## 🏆 Ranking Models
+
+We use MMLU (zero-shot) via [llama-perplexity](https://github.com/ggerganov/llama.cpp/tree/master/examples/perplexity) for performance evaluation, focusing on key metrics relevant for edge applications:
+
+1. **Prefill Latency (Time to First Token - TTFT):**  Measures the time to generate the first token. Low TTFT ensures a smooth user experience, especially for real-time interactions in edge use cases.
+
+2. **Decode Latency (Generation Speed):**  Indicates the speed of generating subsequent tokens, critical for real-time tasks like transcription or extended dialogue sessions.
+
+3. **Model Size:**  Smaller models are better suited for edge devices with limited secondary storage compared to cloud or GPU systems, making efficient deployment possible.
+
+These metrics collectively address the unique challenges of deploying LLMs on edge devices, balancing performance, responsiveness, and memory constraints.
+
 ## 🏃 How to run locally 
 
-To run the Edge-LLM Leaderboard locally on your machine, follow these steps:
+To run the Edge LLM Leaderboard locally on your machine, follow these steps:
 
 ### 1. Clone the Repository
 

app.py

@@ -4,7 +4,7 @@ import src.dependency  # noqa
 from src.assets import custom_css
 
 # from src.attention import create_attn_plots
-from src.content import ABOUT, CITATION_BUTTON, CITATION_BUTTON_LABEL, LOGO, TITLE
+from src.content import ABOUT, CITATION_BUTTON, CITATION_BUTTON_LABEL, LOGO, LOGO2, TITLE
 from src.hardware import load_hardware_configs
 from src.leaderboard import create_leaderboard_table
 from src.llm_perf import get_llm_perf_df
@@ -22,12 +22,14 @@ demo = gr.Blocks(
     theme=gr.themes.Default(primary_hue="indigo", secondary_hue="indigo"),
 )
 with demo:
-    gr.HTML(LOGO, elem_classes="logo")
-    gr.HTML(TITLE, elem_classes="title")
+    with gr.Row():
+        gr.HTML(LOGO, elem_classes="logo")
+        gr.HTML(LOGO2, elem_classes="logo2")
+        gr.HTML(TITLE, elem_classes="title")
     ####################### HARDWARE TABS #######################
     with gr.Tabs(elem_classes="tabs"):
         for id, config in enumerate(configs):
-            with gr.TabItem(config.description, id=id):
+            with gr.TabItem("Leaderboard", id=id):
                 ####################### HARDWARE DETAILS #######################
                 if config.detail:
                     gr.Markdown(config.detail, elem_classes="descriptive-text")
@@ -57,7 +59,7 @@ with demo:
                         hardware_type=config.hardware_type,
                     )
                     ####################### LEADERBOARD TAB #######################
-                    with gr.TabItem("Leaderboard 🏅", id=0):
+                    with gr.TabItem("Raspberry Pi 5 - Cortex A76", id=0):
                         search_bar, columns_checkboxes, leaderboard_table = (
                             create_leaderboard_table(open_llm_perf_df)
                         )
@@ -114,6 +116,7 @@ with demo:
                 label=CITATION_BUTTON_LABEL,
                 elem_id="citation-button",
                 show_copy_button=True,
+                lines=7
             )
 
 if __name__ == "__main__":

dataset/llm-perf-leaderboard-Raspberry Pi 5(8GB).csv

@@ -1,129 +1,129 @@
 Model,Quantization,Params (B),Model Size (GB),Prefill (tokens/s),Decode (tokens/s),Backend,MMLU Accuracy
-gemma-2-9b,Q8_0,10.159,10.796,2.169,0.012,llama_cpp,42.365
-DeepSeek-V2-Lite,Q4_K_M,15.706,10.36,4.304,1.764,llama_cpp,38.908
-aya-expanse-8b,Q8_0,9.077,9.644,3.1,0.027,llama_cpp,41.361
-aya-23-8B,Q8_0,9.077,9.644,3.174,0.027,llama_cpp,38.263
-Yi-1.5-9B,Q8_0,8.829,9.382,2.585,0.019,llama_cpp,41.77
-Qwen2.5-14B,Q4_K_M,14.77,8.982,1.916,0.018,llama_cpp,42.48
-DeepSeek-V2-Lite,Q4_0_4_4,15.706,8.901,7.788,3.867,llama_cpp,38.629
-Phi-3-medium-128k-instruct,Q4_K_M,13.96,8.566,1.819,0.02,llama_cpp,42.674
-Hermes-3-Llama-3.1-8B,Q8_0,8.03,8.533,3.286,0.922,llama_cpp,41.806
-Qwen2.5-14B,Q4_0_4_4,14.77,8.512,4.698,0.028,llama_cpp,42.093
-internlm2_5-7b-chat,Q8_0,7.738,8.222,3.258,1.238,llama_cpp,41.684
-dolphin-2.9.2-qwen2-7b,Q8_0,7.616,8.093,4.241,1.301,llama_cpp,38.521
-Qwen2.5-7B,Q8_0,7.616,8.093,4.253,1.302,llama_cpp,40.364
-Phi-3-medium-128k-instruct,Q4_0_4_4,13.96,7.896,4.715,0.038,llama_cpp,42.136
-NexusRaven-V2-13B,Q4_K_M,13.016,7.865,2.066,0.035,llama_cpp,32.934
-Mistral-7B-Instruct-v0.3,Q8_0,7.248,7.702,4.104,1.29,llama_cpp,43.204
-dolphin-2.9.3-mistral-7B-32k,Q8_0,7.248,7.702,4.135,1.294,llama_cpp,40.436
-Yarn-Mistral-7b-128k,Q8_0,7.242,7.695,4.082,1.292,llama_cpp,40.171
-Starling-LM-7B-beta,Q8_0,7.242,7.695,4.132,1.296,llama_cpp,41.318
-Mistral-Nemo-Base-2407,Q4_K_M,12.248,7.469,2.453,1.358,llama_cpp,41.204
-NexusRaven-V2-13B,Q4_0_4_4,13.016,7.365,4.979,1.348,llama_cpp,32.977
-OLMoE-1B-7B-0924,Q8_0,6.919,7.358,26.942,7.489,llama_cpp,38.349
-OLMo-7B-0724-hf,Q8_0,6.888,7.319,4.515,1.371,llama_cpp,36.219
-mpt-7b-instruct,Q8_0,6.856,7.285,4.287,1.367,llama_cpp,35.33
-Amber,Q8_0,6.738,7.16,4.442,1.373,llama_cpp,33.149
-Mistral-Nemo-Base-2407,Q4_0_4_4,12.248,7.064,9.103,1.48,llama_cpp,41.885
-gemma-2-9b,Q4_K_M,10.159,6.508,3.531,1.629,llama_cpp,41.813
-Yarn-Solar-10b-64k,Q4_K_M,10.732,6.461,2.905,1.503,llama_cpp,38.815
-SOLAR-10.7B-v1.0,Q4_K_M,10.732,6.461,2.925,1.505,llama_cpp,39.446
-SOLAR-10.7B-Instruct-v1.0,Q4_K_M,10.732,6.461,2.916,1.506,llama_cpp,40.386
-Yi-1.5-6B,Q8_0,6.061,6.441,5.269,1.584,llama_cpp,39.941
-gemma-2-9b,Q4_0_4_4,10.159,6.19,10.553,1.757,llama_cpp,42.351
-SOLAR-10.7B-v1.0,Q4_0_4_4,10.732,6.072,9.315,1.635,llama_cpp,39.504
-SOLAR-10.7B-Instruct-v1.0,Q4_0_4_4,10.732,6.072,9.332,1.635,llama_cpp,40.673
-Yarn-Solar-10b-64k,Q4_0_4_4,10.732,6.072,9.352,1.638,llama_cpp,39.41
-aya-expanse-8b,Q4_K_M,9.077,5.906,4.406,1.911,llama_cpp,41.612
-aya-23-8B,Q4_K_M,9.077,5.906,4.428,1.914,llama_cpp,37.804
-aya-expanse-8b,Q4_0_4_4,9.077,5.647,14.074,2.05,llama_cpp,41.483
-aya-23-8B,Q4_0_4_4,9.077,5.647,14.113,2.051,llama_cpp,38.277
-Yi-1.5-9B,Q4_K_M,8.829,5.327,3.681,1.85,llama_cpp,41.218
-Yi-1.5-9B,Q4_0_4_4,8.829,5.035,11.33,2.0,llama_cpp,40.479
-Hermes-3-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.375,2.078,llama_cpp,41.225
-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.403,2.086,llama_cpp,40.45
-internlm2_5-7b-chat,Q4_K_M,7.738,4.711,4.4,2.133,llama_cpp,41.333
-Qwen2.5-7B,Q4_K_M,7.616,4.677,4.769,2.201,llama_cpp,40.199
-dolphin-2.9.2-qwen2-7b,Q4_K_M,7.616,4.677,4.759,2.204,llama_cpp,38.084
-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,13.99,2.245,llama_cpp,39.676
-Hermes-3-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,14.006,2.245,llama_cpp,40.68
-internlm2_5-7b-chat,Q4_0_4_4,7.738,4.451,14.036,2.31,llama_cpp,41.691
-mpt-7b-instruct,Q4_K_M,6.856,4.442,4.162,2.213,llama_cpp,35.265
-Qwen2.5-7B,Q4_0_4_4,7.616,4.425,15.563,2.386,llama_cpp,40.063
-dolphin-2.9.2-qwen2-7b,Q4_0_4_4,7.616,4.425,15.58,2.387,llama_cpp,37.704
-dolphin-2.9.3-mistral-7B-32k,Q4_K_M,7.248,4.372,4.387,2.227,llama_cpp,39.748
-Mistral-7B-Instruct-v0.3,Q4_K_M,7.248,4.372,4.462,2.241,llama_cpp,42.853
-Starling-LM-7B-beta,Q4_K_M,7.242,4.368,4.406,2.234,llama_cpp,41.038
-Yarn-Mistral-7b-128k,Q4_K_M,7.242,4.368,4.434,2.245,llama_cpp,40.085
-OLMoE-1B-7B-0924,Q4_K_M,6.919,4.212,26.902,12.119,llama_cpp,38.284
-OLMo-7B-0724-hf,Q4_K_M,6.888,4.183,4.706,2.339,llama_cpp,36.169
-dolphin-2.9.3-mistral-7B-32k,Q4_0_4_4,7.248,4.113,14.053,2.427,llama_cpp,40.314
-Mistral-7B-Instruct-v0.3,Q4_0_4_4,7.248,4.113,14.177,2.43,llama_cpp,42.882
-Starling-LM-7B-beta,Q4_0_4_4,7.242,4.108,14.068,2.427,llama_cpp,41.297
-Yarn-Mistral-7b-128k,Q4_0_4_4,7.242,4.108,14.139,2.436,llama_cpp,40.264
-Amber,Q4_K_M,6.738,4.08,4.594,2.351,llama_cpp,32.662
-Phi-3.5-mini-instruct,Q8_0,3.821,4.06,7.951,2.423,llama_cpp,41.77
-Phi-3-mini-128k-instruct,Q8_0,3.821,4.06,7.947,2.426,llama_cpp,41.361
-mpt-7b-instruct,Q4_0_4_4,6.856,3.964,14.569,2.533,llama_cpp,34.928
-OLMoE-1B-7B-0924,Q4_0_4_4,6.919,3.926,50.413,12.989,llama_cpp,37.998
-Llama-3.2-3B,Q8_0,3.607,3.833,10.31,2.83,llama_cpp,37.481
-Amber,Q4_0_4_4,6.738,3.825,14.442,2.57,llama_cpp,33.085
-Yi-1.5-6B,Q4_K_M,6.061,3.672,5.58,2.72,llama_cpp,39.253
-Qwen2.5-3B,Q8_0,3.397,3.61,10.473,2.939,llama_cpp,38.557
-Yi-1.5-6B,Q4_0_4_4,6.061,3.478,17.017,2.945,llama_cpp,39.195
-dolphin-2.9.4-gemma2-2b,Q8_0,3.204,3.405,13.966,3.381,llama_cpp,37.202
-gemma-2-2b,Q8_0,3.204,3.405,13.996,3.385,llama_cpp,37.323
-stable-code-instruct-3b,Q8_0,2.795,2.971,10.668,3.316,llama_cpp,29.886
-Phi-3.5-mini-instruct,Q4_K_M,3.821,2.393,7.502,3.936,llama_cpp,41.082
-Phi-3-mini-128k-instruct,Q4_K_M,3.821,2.393,7.519,3.938,llama_cpp,40.895
-Llama-3.2-3B,Q4_K_M,3.607,2.335,10.691,4.674,llama_cpp,37.338
-Llama-3.2-3B,Q4_0_4_4,3.607,2.233,31.72,5.025,llama_cpp,36.814
-gemma-2-2b,Q4_K_M,3.204,2.186,14.202,5.253,llama_cpp,36.958
-dolphin-2.9.4-gemma2-2b,Q4_K_M,3.204,2.186,14.218,5.253,llama_cpp,37.302
-Qwen2.5-3B,Q4_K_M,3.397,2.179,10.638,4.808,llama_cpp,38.162
-Phi-3.5-mini-instruct,Q4_0_4_4,3.821,2.175,23.369,4.428,llama_cpp,41.383
-Phi-3-mini-128k-instruct,Q4_0_4_4,3.821,2.175,23.461,4.436,llama_cpp,40.608
-gemma-2-2b,Q4_0_4_4,3.204,2.107,40.616,5.552,llama_cpp,37.374
-dolphin-2.9.4-gemma2-2b,Q4_0_4_4,3.204,2.107,40.977,5.58,llama_cpp,37.051
-Qwen2.5-3B,Q4_0_4_4,3.397,2.072,32.434,5.239,llama_cpp,37.912
-internlm2_5-1_8b-chat,Q8_0,1.889,2.007,19.329,5.279,llama_cpp,33.996
-SmolLM2-1.7B-Instruct,Q8_0,1.812,1.926,17.524,5.177,llama_cpp,35.989
-Qwen2.5-1.5B,Q8_0,1.777,1.889,21.927,5.793,llama_cpp,35.81
+gemma-2-9b,Q8_0,10.159,10.796,2.169,0.012,llama_cpp,42.4
+DeepSeek-V2-Lite,Q4_K_M,15.706,10.36,4.304,1.764,llama_cpp,38.9
+aya-expanse-8b,Q8_0,9.077,9.644,3.1,0.027,llama_cpp,41.4
+aya-23-8B,Q8_0,9.077,9.644,3.174,0.027,llama_cpp,38.3
+Yi-1.5-9B,Q8_0,8.829,9.382,2.585,0.019,llama_cpp,41.8
+Qwen2.5-14B,Q4_K_M,14.77,8.982,1.916,0.018,llama_cpp,42.5
+DeepSeek-V2-Lite,Q4_0_4_4,15.706,8.901,7.788,3.867,llama_cpp,38.6
+Phi-3-medium-128k-instruct,Q4_K_M,13.96,8.566,1.819,0.02,llama_cpp,42.7
+Hermes-3-Llama-3.1-8B,Q8_0,8.03,8.533,3.286,0.922,llama_cpp,41.8
+Qwen2.5-14B,Q4_0_4_4,14.77,8.512,4.698,0.028,llama_cpp,42.1
+internlm2_5-7b-chat,Q8_0,7.738,8.222,3.258,1.238,llama_cpp,41.7
+dolphin-2.9.2-qwen2-7b,Q8_0,7.616,8.093,4.241,1.301,llama_cpp,38.5
+Qwen2.5-7B,Q8_0,7.616,8.093,4.253,1.302,llama_cpp,40.4
+Phi-3-medium-128k-instruct,Q4_0_4_4,13.96,7.896,4.715,0.038,llama_cpp,42.1
+NexusRaven-V2-13B,Q4_K_M,13.016,7.865,2.066,0.035,llama_cpp,32.9
+Mistral-7B-Instruct-v0.3,Q8_0,7.248,7.702,4.104,1.29,llama_cpp,43.2
+dolphin-2.9.3-mistral-7B-32k,Q8_0,7.248,7.702,4.135,1.294,llama_cpp,40.4
+Yarn-Mistral-7b-128k,Q8_0,7.242,7.695,4.082,1.292,llama_cpp,40.2
+Starling-LM-7B-beta,Q8_0,7.242,7.695,4.132,1.296,llama_cpp,41.3
+Mistral-Nemo-Base-2407,Q4_K_M,12.248,7.469,2.453,1.358,llama_cpp,41.2
+NexusRaven-V2-13B,Q4_0_4_4,13.016,7.365,4.979,1.348,llama_cpp,33.0
+OLMoE-1B-7B-0924,Q8_0,6.919,7.358,26.942,7.489,llama_cpp,38.3
+OLMo-7B-0724-hf,Q8_0,6.888,7.319,4.515,1.371,llama_cpp,36.2
+mpt-7b-instruct,Q8_0,6.856,7.285,4.287,1.367,llama_cpp,35.3
+Amber,Q8_0,6.738,7.16,4.442,1.373,llama_cpp,33.1
+Mistral-Nemo-Base-2407,Q4_0_4_4,12.248,7.064,9.103,1.48,llama_cpp,41.9
+gemma-2-9b,Q4_K_M,10.159,6.508,3.531,1.629,llama_cpp,41.8
+Yarn-Solar-10b-64k,Q4_K_M,10.732,6.461,2.905,1.503,llama_cpp,38.8
+SOLAR-10.7B-v1.0,Q4_K_M,10.732,6.461,2.925,1.505,llama_cpp,39.4
+SOLAR-10.7B-Instruct-v1.0,Q4_K_M,10.732,6.461,2.916,1.506,llama_cpp,40.4
+Yi-1.5-6B,Q8_0,6.061,6.441,5.269,1.584,llama_cpp,39.9
+gemma-2-9b,Q4_0_4_4,10.159,6.19,10.553,1.757,llama_cpp,42.4
+SOLAR-10.7B-v1.0,Q4_0_4_4,10.732,6.072,9.315,1.635,llama_cpp,39.5
+SOLAR-10.7B-Instruct-v1.0,Q4_0_4_4,10.732,6.072,9.332,1.635,llama_cpp,40.7
+Yarn-Solar-10b-64k,Q4_0_4_4,10.732,6.072,9.352,1.638,llama_cpp,39.4
+aya-expanse-8b,Q4_K_M,9.077,5.906,4.406,1.911,llama_cpp,41.6
+aya-23-8B,Q4_K_M,9.077,5.906,4.428,1.914,llama_cpp,37.8
+aya-expanse-8b,Q4_0_4_4,9.077,5.647,14.074,2.05,llama_cpp,41.5
+aya-23-8B,Q4_0_4_4,9.077,5.647,14.113,2.051,llama_cpp,38.3
+Yi-1.5-9B,Q4_K_M,8.829,5.327,3.681,1.85,llama_cpp,41.2
+Yi-1.5-9B,Q4_0_4_4,8.829,5.035,11.33,2.0,llama_cpp,40.5
+Hermes-3-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.375,2.078,llama_cpp,41.2
+Llama-3.1-8B,Q4_K_M,8.03,4.913,4.403,2.086,llama_cpp,40.5
+internlm2_5-7b-chat,Q4_K_M,7.738,4.711,4.4,2.133,llama_cpp,41.3
+Qwen2.5-7B,Q4_K_M,7.616,4.677,4.769,2.201,llama_cpp,40.2
+dolphin-2.9.2-qwen2-7b,Q4_K_M,7.616,4.677,4.759,2.204,llama_cpp,38.1
+Llama-3.1-8B,Q4_0_4_4,8.03,4.653,13.99,2.245,llama_cpp,39.7
+Hermes-3-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,14.006,2.245,llama_cpp,40.7
+internlm2_5-7b-chat,Q4_0_4_4,7.738,4.451,14.036,2.31,llama_cpp,41.7
+mpt-7b-instruct,Q4_K_M,6.856,4.442,4.162,2.213,llama_cpp,35.3
+Qwen2.5-7B,Q4_0_4_4,7.616,4.425,15.563,2.386,llama_cpp,40.1
+dolphin-2.9.2-qwen2-7b,Q4_0_4_4,7.616,4.425,15.58,2.387,llama_cpp,37.7
+dolphin-2.9.3-mistral-7B-32k,Q4_K_M,7.248,4.372,4.387,2.227,llama_cpp,39.7
+Mistral-7B-Instruct-v0.3,Q4_K_M,7.248,4.372,4.462,2.241,llama_cpp,42.9
+Starling-LM-7B-beta,Q4_K_M,7.242,4.368,4.406,2.234,llama_cpp,41.0
+Yarn-Mistral-7b-128k,Q4_K_M,7.242,4.368,4.434,2.245,llama_cpp,40.1
+OLMoE-1B-7B-0924,Q4_K_M,6.919,4.212,26.902,12.119,llama_cpp,38.3
+OLMo-7B-0724-hf,Q4_K_M,6.888,4.183,4.706,2.339,llama_cpp,36.2
+dolphin-2.9.3-mistral-7B-32k,Q4_0_4_4,7.248,4.113,14.053,2.427,llama_cpp,40.3
+Mistral-7B-Instruct-v0.3,Q4_0_4_4,7.248,4.113,14.177,2.43,llama_cpp,42.9
+Starling-LM-7B-beta,Q4_0_4_4,7.242,4.108,14.068,2.427,llama_cpp,41.3
+Yarn-Mistral-7b-128k,Q4_0_4_4,7.242,4.108,14.139,2.436,llama_cpp,40.3
+Amber,Q4_K_M,6.738,4.08,4.594,2.351,llama_cpp,32.7
+Phi-3.5-mini-instruct,Q8_0,3.821,4.06,7.951,2.423,llama_cpp,41.8
+Phi-3-mini-128k-instruct,Q8_0,3.821,4.06,7.947,2.426,llama_cpp,41.4
+mpt-7b-instruct,Q4_0_4_4,6.856,3.964,14.569,2.533,llama_cpp,34.9
+OLMoE-1B-7B-0924,Q4_0_4_4,6.919,3.926,50.413,12.989,llama_cpp,38.0
+Llama-3.2-3B,Q8_0,3.607,3.833,10.31,2.83,llama_cpp,37.5
+Amber,Q4_0_4_4,6.738,3.825,14.442,2.57,llama_cpp,33.1
+Yi-1.5-6B,Q4_K_M,6.061,3.672,5.58,2.72,llama_cpp,39.3
+Qwen2.5-3B,Q8_0,3.397,3.61,10.473,2.939,llama_cpp,38.6
+Yi-1.5-6B,Q4_0_4_4,6.061,3.478,17.017,2.945,llama_cpp,39.2
+dolphin-2.9.4-gemma2-2b,Q8_0,3.204,3.405,13.966,3.381,llama_cpp,37.2
+gemma-2-2b,Q8_0,3.204,3.405,13.996,3.385,llama_cpp,37.3
+stable-code-instruct-3b,Q8_0,2.795,2.971,10.668,3.316,llama_cpp,29.9
+Phi-3.5-mini-instruct,Q4_K_M,3.821,2.393,7.502,3.936,llama_cpp,41.1
+Phi-3-mini-128k-instruct,Q4_K_M,3.821,2.393,7.519,3.938,llama_cpp,40.9
+Llama-3.2-3B,Q4_K_M,3.607,2.335,10.691,4.674,llama_cpp,37.3
+Llama-3.2-3B,Q4_0_4_4,3.607,2.233,31.72,5.025,llama_cpp,36.8
+gemma-2-2b,Q4_K_M,3.204,2.186,14.202,5.253,llama_cpp,37.0
+dolphin-2.9.4-gemma2-2b,Q4_K_M,3.204,2.186,14.218,5.253,llama_cpp,37.3
+Qwen2.5-3B,Q4_K_M,3.397,2.179,10.638,4.808,llama_cpp,38.2
+Phi-3.5-mini-instruct,Q4_0_4_4,3.821,2.175,23.369,4.428,llama_cpp,41.4
+Phi-3-mini-128k-instruct,Q4_0_4_4,3.821,2.175,23.461,4.436,llama_cpp,40.6
+gemma-2-2b,Q4_0_4_4,3.204,2.107,40.616,5.552,llama_cpp,37.4
+dolphin-2.9.4-gemma2-2b,Q4_0_4_4,3.204,2.107,40.977,5.58,llama_cpp,37.1
+Qwen2.5-3B,Q4_0_4_4,3.397,2.072,32.434,5.239,llama_cpp,37.9
+internlm2_5-1_8b-chat,Q8_0,1.889,2.007,19.329,5.279,llama_cpp,34.0
+SmolLM2-1.7B-Instruct,Q8_0,1.812,1.926,17.524,5.177,llama_cpp,36.0
+Qwen2.5-1.5B,Q8_0,1.777,1.889,21.927,5.793,llama_cpp,35.8
 stable-code-instruct-3b,Q4_K_M,2.795,1.707,10.803,5.564,llama_cpp,29.8
-stable-code-instruct-3b,Q4_0_4_4,2.795,1.607,28.926,5.957,llama_cpp,29.843
-Llama-3.2-1B,Q8_0,1.498,1.592,29.722,7.295,llama_cpp,33.974
-Yi-Coder-1.5B,Q8_0,1.476,1.569,23.894,6.596,llama_cpp,29.334
-OLMo-1B-0724-hf,Q8_0,1.28,1.36,27.787,7.591,llama_cpp,31.693
-Qwen2.5-1.5B,Q4_K_M,1.777,1.172,22.326,9.56,llama_cpp,35.832
-internlm2_5-1_8b-chat,Q4_K_M,1.889,1.17,19.453,8.56,llama_cpp,33.709
-TinyLlama-1.1B-Chat-v1.0,Q8_0,1.1,1.169,28.472,8.637,llama_cpp,30.897
-TinyLlama_v1.1,Q8_0,1.1,1.169,28.538,8.652,llama_cpp,28.186
-SmolLM2-1.7B-Instruct,Q4_K_M,1.812,1.136,17.72,8.497,llama_cpp,35.358
-Qwen2.5-1.5B,Q4_0_4_4,1.777,1.12,65.915,10.128,llama_cpp,35.064
-internlm2_5-1_8b-chat,Q4_0_4_4,1.889,1.112,57.736,9.243,llama_cpp,32.21
-SmolLM2-1.7B-Instruct,Q4_0_4_4,1.812,1.072,50.27,9.239,llama_cpp,35.136
-Llama-3.2-1B,Q4_K_M,1.498,1.015,30.451,11.51,llama_cpp,33.472
-Llama-3.2-1B,Q4_0_4_4,1.498,0.979,86.772,12.364,llama_cpp,33.386
-Yi-Coder-1.5B,Q4_K_M,1.476,0.962,23.267,10.03,llama_cpp,29.391
-Yi-Coder-1.5B,Q4_0_4_4,1.476,0.865,67.713,11.422,llama_cpp,29.24
-OLMo-1B-0724-hf,Q4_K_M,1.28,0.79,28.276,12.321,llama_cpp,31.865
-OLMo-1B-0724-hf,Q4_0_4_4,1.28,0.746,84.882,13.339,llama_cpp,31.457
-Qwen2.5-0.5B,Q8_0,0.63,0.67,75.456,18.06,llama_cpp,31.937
-TinyLlama-1.1B-Chat-v1.0,Q4_K_M,1.1,0.667,29.44,14.305,llama_cpp,30.653
-TinyLlama_v1.1,Q4_K_M,1.1,0.667,29.397,14.346,llama_cpp,28.043
-TinyLlama-1.1B-Chat-v1.0,Q4_0_4_4,1.1,0.636,77.823,15.509,llama_cpp,30.861
-TinyLlama_v1.1,Q4_0_4_4,1.1,0.636,77.943,15.543,llama_cpp,28.315
-Qwen2.5-0.5B,Q4_K_M,0.63,0.537,52.916,22.324,llama_cpp,31.442
-Qwen2.5-0.5B,Q4_0_4_4,0.63,0.491,189.874,26.738,llama_cpp,31.256
-gpt2-medium,Q8_0,0.406,0.436,83.423,23.016,llama_cpp,29.032
-SmolLM2-360M-Instruct,Q8_0,0.409,0.435,79.518,22.857,llama_cpp,32.303
-SmolLM2-360M-Instruct,Q4_K_M,0.409,0.319,55.774,30.718,llama_cpp,31.944
-SmolLM2-360M-Instruct,Q4_0_4_4,0.409,0.277,173.275,37.176,llama_cpp,32.038
-gpt2-medium,Q4_K_M,0.406,0.269,73.615,33.913,llama_cpp,28.81
-gpt2-medium,Q4_0_4_4,0.406,0.247,178.73,37.89,llama_cpp,28.824
-gpt2,Q8_0,0.163,0.176,302.932,68.191,llama_cpp,27.24
-SmolLM2-135M-Instruct,Q8_0,0.163,0.173,212.146,57.992,llama_cpp,29.893
-SmolLM2-135M-Instruct,Q4_K_M,0.163,0.134,153.439,73.272,llama_cpp,29.492
-SmolLM2-135M-Instruct,Q4_0_4_4,0.163,0.12,381.667,86.735,llama_cpp,29.821
-gpt2,Q4_K_M,0.163,0.111,269.906,92.707,llama_cpp,27.598
-gpt2,Q4_0_4_4,0.163,0.105,582.32,101.509,llama_cpp,27.899
+stable-code-instruct-3b,Q4_0_4_4,2.795,1.607,28.926,5.957,llama_cpp,29.8
+Llama-3.2-1B,Q8_0,1.498,1.592,29.722,7.295,llama_cpp,34.0
+Yi-Coder-1.5B,Q8_0,1.476,1.569,23.894,6.596,llama_cpp,29.3
+OLMo-1B-0724-hf,Q8_0,1.28,1.36,27.787,7.591,llama_cpp,31.7
+Qwen2.5-1.5B,Q4_K_M,1.777,1.172,22.326,9.56,llama_cpp,35.8
+internlm2_5-1_8b-chat,Q4_K_M,1.889,1.17,19.453,8.56,llama_cpp,33.7
+TinyLlama-1.1B-Chat-v1.0,Q8_0,1.1,1.169,28.472,8.637,llama_cpp,30.9
+TinyLlama_v1.1,Q8_0,1.1,1.169,28.538,8.652,llama_cpp,28.2
+SmolLM2-1.7B-Instruct,Q4_K_M,1.812,1.136,17.72,8.497,llama_cpp,35.4
+Qwen2.5-1.5B,Q4_0_4_4,1.777,1.12,65.915,10.128,llama_cpp,35.1
+internlm2_5-1_8b-chat,Q4_0_4_4,1.889,1.112,57.736,9.243,llama_cpp,32.2
+SmolLM2-1.7B-Instruct,Q4_0_4_4,1.812,1.072,50.27,9.239,llama_cpp,35.1
+Llama-3.2-1B,Q4_K_M,1.498,1.015,30.451,11.51,llama_cpp,33.5
+Llama-3.2-1B,Q4_0_4_4,1.498,0.979,86.772,12.364,llama_cpp,33.4
+Yi-Coder-1.5B,Q4_K_M,1.476,0.962,23.267,10.03,llama_cpp,29.4
+Yi-Coder-1.5B,Q4_0_4_4,1.476,0.865,67.713,11.422,llama_cpp,29.2
+OLMo-1B-0724-hf,Q4_K_M,1.28,0.79,28.276,12.321,llama_cpp,31.9
+OLMo-1B-0724-hf,Q4_0_4_4,1.28,0.746,84.882,13.339,llama_cpp,31.5
+Qwen2.5-0.5B,Q8_0,0.63,0.67,75.456,18.06,llama_cpp,31.9
+TinyLlama-1.1B-Chat-v1.0,Q4_K_M,1.1,0.667,29.44,14.305,llama_cpp,30.7
+TinyLlama_v1.1,Q4_K_M,1.1,0.667,29.397,14.346,llama_cpp,28.0
+TinyLlama-1.1B-Chat-v1.0,Q4_0_4_4,1.1,0.636,77.823,15.509,llama_cpp,30.9
+TinyLlama_v1.1,Q4_0_4_4,1.1,0.636,77.943,15.543,llama_cpp,28.3
+Qwen2.5-0.5B,Q4_K_M,0.63,0.537,52.916,22.324,llama_cpp,31.4
+Qwen2.5-0.5B,Q4_0_4_4,0.63,0.491,189.874,26.738,llama_cpp,31.3
+gpt2-medium,Q8_0,0.406,0.436,83.423,23.016,llama_cpp,29.0
+SmolLM2-360M-Instruct,Q8_0,0.409,0.435,79.518,22.857,llama_cpp,32.3
+SmolLM2-360M-Instruct,Q4_K_M,0.409,0.319,55.774,30.718,llama_cpp,31.9
+SmolLM2-360M-Instruct,Q4_0_4_4,0.409,0.277,173.275,37.176,llama_cpp,32.0
+gpt2-medium,Q4_K_M,0.406,0.269,73.615,33.913,llama_cpp,28.8
+gpt2-medium,Q4_0_4_4,0.406,0.247,178.73,37.89,llama_cpp,28.8
+gpt2,Q8_0,0.163,0.176,302.932,68.191,llama_cpp,27.2
+SmolLM2-135M-Instruct,Q8_0,0.163,0.173,212.146,57.992,llama_cpp,29.9
+SmolLM2-135M-Instruct,Q4_K_M,0.163,0.134,153.439,73.272,llama_cpp,29.5
+SmolLM2-135M-Instruct,Q4_0_4_4,0.163,0.12,381.667,86.735,llama_cpp,29.8
+gpt2,Q4_K_M,0.163,0.111,269.906,92.707,llama_cpp,27.6
+gpt2,Q4_0_4_4,0.163,0.105,582.32,101.509,llama_cpp,27.9

hardware.yaml

@@ -1,5 +1,4 @@
 - machine: Raspberry Pi 5(8GB)
-  description: Cortex A76
   hardware_provider: ARM
   hardware_type: arm cortex a76
   subsets:

src/assets.py

@@ -1,12 +1,33 @@
 custom_css = """
 .logo {
-    width: 300px;
+    width: 200px;
     height: auto;
-    margin: 0 auto;
-    max-width: 100%
+    margin: 0;
+    max-width: 220px;
     object-fit: contain;
     overflow: visible !important;
+    display: flex;
+    align-items: center;
 }
+
+.logo2 {
+    width: 100px;
+    height: auto;
+    margin: 0;
+    max-width: 120px;
+    object-fit: contain;
+    overflow: visible !important;
+    display: flex;
+    align-items: center;
+}
+
+.title {
+    display: flex;
+    align-items: center;
+    font-size: 24px !important;
+    margin-top:6px;
+}
+
 .text {
     font-size: 16px !important;
 }

src/content.py

@@ -1,10 +1,12 @@
 LOGO = '<img src="https://nyunai.com/assets/images/logo.png">'
 
-TITLE = """<h1 align="center" id="space-title"> Edge-LLM Leaderboard </h1>"""
+LOGO2 = '<img src="https://huggingface.co/front/assets/huggingface_logo-noborder.svg">'
+
+TITLE = """<h1 align="left" id="space-title"> Edge LLM Leaderboard </h1>"""
 
 ABOUT = """
 ## 📝 About
-The Edge-LLM Leaderboard is a leaderboard to gauge practical performance and quality of edge LLMs.
+The Edge LLM Leaderboard is a leaderboard to gauge practical performance and quality of edge LLMs.
 Its aim is to benchmark the performance (throughput and memory) 
 of Large Language Models (LLMs) on Edge hardware - starting with a Raspberry Pi 5 (8GB) based on the ARM Cortex A76 CPU.
 
@@ -21,13 +23,26 @@ configuration for automated benchmarking:
 All of our throughput benchmarks are ran by this single tool
 [llama-bench](https://github.com/ggerganov/llama.cpp/tree/master/examples/llama-bench)
 using the power of [llama.cpp](https://github.com/ggerganov/llama.cpp) to guarantee reproducibility and consistency.
+
+## 🏆 Ranking Models
+
+We use MMLU (zero-shot) via [llama-perplexity](https://github.com/ggerganov/llama.cpp/tree/master/examples/perplexity) for performance evaluation, focusing on key metrics relevant for edge applications:
+
+1. **Prefill Latency (Time to First Token - TTFT):**  Measures the time to generate the first token. Low TTFT ensures a smooth user experience, especially for real-time interactions in edge use cases.
+
+2. **Decode Latency (Generation Speed):**  Indicates the speed of generating subsequent tokens, critical for real-time tasks like transcription or extended dialogue sessions.
+
+3. **Model Size:**  Smaller models are better suited for edge devices with limited secondary storage compared to cloud or GPU systems, making efficient deployment possible.
+
+These metrics collectively address the unique challenges of deploying LLMs on edge devices, balancing performance, responsiveness, and memory constraints.
+
 """
 
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
 CITATION_BUTTON = r"""@misc{edge-llm-leaderboard,
   author = {},
-  title = {Edge-LLM Leaderboard},
+  title = {Edge LLM Leaderboard},
   year = {2024},
   publisher = {},
   howpublished = "\url{https://huggingface.co/spaces/nyunai/edge-llm-leaderboard}",

src/hardware.py

@@ -6,7 +6,6 @@ import yaml
 class HardwareConfig:
     def __init__(self, data: Dict[str, Any]):
         self.machine: str = data["machine"]
-        self.description: str = data["description"]
         self.hardware_provider: str = data["hardware_provider"]
         self.hardware_type: str = data["hardware_type"]
         self.subsets: List[str] = data["subsets"]

src/llm_perf.py

@@ -68,7 +68,7 @@ def processed_llm_perf_df(llm_perf_df):
             "Decode (tokens/s)": 3,
             "Model Size (GB)": 3,
             "#Params (B)": 3,
-            "MMLU Accuracy": 3,
+            "MMLU Accuracy": 1,
         }
     )
     # sort by metric