add mmlu acc

dataset/llm-perf-leaderboard-Raspberry Pi 5(8GB).csv

@@ -1,129 +1,129 @@
-Model,Quantization,Params (B),Model Size (GB),Prefill (tokens/s),Decode (tokens/s),Backend
-gemma-2-9b,Q8_0,10.159,10.796,2.169,0.012,llama_cpp
-DeepSeek-V2-Lite,Q4_K_M,15.706,10.36,4.304,1.764,llama_cpp
-aya-expanse-8b,Q8_0,9.077,9.644,3.1,0.027,llama_cpp
-aya-23-8b,Q8_0,9.077,9.644,3.174,0.027,llama_cpp
-Yi-1.5-9B,Q8_0,8.829,9.382,2.585,0.019,llama_cpp
-Qwen2.5-14B,Q4_K_M,14.77,8.982,1.916,0.018,llama_cpp
-DeepSeek-V2-Lite,Q4_0_4_4,15.706,8.901,7.788,3.867,llama_cpp
-Phi-3-medium-128k-instruct,Q4_K_M,13.96,8.566,1.819,0.02,llama_cpp
-Hermes-3-Llama-3.1-8B,Q8_0,8.03,8.533,3.286,0.922,llama_cpp
-Qwen2.5-14B,Q4_0_4_4,14.77,8.512,4.698,0.028,llama_cpp
-internlm2_5-7b-chat,Q8_0,7.738,8.222,3.258,1.238,llama_cpp
-dolphin-2.9.2-qwen2-7b,Q8_0,7.616,8.093,4.241,1.301,llama_cpp
-Qwen2.5-7B,Q8_0,7.616,8.093,4.253,1.302,llama_cpp
-Phi-3-medium-128k-instruct,Q4_0_4_4,13.96,7.896,4.715,0.038,llama_cpp
-NexusRaven-V2-13B,Q4_K_M,13.016,7.865,2.066,0.035,llama_cpp
-Mistral-7B-Instruct-v0.3,Q8_0,7.248,7.702,4.104,1.29,llama_cpp
-dolphin-2.9.3-mistral-7B-32k,Q8_0,7.248,7.702,4.135,1.294,llama_cpp
-Yarn-Mistral-7b-128k,Q8_0,7.242,7.695,4.082,1.292,llama_cpp
-Starling-LM-7B-beta,Q8_0,7.242,7.695,4.132,1.296,llama_cpp
-Mistral-Nemo-Base-2407,Q4_K_M,12.248,7.469,2.453,1.358,llama_cpp
-NexusRaven-V2-13B,Q4_0_4_4,13.016,7.365,4.979,1.348,llama_cpp
-OLMoE-1B-7B-0924,Q8_0,6.919,7.358,26.942,7.489,llama_cpp
-OLMo-7B-0724-hf,Q8_0,6.888,7.319,4.515,1.371,llama_cpp
-mpt-7b-instruct,Q8_0,6.856,7.285,4.287,1.367,llama_cpp
-Amber,Q8_0,6.738,7.16,4.442,1.373,llama_cpp
-Mistral-Nemo-Base-2407,Q4_0_4_4,12.248,7.064,9.103,1.48,llama_cpp
-gemma-2-9b,Q4_K_M,10.159,6.508,3.531,1.629,llama_cpp
-Yarn-Solar-10b-64k,Q4_K_M,10.732,6.461,2.905,1.503,llama_cpp
-SOLAR-10.7B-v1.0,Q4_K_M,10.732,6.461,2.925,1.505,llama_cpp
-SOLAR-10.7B-Instruct-v1.0,Q4_K_M,10.732,6.461,2.916,1.506,llama_cpp
-Yi-1.5-6B,Q8_0,6.061,6.441,5.269,1.584,llama_cpp
-gemma-2-9b,Q4_0_4_4,10.159,6.19,10.553,1.757,llama_cpp
-SOLAR-10.7B-v1.0,Q4_0_4_4,10.732,6.072,9.315,1.635,llama_cpp
-SOLAR-10.7B-Instruct-v1.0,Q4_0_4_4,10.732,6.072,9.332,1.635,llama_cpp
-Yarn-Solar-10b-64k,Q4_0_4_4,10.732,6.072,9.352,1.638,llama_cpp
-aya-expanse-8b,Q4_K_M,9.077,5.906,4.406,1.911,llama_cpp
-aya-23-8B,Q4_K_M,9.077,5.906,4.428,1.914,llama_cpp
-aya-expanse-8b,Q4_0_4_4,9.077,5.647,14.074,2.05,llama_cpp
-aya-23-8B,Q4_0_4_4,9.077,5.647,14.113,2.051,llama_cpp
-Yi-1.5-9B,Q4_K_M,8.829,5.327,3.681,1.85,llama_cpp
-Yi-1.5-9B,Q4_0_4_4,8.829,5.035,11.33,2.0,llama_cpp
-Hermes-3-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.375,2.078,llama_cpp
-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.403,2.086,llama_cpp
-internlm2_5-7b-chat,Q4_K_M,7.738,4.711,4.4,2.133,llama_cpp
-Qwen2.5-7B,Q4_K_M,7.616,4.677,4.769,2.201,llama_cpp
-dolphin-2.9.2-qwen2-7b,Q4_K_M,7.616,4.677,4.759,2.204,llama_cpp
-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,13.99,2.245,llama_cpp
-Hermes-3-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,14.006,2.245,llama_cpp
-internlm2_5-7b-chat,Q4_0_4_4,7.738,4.451,14.036,2.31,llama_cpp
-mpt-7b-instruct,Q4_K_M,6.856,4.442,4.162,2.213,llama_cpp
-Qwen2.5-7B,Q4_0_4_4,7.616,4.425,15.563,2.386,llama_cpp
-dolphin-2.9.2-qwen2-7b,Q4_0_4_4,7.616,4.425,15.58,2.387,llama_cpp
-dolphin-2.9.3-mistral-7B-32k,Q4_K_M,7.248,4.372,4.387,2.227,llama_cpp
-Mistral-7B-Instruct-v0.3,Q4_K_M,7.248,4.372,4.462,2.241,llama_cpp
-Starling-LM-7B-beta,Q4_K_M,7.242,4.368,4.406,2.234,llama_cpp
-Yarn-Mistral-7b-128k,Q4_K_M,7.242,4.368,4.434,2.245,llama_cpp
-OLMoE-1B-7B-0924,Q4_K_M,6.919,4.212,26.902,12.119,llama_cpp
-OLMo-7B-0724-hf,Q4_K_M,6.888,4.183,4.706,2.339,llama_cpp
-dolphin-2.9.3-mistral-7B-32k,Q4_0_4_4,7.248,4.113,14.053,2.427,llama_cpp
-Mistral-7B-Instruct-v0.3,Q4_0_4_4,7.248,4.113,14.177,2.43,llama_cpp
-Starling-LM-7B-beta,Q4_0_4_4,7.242,4.108,14.068,2.427,llama_cpp
-Yarn-Mistral-7b-128k,Q4_0_4_4,7.242,4.108,14.139,2.436,llama_cpp
-Amber,Q4_K_M,6.738,4.08,4.594,2.351,llama_cpp
-Phi-3.5-mini-instruct,Q8_0,3.821,4.06,7.951,2.423,llama_cpp
-Phi-3-mini-128k-instruct,Q8_0,3.821,4.06,7.947,2.426,llama_cpp
-mpt-7b-instruct,Q4_0_4_4,6.856,3.964,14.569,2.533,llama_cpp
-OLMoE-1B-7B-0924,Q4_0_4_4,6.919,3.926,50.413,12.989,llama_cpp
-Llama-3.2-3B,Q8_0,3.607,3.833,10.31,2.83,llama_cpp
-Amber,Q4_0_4_4,6.738,3.825,14.442,2.57,llama_cpp
-Yi-1.5-6B,Q4_K_M,6.061,3.672,5.58,2.72,llama_cpp
-Qwen2.5-3B,Q8_0,3.397,3.61,10.473,2.939,llama_cpp
-Yi-1.5-6B,Q4_0_4_4,6.061,3.478,17.017,2.945,llama_cpp
-dolphin-2.9.4-gemma2-2b,Q8_0,3.204,3.405,13.966,3.381,llama_cpp
-gemma-2-2b,Q8_0,3.204,3.405,13.996,3.385,llama_cpp
-stable-code-instruct-3b,Q8_0,2.795,2.971,10.668,3.316,llama_cpp
-Phi-3.5-mini-instruct,Q4_K_M,3.821,2.393,7.502,3.936,llama_cpp
-Phi-3-mini-128k-instruct,Q4_K_M,3.821,2.393,7.519,3.938,llama_cpp
-Llama-3.2-3B,Q4_K_M,3.607,2.335,10.691,4.674,llama_cpp
-Llama-3.2-3B,Q4_0_4_4,3.607,2.233,31.72,5.025,llama_cpp
-gemma-2-2b,Q4_K_M,3.204,2.186,14.202,5.253,llama_cpp
-dolphin-2.9.4-gemma2-2b,Q4_K_M,3.204,2.186,14.218,5.253,llama_cpp
-Qwen2.5-3B,Q4_K_M,3.397,2.179,10.638,4.808,llama_cpp
-Phi-3.5-mini-instruct,Q4_0_4_4,3.821,2.175,23.369,4.428,llama_cpp
-Phi-3-mini-128k-instruct,Q4_0_4_4,3.821,2.175,23.461,4.436,llama_cpp
-gemma-2-2b,Q4_0_4_4,3.204,2.107,40.616,5.552,llama_cpp
-dolphin-2.9.4-gemma2-2b,Q4_0_4_4,3.204,2.107,40.977,5.58,llama_cpp
-Qwen2.5-3B,Q4_0_4_4,3.397,2.072,32.434,5.239,llama_cpp
-internlm2_5-1_8b-chat,Q8_0,1.889,2.007,19.329,5.279,llama_cpp
-SmolLM2-1.7B-Instruct,Q8_0,1.812,1.926,17.524,5.177,llama_cpp
-Qwen2.5-1.5B,Q8_0,1.777,1.889,21.927,5.793,llama_cpp
-stable-code-instruct-3b,Q4_K_M,2.795,1.707,10.803,5.564,llama_cpp
-stable-code-instruct-3b,Q4_0_4_4,2.795,1.607,28.926,5.957,llama_cpp
-Llama-3.2-1B,Q8_0,1.498,1.592,29.722,7.295,llama_cpp
-Yi-Coder-1.5B,Q8_0,1.476,1.569,23.894,6.596,llama_cpp
-OLMo-1B-0724-hf,Q8_0,1.28,1.36,27.787,7.591,llama_cpp
-Qwen2.5-1.5B,Q4_K_M,1.777,1.172,22.326,9.56,llama_cpp
-internlm2_5-1_8b-chat,Q4_K_M,1.889,1.17,19.453,8.56,llama_cpp
-TinyLlama-1.1B-Chat-v1.0,Q8_0,1.1,1.169,28.472,8.637,llama_cpp
-TinyLlama_v1.1,Q8_0,1.1,1.169,28.538,8.652,llama_cpp
-SmolLM2-1.7B-Instruct,Q4_K_M,1.812,1.136,17.72,8.497,llama_cpp
-Qwen2.5-1.5B,Q4_0_4_4,1.777,1.12,65.915,10.128,llama_cpp
-internlm2_5-1_8b-chat,Q4_0_4_4,1.889,1.112,57.736,9.243,llama_cpp
-SmolLM2-1.7B-Instruct,Q4_0_4_4,1.812,1.072,50.27,9.239,llama_cpp
-Llama-3.2-1B,Q4_K_M,1.498,1.015,30.451,11.51,llama_cpp
-Llama-3.2-1B,Q4_0_4_4,1.498,0.979,86.772,12.364,llama_cpp
-Yi-Coder-1.5B,Q4_K_M,1.476,0.962,23.267,10.03,llama_cpp
-Yi-Coder-1.5B,Q4_0_4_4,1.476,0.865,67.713,11.422,llama_cpp
-OLMo-1B-0724-hf,Q4_K_M,1.28,0.79,28.276,12.321,llama_cpp
-OLMo-1B-0724-hf,Q4_0_4_4,1.28,0.746,84.882,13.339,llama_cpp
-Qwen2.5-0.5B,Q8_0,0.63,0.67,75.456,18.06,llama_cpp
-TinyLlama-1.1B-Chat-v1.0,Q4_K_M,1.1,0.667,29.44,14.305,llama_cpp
-TinyLlama_v1.1,Q4_K_M,1.1,0.667,29.397,14.346,llama_cpp
-TinyLlama-1.1B-Chat-v1.0,Q4_0_4_4,1.1,0.636,77.823,15.509,llama_cpp
-TinyLlama_v1.1,Q4_0_4_4,1.1,0.636,77.943,15.543,llama_cpp
-Qwen2.5-0.5B,Q4_K_M,0.63,0.537,52.916,22.324,llama_cpp
-Qwen2.5-0.5B,Q4_0_4_4,0.63,0.491,189.874,26.738,llama_cpp
-gpt2-medium,Q8_0,0.406,0.436,83.423,23.016,llama_cpp
-SmolLM2-360M-Instruct,Q8_0,0.409,0.435,79.518,22.857,llama_cpp
-SmolLM2-360M-Instruct,Q4_K_M,0.409,0.319,55.774,30.718,llama_cpp
-SmolLM2-360M-Instruct,Q4_0_4_4,0.409,0.277,173.275,37.176,llama_cpp
-gpt2-medium,Q4_K_M,0.406,0.269,73.615,33.913,llama_cpp
-gpt2-medium,Q4_0_4_4,0.406,0.247,178.73,37.89,llama_cpp
-gpt2,Q8_0,0.163,0.176,302.932,68.191,llama_cpp
-SmolLM2-135M-Instruct,Q8_0,0.163,0.173,212.146,57.992,llama_cpp
-SmolLM2-135M-Instruct,Q4_K_M,0.163,0.134,153.439,73.272,llama_cpp
-SmolLM2-135M-Instruct,Q4_0_4_4,0.163,0.12,381.667,86.735,llama_cpp
-gpt2,Q4_K_M,0.163,0.111,269.906,92.707,llama_cpp
-gpt2,Q4_0_4_4,0.163,0.105,582.32,101.509,llama_cpp
+Model,Quantization,Params (B),Model Size (GB),Prefill (tokens/s),Decode (tokens/s),Backend,MMLU Accuracy
+gemma-2-9b,Q8_0,10.159,10.796,2.169,0.012,llama_cpp,42.365
+DeepSeek-V2-Lite,Q4_K_M,15.706,10.36,4.304,1.764,llama_cpp,38.908
+aya-expanse-8b,Q8_0,9.077,9.644,3.1,0.027,llama_cpp,41.361
+aya-23-8B,Q8_0,9.077,9.644,3.174,0.027,llama_cpp,38.263
+Yi-1.5-9B,Q8_0,8.829,9.382,2.585,0.019,llama_cpp,41.77
+Qwen2.5-14B,Q4_K_M,14.77,8.982,1.916,0.018,llama_cpp,42.48
+DeepSeek-V2-Lite,Q4_0_4_4,15.706,8.901,7.788,3.867,llama_cpp,38.629
+Phi-3-medium-128k-instruct,Q4_K_M,13.96,8.566,1.819,0.02,llama_cpp,42.674
+Hermes-3-Llama-3.1-8B,Q8_0,8.03,8.533,3.286,0.922,llama_cpp,41.806
+Qwen2.5-14B,Q4_0_4_4,14.77,8.512,4.698,0.028,llama_cpp,42.093
+internlm2_5-7b-chat,Q8_0,7.738,8.222,3.258,1.238,llama_cpp,41.684
+dolphin-2.9.2-qwen2-7b,Q8_0,7.616,8.093,4.241,1.301,llama_cpp,38.521
+Qwen2.5-7B,Q8_0,7.616,8.093,4.253,1.302,llama_cpp,40.364
+Phi-3-medium-128k-instruct,Q4_0_4_4,13.96,7.896,4.715,0.038,llama_cpp,42.136
+NexusRaven-V2-13B,Q4_K_M,13.016,7.865,2.066,0.035,llama_cpp,32.934
+Mistral-7B-Instruct-v0.3,Q8_0,7.248,7.702,4.104,1.29,llama_cpp,43.204
+dolphin-2.9.3-mistral-7B-32k,Q8_0,7.248,7.702,4.135,1.294,llama_cpp,40.436
+Yarn-Mistral-7b-128k,Q8_0,7.242,7.695,4.082,1.292,llama_cpp,40.171
+Starling-LM-7B-beta,Q8_0,7.242,7.695,4.132,1.296,llama_cpp,41.318
+Mistral-Nemo-Base-2407,Q4_K_M,12.248,7.469,2.453,1.358,llama_cpp,41.204
+NexusRaven-V2-13B,Q4_0_4_4,13.016,7.365,4.979,1.348,llama_cpp,32.977
+OLMoE-1B-7B-0924,Q8_0,6.919,7.358,26.942,7.489,llama_cpp,38.349
+OLMo-7B-0724-hf,Q8_0,6.888,7.319,4.515,1.371,llama_cpp,36.219
+mpt-7b-instruct,Q8_0,6.856,7.285,4.287,1.367,llama_cpp,35.33
+Amber,Q8_0,6.738,7.16,4.442,1.373,llama_cpp,33.149
+Mistral-Nemo-Base-2407,Q4_0_4_4,12.248,7.064,9.103,1.48,llama_cpp,41.885
+gemma-2-9b,Q4_K_M,10.159,6.508,3.531,1.629,llama_cpp,41.813
+Yarn-Solar-10b-64k,Q4_K_M,10.732,6.461,2.905,1.503,llama_cpp,38.815
+SOLAR-10.7B-v1.0,Q4_K_M,10.732,6.461,2.925,1.505,llama_cpp,39.446
+SOLAR-10.7B-Instruct-v1.0,Q4_K_M,10.732,6.461,2.916,1.506,llama_cpp,40.386
+Yi-1.5-6B,Q8_0,6.061,6.441,5.269,1.584,llama_cpp,39.941
+gemma-2-9b,Q4_0_4_4,10.159,6.19,10.553,1.757,llama_cpp,42.351
+SOLAR-10.7B-v1.0,Q4_0_4_4,10.732,6.072,9.315,1.635,llama_cpp,39.504
+SOLAR-10.7B-Instruct-v1.0,Q4_0_4_4,10.732,6.072,9.332,1.635,llama_cpp,40.673
+Yarn-Solar-10b-64k,Q4_0_4_4,10.732,6.072,9.352,1.638,llama_cpp,39.41
+aya-expanse-8b,Q4_K_M,9.077,5.906,4.406,1.911,llama_cpp,41.612
+aya-23-8B,Q4_K_M,9.077,5.906,4.428,1.914,llama_cpp,37.804
+aya-expanse-8b,Q4_0_4_4,9.077,5.647,14.074,2.05,llama_cpp,41.483
+aya-23-8B,Q4_0_4_4,9.077,5.647,14.113,2.051,llama_cpp,38.277
+Yi-1.5-9B,Q4_K_M,8.829,5.327,3.681,1.85,llama_cpp,41.218
+Yi-1.5-9B,Q4_0_4_4,8.829,5.035,11.33,2.0,llama_cpp,40.479
+Hermes-3-Llama-3.1-8B,Q4_K_M,8.03,4.913,4.375,2.078,llama_cpp,41.225
+Llama-3.1-8B,Q4_K_M,8.03,4.913,4.403,2.086,llama_cpp,40.45
+internlm2_5-7b-chat,Q4_K_M,7.738,4.711,4.4,2.133,llama_cpp,41.333
+Qwen2.5-7B,Q4_K_M,7.616,4.677,4.769,2.201,llama_cpp,40.199
+dolphin-2.9.2-qwen2-7b,Q4_K_M,7.616,4.677,4.759,2.204,llama_cpp,38.084
+Llama-3.1-8B,Q4_0_4_4,8.03,4.653,13.99,2.245,llama_cpp,39.676
+Hermes-3-Llama-3.1-8B,Q4_0_4_4,8.03,4.653,14.006,2.245,llama_cpp,40.68
+internlm2_5-7b-chat,Q4_0_4_4,7.738,4.451,14.036,2.31,llama_cpp,41.691
+mpt-7b-instruct,Q4_K_M,6.856,4.442,4.162,2.213,llama_cpp,35.265
+Qwen2.5-7B,Q4_0_4_4,7.616,4.425,15.563,2.386,llama_cpp,40.063
+dolphin-2.9.2-qwen2-7b,Q4_0_4_4,7.616,4.425,15.58,2.387,llama_cpp,37.704
+dolphin-2.9.3-mistral-7B-32k,Q4_K_M,7.248,4.372,4.387,2.227,llama_cpp,39.748
+Mistral-7B-Instruct-v0.3,Q4_K_M,7.248,4.372,4.462,2.241,llama_cpp,42.853
+Starling-LM-7B-beta,Q4_K_M,7.242,4.368,4.406,2.234,llama_cpp,41.038
+Yarn-Mistral-7b-128k,Q4_K_M,7.242,4.368,4.434,2.245,llama_cpp,40.085
+OLMoE-1B-7B-0924,Q4_K_M,6.919,4.212,26.902,12.119,llama_cpp,38.284
+OLMo-7B-0724-hf,Q4_K_M,6.888,4.183,4.706,2.339,llama_cpp,36.169
+dolphin-2.9.3-mistral-7B-32k,Q4_0_4_4,7.248,4.113,14.053,2.427,llama_cpp,40.314
+Mistral-7B-Instruct-v0.3,Q4_0_4_4,7.248,4.113,14.177,2.43,llama_cpp,42.882
+Starling-LM-7B-beta,Q4_0_4_4,7.242,4.108,14.068,2.427,llama_cpp,41.297
+Yarn-Mistral-7b-128k,Q4_0_4_4,7.242,4.108,14.139,2.436,llama_cpp,40.264
+Amber,Q4_K_M,6.738,4.08,4.594,2.351,llama_cpp,32.662
+Phi-3.5-mini-instruct,Q8_0,3.821,4.06,7.951,2.423,llama_cpp,41.77
+Phi-3-mini-128k-instruct,Q8_0,3.821,4.06,7.947,2.426,llama_cpp,41.361
+mpt-7b-instruct,Q4_0_4_4,6.856,3.964,14.569,2.533,llama_cpp,34.928
+OLMoE-1B-7B-0924,Q4_0_4_4,6.919,3.926,50.413,12.989,llama_cpp,37.998
+Llama-3.2-3B,Q8_0,3.607,3.833,10.31,2.83,llama_cpp,37.481
+Amber,Q4_0_4_4,6.738,3.825,14.442,2.57,llama_cpp,33.085
+Yi-1.5-6B,Q4_K_M,6.061,3.672,5.58,2.72,llama_cpp,39.253
+Qwen2.5-3B,Q8_0,3.397,3.61,10.473,2.939,llama_cpp,38.557
+Yi-1.5-6B,Q4_0_4_4,6.061,3.478,17.017,2.945,llama_cpp,39.195
+dolphin-2.9.4-gemma2-2b,Q8_0,3.204,3.405,13.966,3.381,llama_cpp,37.202
+gemma-2-2b,Q8_0,3.204,3.405,13.996,3.385,llama_cpp,37.323
+stable-code-instruct-3b,Q8_0,2.795,2.971,10.668,3.316,llama_cpp,29.886
+Phi-3.5-mini-instruct,Q4_K_M,3.821,2.393,7.502,3.936,llama_cpp,41.082
+Phi-3-mini-128k-instruct,Q4_K_M,3.821,2.393,7.519,3.938,llama_cpp,40.895
+Llama-3.2-3B,Q4_K_M,3.607,2.335,10.691,4.674,llama_cpp,37.338
+Llama-3.2-3B,Q4_0_4_4,3.607,2.233,31.72,5.025,llama_cpp,36.814
+gemma-2-2b,Q4_K_M,3.204,2.186,14.202,5.253,llama_cpp,36.958
+dolphin-2.9.4-gemma2-2b,Q4_K_M,3.204,2.186,14.218,5.253,llama_cpp,37.302
+Qwen2.5-3B,Q4_K_M,3.397,2.179,10.638,4.808,llama_cpp,38.162
+Phi-3.5-mini-instruct,Q4_0_4_4,3.821,2.175,23.369,4.428,llama_cpp,41.383
+Phi-3-mini-128k-instruct,Q4_0_4_4,3.821,2.175,23.461,4.436,llama_cpp,40.608
+gemma-2-2b,Q4_0_4_4,3.204,2.107,40.616,5.552,llama_cpp,37.374
+dolphin-2.9.4-gemma2-2b,Q4_0_4_4,3.204,2.107,40.977,5.58,llama_cpp,37.051
+Qwen2.5-3B,Q4_0_4_4,3.397,2.072,32.434,5.239,llama_cpp,37.912
+internlm2_5-1_8b-chat,Q8_0,1.889,2.007,19.329,5.279,llama_cpp,33.996
+SmolLM2-1.7B-Instruct,Q8_0,1.812,1.926,17.524,5.177,llama_cpp,35.989
+Qwen2.5-1.5B,Q8_0,1.777,1.889,21.927,5.793,llama_cpp,35.81
+stable-code-instruct-3b,Q4_K_M,2.795,1.707,10.803,5.564,llama_cpp,29.8
+stable-code-instruct-3b,Q4_0_4_4,2.795,1.607,28.926,5.957,llama_cpp,29.843
+Llama-3.2-1B,Q8_0,1.498,1.592,29.722,7.295,llama_cpp,33.974
+Yi-Coder-1.5B,Q8_0,1.476,1.569,23.894,6.596,llama_cpp,29.334
+OLMo-1B-0724-hf,Q8_0,1.28,1.36,27.787,7.591,llama_cpp,31.693
+Qwen2.5-1.5B,Q4_K_M,1.777,1.172,22.326,9.56,llama_cpp,35.832
+internlm2_5-1_8b-chat,Q4_K_M,1.889,1.17,19.453,8.56,llama_cpp,33.709
+TinyLlama-1.1B-Chat-v1.0,Q8_0,1.1,1.169,28.472,8.637,llama_cpp,30.897
+TinyLlama_v1.1,Q8_0,1.1,1.169,28.538,8.652,llama_cpp,28.186
+SmolLM2-1.7B-Instruct,Q4_K_M,1.812,1.136,17.72,8.497,llama_cpp,35.358
+Qwen2.5-1.5B,Q4_0_4_4,1.777,1.12,65.915,10.128,llama_cpp,35.064
+internlm2_5-1_8b-chat,Q4_0_4_4,1.889,1.112,57.736,9.243,llama_cpp,32.21
+SmolLM2-1.7B-Instruct,Q4_0_4_4,1.812,1.072,50.27,9.239,llama_cpp,35.136
+Llama-3.2-1B,Q4_K_M,1.498,1.015,30.451,11.51,llama_cpp,33.472
+Llama-3.2-1B,Q4_0_4_4,1.498,0.979,86.772,12.364,llama_cpp,33.386
+Yi-Coder-1.5B,Q4_K_M,1.476,0.962,23.267,10.03,llama_cpp,29.391
+Yi-Coder-1.5B,Q4_0_4_4,1.476,0.865,67.713,11.422,llama_cpp,29.24
+OLMo-1B-0724-hf,Q4_K_M,1.28,0.79,28.276,12.321,llama_cpp,31.865
+OLMo-1B-0724-hf,Q4_0_4_4,1.28,0.746,84.882,13.339,llama_cpp,31.457
+Qwen2.5-0.5B,Q8_0,0.63,0.67,75.456,18.06,llama_cpp,31.937
+TinyLlama-1.1B-Chat-v1.0,Q4_K_M,1.1,0.667,29.44,14.305,llama_cpp,30.653
+TinyLlama_v1.1,Q4_K_M,1.1,0.667,29.397,14.346,llama_cpp,28.043
+TinyLlama-1.1B-Chat-v1.0,Q4_0_4_4,1.1,0.636,77.823,15.509,llama_cpp,30.861
+TinyLlama_v1.1,Q4_0_4_4,1.1,0.636,77.943,15.543,llama_cpp,28.315
+Qwen2.5-0.5B,Q4_K_M,0.63,0.537,52.916,22.324,llama_cpp,31.442
+Qwen2.5-0.5B,Q4_0_4_4,0.63,0.491,189.874,26.738,llama_cpp,31.256
+gpt2-medium,Q8_0,0.406,0.436,83.423,23.016,llama_cpp,29.032
+SmolLM2-360M-Instruct,Q8_0,0.409,0.435,79.518,22.857,llama_cpp,32.303
+SmolLM2-360M-Instruct,Q4_K_M,0.409,0.319,55.774,30.718,llama_cpp,31.944
+SmolLM2-360M-Instruct,Q4_0_4_4,0.409,0.277,173.275,37.176,llama_cpp,32.038
+gpt2-medium,Q4_K_M,0.406,0.269,73.615,33.913,llama_cpp,28.81
+gpt2-medium,Q4_0_4_4,0.406,0.247,178.73,37.89,llama_cpp,28.824
+gpt2,Q8_0,0.163,0.176,302.932,68.191,llama_cpp,27.24
+SmolLM2-135M-Instruct,Q8_0,0.163,0.173,212.146,57.992,llama_cpp,29.893
+SmolLM2-135M-Instruct,Q4_K_M,0.163,0.134,153.439,73.272,llama_cpp,29.492
+SmolLM2-135M-Instruct,Q4_0_4_4,0.163,0.12,381.667,86.735,llama_cpp,29.821
+gpt2,Q4_K_M,0.163,0.111,269.906,92.707,llama_cpp,27.598
+gpt2,Q4_0_4_4,0.163,0.105,582.32,101.509,llama_cpp,27.899

src/leaderboard.py

@@ -16,6 +16,7 @@ LEADERBOARD_COLUMN_TO_DATATYPE = {
     # "Reserved Memory (MB)": "number",
     # "Used Memory (MB)": "number",
     "Params (B)": "number",
+    "MMLU Accuracy": "number",
 }
 
 PRIMARY_COLUMNS = [
@@ -24,6 +25,7 @@ PRIMARY_COLUMNS = [
     "Prefill (tokens/s)",
     "Decode (tokens/s)",
     "Model Size (GB)",
+    "MMLU Accuracy"
 ]
 
 

src/llm_perf.py

@@ -5,21 +5,21 @@ import pandas as pd
 
 DATASET_DIRECTORY = "dataset"
 
-COLUMNS_MAPPING = {
-    "config.name": "Quantization",
-    "config.backend.model": "Model",
-    # primary measurements
-    "report.prefill.throughput.value": "Prefill (tokens/s)",
-    "report.decode.throughput.value": "Decode (tokens/s)",
-    "report.memory": "Model Size (GB)",
-    # deployment settings
-    "config.backend.name": "Backend",
-    "quantization": "Quantization",
-    # additional information
-    "#Params (B)": "Params (B)",
-}
-SORTING_COLUMNS = ["Model Size (GB)", "Decode (tokens/s)", "Prefill (tokens/s)"]
-SORTING_ASCENDING = [False, True, True]
+# COLUMNS_MAPPING = {
+#     "config.name": "Quantization",
+#     "config.backend.model": "Model",
+#     # primary measurements
+#     "report.prefill.throughput.value": "Prefill (tokens/s)",
+#     "report.decode.throughput.value": "Decode (tokens/s)",
+#     "report.memory": "Model Size (GB)",
+#     # deployment settings
+#     "config.backend.name": "Backend",
+#     "quantization": "Quantization",
+#     # additional information
+#     "#Params (B)": "Params (B)",
+# }
+SORTING_COLUMNS = ["Model Size (GB)", "Decode (tokens/s)", "Prefill (tokens/s)", "MMLU Accuracy"]
+SORTING_ASCENDING = [False, True, True, True]
 
 
 def get_raw_llm_perf_df(
@@ -28,7 +28,7 @@ def get_raw_llm_perf_df(
     dfs = []
     try:
         dfs.append(
-            pd.read_csv("/Users/arnavchavan/leaderboard/benchmark_results.csv")
+            pd.read_csv("/Users/arnavchavan/leaderboard/benchmark_results_with_mmlu.csv")
             # pd.read_csv(
             #     f"hf://datasets/nyunai/edge-llm-leaderboard/perf-df-{hardware_type}-{machine}-{backends}.csv"
             # )
@@ -68,6 +68,7 @@ def processed_llm_perf_df(llm_perf_df):
             "Decode (tokens/s)": 3,
             "Model Size (GB)": 3,
             "#Params (B)": 3,
+            "MMLU Accuracy": 3,
         }
     )
     # sort by metric