Update README.md

README.md

@@ -144,30 +144,32 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
 
 <table>
   <tr>
+   <td><strong>Category</strong>
+   </td>
    <td><strong>Benchmark</strong>
    </td>
    <td><strong>Meta-Llama-3.1-8B-Instruct </strong>
    </td>
-   <td><strong>Meta-Llama-3.1-8B-Instruct-quantized.w4a16 (this model)</strong>
+   <td><strong>Meta-Llama-3.1-8B-Instruct-quantized.w8a8 (this model)</strong>
    </td>
    <td><strong>Recovery</strong>
    </td>
   </tr>
   <tr>
-   <td><strong>Arena Hard</strong>
+   <td rowspan="1" ><strong>LLM as a judge</strong>
+   </td>    
+   <td>Arena Hard
    </td>
    <td>25.8 (25.1 / 26.5)
    </td>
-   <td>24.0 (23.4 / 24.6)
+   <td>27.2 (27.6 / 26.7)
    </td>
-   <td>93.0%
+   <td>105.4%
    </td>
   </tr>
   <tr>
-   <td><strong>OpenLLM v1</strong>
+   <td rowspan="8" ><strong>OpenLLM v1</strong>
    </td>
-  </tr>
-  <tr>
    <td>MMLU (5-shot)
    </td>
    <td>68.3
@@ -248,10 +250,8 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td><strong>OpenLLM v2</strong>
+   <td rowspan="7" ><strong>OpenLLM v2</strong>
    </td>
-  </tr>
-  <tr>
    <td>MMLU-Pro (5-shot)
    </td>
    <td>30.8
@@ -282,7 +282,7 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td>Math-|v|-5 (4-shot)
+   <td>Math-lvl-5 (4-shot)
    </td>
    <td>15.7
    </td>
@@ -322,10 +322,8 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td><strong>Coding</strong>
+   <td rowspan="2" ><strong>Coding</strong>
    </td>
-  </tr>
-  <tr>
    <td>HumanEval pass@1
    </td>
    <td>67.3
@@ -345,8 +343,81 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    <td>97.4%
    </td>
   </tr>
+  <tr>
+   <td rowspan="9" ><strong>Multilingual</strong>
+   </td>
+   <td>Portuguese MMLU (5-shot)
+   </td>
+   <td>59.96
+   </td>
+   <td>58.69
+   </td>
+   <td>97.9%
+   </td>
+  </tr>
+  <tr>
+   <td>Spanish MMLU (5-shot)
+   </td>
+   <td>60.25
+   </td>
+   <td>58.39
+   </td>
+   <td>96.9%
+   </td>
+  </tr>
+  <tr>
+   <td>Italian MMLU (5-shot)
+   </td>
+   <td>59.23
+   </td>
+   <td>57.82
+   </td>
+   <td>97.6%
+   </td>
+  </tr>
+  <tr>
+   <td>German MMLU (5-shot)
+   </td>
+   <td>58.63
+   </td>
+   <td>56.22
+   </td>
+   <td>95.9%
+   </td>
+  </tr>
+  <tr>
+   <td>French MMLU (5-shot)
+   </td>
+   <td>59.65
+   </td>
+   <td>57.58
+   </td>
+   <td>96.5%
+   </td>
+  </tr>
+  <tr>
+   <td>Hindi MMLU (5-shot)
+   </td>
+   <td>50.10
+   </td>
+   <td>47.14
+   </td>
+   <td>94.1%
+   </td>
+  </tr>
+  <tr>
+   <td>Thai MMLU (5-shot)
+   </td>
+   <td>49.12
+   </td>
+   <td>46.72
+   </td>
+   <td>95.1%
+   </td>
+  </tr>
 </table>
 
+
 ### Reproduction
 
 The results were obtained using the following commands:
@@ -438,6 +509,90 @@ lm_eval \
   --batch_size auto
 ```
 
+#### MMLU Portuguese
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_pt_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Spanish
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_es_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Italian
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_it_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU German
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_de_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU French
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_fr_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Hindi
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_hi_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Thai
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_th_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
 #### HumanEval and HumanEval+
 ##### Generation
 ```