Update README.md
Browse files
README.md
CHANGED
@@ -83,13 +83,13 @@ pip3 install lm-eval==0.4.2
|
|
83 |
|
84 |
```bash
|
85 |
git clone https://github.com/intel/auto-round
|
86 |
-
|
87 |
-
|
88 |
```
|
89 |
|
90 |
| Metric | BF16 | INT4(group_size 128) | INT4(group_size 32) |
|
91 |
-
|
92 |
-
| Avg | 0.
|
93 |
| mmlu | 0.4587 | 0.4440 | 0.4452 |
|
94 |
| cmmlu | 0.5036 | 0.4580 | 0.4683 |
|
95 |
| ceval-valid | 0.5312 | 0.4844 | 0.5015 |
|
@@ -98,7 +98,6 @@ auto-round --model "Intel/Qwen2.5-0.5B-Instruct-int4-inc" --eval --eval_bs 16 -
|
|
98 |
| winogrande | 0.5604 | 0.5683 | 0.5556 |
|
99 |
| piqa | 0.7035 | 0.7008 | 0.7089 |
|
100 |
| truthfulqa_mc1 | 0.2693 | 0.2583 | 0.2570 |
|
101 |
-
| truthfulqa_mc2 | 0.4183 | 0.4123 | 0.4025 |
|
102 |
| openbookqa | 0.2400 | 0.2360 | 0.2340 |
|
103 |
| boolq | 0.6798 | 0.6826 | 0.6676 |
|
104 |
| arc_easy | 0.6557 | 0.6536 | 0.6334 |
|
@@ -115,8 +114,8 @@ Here is the sample command to reproduce the model. We observed a larger accuracy
|
|
115 |
|
116 |
```bash
|
117 |
git clone https://github.com/intel/auto-round
|
118 |
-
|
119 |
-
|
120 |
--model_name Qwen/Qwen2.5-0.5B-Instruct \
|
121 |
--device 0 \
|
122 |
--group_size 128 \
|
|
|
83 |
|
84 |
```bash
|
85 |
git clone https://github.com/intel/auto-round
|
86 |
+
cd auto-round
|
87 |
+
python -m auto_round --model "Intel/Qwen2.5-0.5B-Instruct-int4-inc" --eval --eval_bs 16 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,gsm8k,cmmlu,ceval-valid
|
88 |
```
|
89 |
|
90 |
| Metric | BF16 | INT4(group_size 128) | INT4(group_size 32) |
|
91 |
+
|:--------------:| :----: | :----: |:------:|
|
92 |
+
| Avg | 0.4637 | 0.4496 | 0.4512 |
|
93 |
| mmlu | 0.4587 | 0.4440 | 0.4452 |
|
94 |
| cmmlu | 0.5036 | 0.4580 | 0.4683 |
|
95 |
| ceval-valid | 0.5312 | 0.4844 | 0.5015 |
|
|
|
98 |
| winogrande | 0.5604 | 0.5683 | 0.5556 |
|
99 |
| piqa | 0.7035 | 0.7008 | 0.7089 |
|
100 |
| truthfulqa_mc1 | 0.2693 | 0.2583 | 0.2570 |
|
|
|
101 |
| openbookqa | 0.2400 | 0.2360 | 0.2340 |
|
102 |
| boolq | 0.6798 | 0.6826 | 0.6676 |
|
103 |
| arc_easy | 0.6557 | 0.6536 | 0.6334 |
|
|
|
114 |
|
115 |
```bash
|
116 |
git clone https://github.com/intel/auto-round
|
117 |
+
cd auto-round
|
118 |
+
python -m auto_round \
|
119 |
--model_name Qwen/Qwen2.5-0.5B-Instruct \
|
120 |
--device 0 \
|
121 |
--group_size 128 \
|