n1ck-guo commited on
Commit
e6cc5a2
·
verified ·
1 Parent(s): c703901

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +6 -7
README.md CHANGED
@@ -83,13 +83,13 @@ pip3 install lm-eval==0.4.2
83
 
84
  ```bash
85
  git clone https://github.com/intel/auto-round
86
- pip install -vvv --no-build-isolation -e .
87
- auto-round --model "Intel/Qwen2.5-0.5B-Instruct-int4-inc" --eval --eval_bs 16 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,truthfulqa_mc2,openbookqa,boolq,arc_easy,arc_challenge,mmlu,gsm8k,cmmlu,ceval-valid
88
  ```
89
 
90
  | Metric | BF16 | INT4(group_size 128) | INT4(group_size 32) |
91
- | -------------- | ------ | ------ | ------ |
92
- | Avg | 0.4604 | 0.4469 | 0.4478 |
93
  | mmlu | 0.4587 | 0.4440 | 0.4452 |
94
  | cmmlu | 0.5036 | 0.4580 | 0.4683 |
95
  | ceval-valid | 0.5312 | 0.4844 | 0.5015 |
@@ -98,7 +98,6 @@ auto-round --model "Intel/Qwen2.5-0.5B-Instruct-int4-inc" --eval --eval_bs 16 -
98
  | winogrande | 0.5604 | 0.5683 | 0.5556 |
99
  | piqa | 0.7035 | 0.7008 | 0.7089 |
100
  | truthfulqa_mc1 | 0.2693 | 0.2583 | 0.2570 |
101
- | truthfulqa_mc2 | 0.4183 | 0.4123 | 0.4025 |
102
  | openbookqa | 0.2400 | 0.2360 | 0.2340 |
103
  | boolq | 0.6798 | 0.6826 | 0.6676 |
104
  | arc_easy | 0.6557 | 0.6536 | 0.6334 |
@@ -115,8 +114,8 @@ Here is the sample command to reproduce the model. We observed a larger accuracy
115
 
116
  ```bash
117
  git clone https://github.com/intel/auto-round
118
- pip install -vvv --no-build-isolation -e .
119
- auto-round \
120
  --model_name Qwen/Qwen2.5-0.5B-Instruct \
121
  --device 0 \
122
  --group_size 128 \
 
83
 
84
  ```bash
85
  git clone https://github.com/intel/auto-round
86
+ cd auto-round
87
+ python -m auto_round --model "Intel/Qwen2.5-0.5B-Instruct-int4-inc" --eval --eval_bs 16 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,gsm8k,cmmlu,ceval-valid
88
  ```
89
 
90
  | Metric | BF16 | INT4(group_size 128) | INT4(group_size 32) |
91
+ |:--------------:| :----: | :----: |:------:|
92
+ | Avg | 0.4637 | 0.4496 | 0.4512 |
93
  | mmlu | 0.4587 | 0.4440 | 0.4452 |
94
  | cmmlu | 0.5036 | 0.4580 | 0.4683 |
95
  | ceval-valid | 0.5312 | 0.4844 | 0.5015 |
 
98
  | winogrande | 0.5604 | 0.5683 | 0.5556 |
99
  | piqa | 0.7035 | 0.7008 | 0.7089 |
100
  | truthfulqa_mc1 | 0.2693 | 0.2583 | 0.2570 |
 
101
  | openbookqa | 0.2400 | 0.2360 | 0.2340 |
102
  | boolq | 0.6798 | 0.6826 | 0.6676 |
103
  | arc_easy | 0.6557 | 0.6536 | 0.6334 |
 
114
 
115
  ```bash
116
  git clone https://github.com/intel/auto-round
117
+ cd auto-round
118
+ python -m auto_round \
119
  --model_name Qwen/Qwen2.5-0.5B-Instruct \
120
  --device 0 \
121
  --group_size 128 \