TheBloke commited on
Commit
a0e39e3
1 Parent(s): c1b9e73

Update for Transformers AWQ support

Browse files
Files changed (1) hide show
  1. config.json +9 -2
config.json CHANGED
@@ -23,5 +23,12 @@
23
  "torch_dtype": "float16",
24
  "transformers_version": "4.35.0",
25
  "use_cache": true,
26
- "vocab_size": 32002
27
- }
 
 
 
 
 
 
 
 
23
  "torch_dtype": "float16",
24
  "transformers_version": "4.35.0",
25
  "use_cache": true,
26
+ "vocab_size": 32002,
27
+ "quantization_config": {
28
+ "quant_method": "awq",
29
+ "zero_point": true,
30
+ "group_size": 128,
31
+ "bits": 4,
32
+ "version": "gemm"
33
+ }
34
+ }