Hugging Face's logo

OPEA
/
Meta-Llama-3.1-405B-Instruct-int4-sym-inc

Safetensors
llama
4-bit precision
intel/auto-round
Model card Files Community
wenhuach commited on
Commit
cb38f4a
•
1 Parent(s): 7faf1e0

autoround format

Browse files

Signed-off-by: wenhuach <wenhuach87@gmail.com>

This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. config.json +58 -0
  3. generation_config.json +12 -0
  4. model-00001-of-00044.safetensors +3 -0
  5. model-00002-of-00044.safetensors +3 -0
  6. model-00003-of-00044.safetensors +3 -0
  7. model-00004-of-00044.safetensors +3 -0
  8. model-00005-of-00044.safetensors +3 -0
  9. model-00006-of-00044.safetensors +3 -0
  10. model-00007-of-00044.safetensors +3 -0
  11. model-00008-of-00044.safetensors +3 -0
  12. model-00009-of-00044.safetensors +3 -0
  13. model-00010-of-00044.safetensors +3 -0
  14. model-00011-of-00044.safetensors +3 -0
  15. model-00012-of-00044.safetensors +3 -0
  16. model-00013-of-00044.safetensors +3 -0
  17. model-00014-of-00044.safetensors +3 -0
  18. model-00015-of-00044.safetensors +3 -0
  19. model-00016-of-00044.safetensors +3 -0
  20. model-00017-of-00044.safetensors +3 -0
  21. model-00018-of-00044.safetensors +3 -0
  22. model-00019-of-00044.safetensors +3 -0
  23. model-00020-of-00044.safetensors +3 -0
  24. model-00021-of-00044.safetensors +3 -0
  25. model-00022-of-00044.safetensors +3 -0
  26. model-00023-of-00044.safetensors +3 -0
  27. model-00024-of-00044.safetensors +3 -0
  28. model-00025-of-00044.safetensors +3 -0
  29. model-00026-of-00044.safetensors +3 -0
  30. model-00027-of-00044.safetensors +3 -0
  31. model-00028-of-00044.safetensors +3 -0
  32. model-00029-of-00044.safetensors +3 -0
  33. model-00030-of-00044.safetensors +3 -0
  34. model-00031-of-00044.safetensors +3 -0
  35. model-00032-of-00044.safetensors +3 -0
  36. model-00033-of-00044.safetensors +3 -0
  37. model-00034-of-00044.safetensors +3 -0
  38. model-00035-of-00044.safetensors +3 -0
  39. model-00036-of-00044.safetensors +3 -0
  40. model-00037-of-00044.safetensors +3 -0
  41. model-00038-of-00044.safetensors +3 -0
  42. model-00039-of-00044.safetensors +3 -0
  43. model-00040-of-00044.safetensors +3 -0
  44. model-00041-of-00044.safetensors +3 -0
  45. model-00042-of-00044.safetensors +3 -0
  46. model-00043-of-00044.safetensors +3 -0
  47. model-00044-of-00044.safetensors +3 -0
  48. model.safetensors.index.json +0 -0
  49. quantization_config.json +24 -0
  50. special_tokens_map.json +16 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/data6/models/Meta-Llama-3.1-405B-Instruct",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": [
10
+ 128001,
11
+ 128008,
12
+ 128009
13
+ ],
14
+ "head_dim": 128,
15
+ "hidden_act": "silu",
16
+ "hidden_size": 16384,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 53248,
19
+ "max_position_embeddings": 131072,
20
+ "mlp_bias": false,
21
+ "model_type": "llama",
22
+ "num_attention_heads": 128,
23
+ "num_hidden_layers": 126,
24
+ "num_key_value_heads": 16,
25
+ "pretraining_tp": 1,
26
+ "quantization_config": {
27
+ "amp": true,
28
+ "autoround_version": "0.4.3",
29
+ "backend": "auto_round:gptq:exllamav2",
30
+ "batch_size": 1,
31
+ "bits": 4,
32
+ "data_type": "int",
33
+ "dataset": "NeelNanda/pile-10k",
34
+ "enable_minmax_tuning": true,
35
+ "enable_norm_bias_tuning": false,
36
+ "enable_quanted_input": true,
37
+ "gradient_accumulate_steps": 4,
38
+ "group_size": -1,
39
+ "iters": 200,
40
+ "low_gpu_mem_usage": true,
41
+ "lr": 0.005,
42
+ "minmax_lr": 0.005,
43
+ "nsamples": 128,
44
+ "quant_method": "intel/auto-round",
45
+ "scale_dtype": "torch.float16",
46
+ "seqlen": 2048,
47
+ "sym": true,
48
+ "to_quant_block_names": null
49
+ },
50
+ "rms_norm_eps": 1e-05,
51
+ "rope_scaling": null,
52
+ "rope_theta": 500000.0,
53
+ "tie_word_embeddings": false,
54
+ "torch_dtype": "bfloat16",
55
+ "transformers_version": "4.47.0",
56
+ "use_cache": true,
57
+ "vocab_size": 128256
58
+ }
generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 128000,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 128001,
6
+ 128008,
7
+ 128009
8
+ ],
9
+ "temperature": 0.6,
10
+ "top_p": 0.9,
11
+ "transformers_version": "4.47.0"
12
+ }
model-00001-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0e012ec3f44deea4607dbebbbe67e1ae7c0c16050cd756518548e30f7d65428
3
+ size 4941445440
model-00002-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2f8259ff72a4e903925be56639246e4510ac4419cb11b6ff0cc25ad32b8746c
3
+ size 4835061656
model-00003-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:149f5e7eaadabc17813947121b978471dc7bc7338ffddf34fabda3e1111e6b98
3
+ size 4835061656
model-00004-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5167bf2c990e8f58a535eebd0c9c2ef2668e5115370f7757b71c18994de1e55d
3
+ size 4835061656
model-00005-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d44db660ffa81ba4afbeab4ac5ca127499367b9a13d19878bbf58c9f18f4ee8
3
+ size 4835061736
model-00006-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c3855cf7aecdf9fae01a3632e9ee90a17434ac4de35f7227e6b4c5b78d03281
3
+ size 4835061752
model-00007-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d57299a546ab60d40a468751a330663338593ed74a9f5465be698ff2695bb0bb
3
+ size 4835061752
model-00008-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f47b6c03f6bcb2e59285a0a403b3504e88cccbe869d74c690f264869cf1a9d54
3
+ size 4835061752
model-00009-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e477431cc9ef00190fa9ef2954197fb39068ea162aa3d160d695d65a9aa93dac
3
+ size 4835061752
model-00010-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9d1ddf3a47512dac382a0b3546950c2e5863ee228c8945cb5e2bee5d578f394
3
+ size 4835061752
model-00011-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0c2e19f2bfb9bfe837f2e8ee4d7819e202e2db3030a5d9feeec8b8cf0b15e48
3
+ size 4835061752
model-00012-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c11ec4493cf163310f26029858759d236d04b0f6378ec4eb7cbf6283821c8e8
3
+ size 4835061752
model-00013-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a206bf08b0b9048548879e3046e07361e22ff992df307488a540d2df1f4f82f8
3
+ size 4835061752
model-00014-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0252d569d041e4c5888f71b756c9c730f7c3fadfef597f51587aa1b9099d46f8
3
+ size 4835061752
model-00015-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:debb9846b20a6cda398269af579dd3da5f293143507c804ed02d57a4d3c2e867
3
+ size 4835061752
model-00016-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:356ffcfa18ac44c18f5557391279d869bc5986deca193f3e7f385ddbc0e40f48
3
+ size 4835061752
model-00017-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41e3b385c7345b5d820f28b6114ba9e212632ac7a785572ae6933515717f96cc
3
+ size 4835061752
model-00018-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a80d005a53159a2fe5fcd1d65d0b1e3119cd0a50ef0392868019f5303fc92fb
3
+ size 4835061752
model-00019-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5edd68e3305fa4d6c535efb69369093e42d4841ba61ff202f515791426fa57f0
3
+ size 4835061752
model-00020-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49e1956c7d365141cedda43d5f62bffd60b6812c22fd4912746cafa9bce0d357
3
+ size 4835061752
model-00021-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27edf55ba8497284bb2684a267efa8adfdc10af2f2cdebd46bdb73f32727b6be
3
+ size 4835061752
model-00022-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e52d195e2b475c3b632b66e3311ad056d0b6b58853d37df94f826e25ead431bd
3
+ size 4835061752
model-00023-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:db951f808f3944001357bf8e5bf4aa6fcb022a9d1685966872b729d72e744b8d
3
+ size 4835061752
model-00024-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae76848da89b0300f8e649d1c49c9de351723c2692c80bc53715567e6165a92b
3
+ size 4835061752
model-00025-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09417331a6dacd03dc6273a5e62917659dd9c7b030e8f90d88354619cb4d6257
3
+ size 4835061752
model-00026-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef43240ab186dd84029eb2e2c78843be4c0c9b4acb85ed875821e62d98d2630c
3
+ size 4835061752
model-00027-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9dc748768c9df33ae5028f9cb9382bdc938c058b47a9001da5cd73309b4ac348
3
+ size 4835061752
model-00028-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd52c89e204eb68f397ecab0264591d3394e972c31b4a4b7b332a71b637b1713
3
+ size 4835061752
model-00029-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d98a912a986079821863e330ba02b0230a2ca4ebd3594aedf5ada74685a8a339
3
+ size 4835061752
model-00030-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e27bd3d17d644d72c8a93ae3a7f544f56b72b13b075d0ceb1283a439b13d8397
3
+ size 4835061752
model-00031-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b7dd207b62db6f459c3142cacd4316078f7cfed927a5c1b5dd586598bde4e66c
3
+ size 4835061752
model-00032-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:363f75e93850dd5bbf21085c46998d2de4be8fe7ea3e6df86be2bb01c60320a3
3
+ size 4835061752
model-00033-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0a6f8859542abce3a64b8490dcce0a74577f7e68a792dc9ee9d2d977a2e81ec
3
+ size 4835061752
model-00034-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d051d247352a1176b33fdede0e1fb8d20863cd5bc6e6b3ca9b1c7d2f1e3f475d
3
+ size 4835061752
model-00035-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d27196eb6805942f7fe28e98ed367e836e0bb9218923e7538ea4e09fb7fd31c1
3
+ size 4835061832
model-00036-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87f2fea2a9d710060eb986ea952f5feb1ce948b229236ca499687146df196032
3
+ size 4835061840
model-00037-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5db7c3d7fa72c39de61c7c1aced4528eb7cafb5fbc32546e63ee7b8864f8d192
3
+ size 4835061840
model-00038-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88ad6d6599bc4299ca5a292c3863fc8a6cdc86ffe5d151fb914494ed685a9ae3
3
+ size 4835061840
model-00039-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79fba7636576365e5b266a55cd1c56b2d40c18f1fb98dc52abb52a1c136ca6a4
3
+ size 4835061840
model-00040-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d570b4ccaf60a7f4d0cfbb999dae5f10fb47b6e35083804469a764bada4dee66
3
+ size 4835061840
model-00041-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8660c30ebc4ce89e1d9f71fd5049ff8b9001d0efc9fc9ec0a7c5b51de1c244f
3
+ size 4835061840
model-00042-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:475c7145b02f53c208e53add8e3defcdfd08d6283d71d498ef21a3a52c7d11b5
3
+ size 4835061840
model-00043-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:051d02b61978157fb7820ce51a20b6918d56835a6cc856e72dfac5cf675dfc35
3
+ size 4096341936
model-00044-of-00044.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71b079a02069ebe5a5f4f8135815f136afb1d6fc4423620dcb2a14fa399cdf12
3
+ size 4202692736
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
quantization_config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bits": 4,
3
+ "group_size": -1,
4
+ "sym": true,
5
+ "data_type": "int",
6
+ "enable_quanted_input": true,
7
+ "enable_minmax_tuning": true,
8
+ "seqlen": 2048,
9
+ "batch_size": 1,
10
+ "scale_dtype": "torch.float16",
11
+ "lr": 0.005,
12
+ "minmax_lr": 0.005,
13
+ "gradient_accumulate_steps": 4,
14
+ "iters": 200,
15
+ "amp": true,
16
+ "nsamples": 128,
17
+ "low_gpu_mem_usage": true,
18
+ "to_quant_block_names": null,
19
+ "enable_norm_bias_tuning": false,
20
+ "dataset": "NeelNanda/pile-10k",
21
+ "autoround_version": "0.4.3",
22
+ "quant_method": "intel/auto-round",
23
+ "backend": "auto_round:gptq:exllamav2"
24
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|begin_of_text|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|eot_id|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ }
16
+ }