Commit
•
4bfb5f2
1
Parent(s):
1d121b1
Upload folder using huggingface_hub
Browse files- README.md +15 -29
- adapter_config.json +1 -26
- config.json +3 -3
- model.safetensors +2 -2
- optimizer.pt +3 -0
- rng_state.pth +3 -0
- scheduler.pt +3 -0
- special_tokens_map.json +3 -21
- trainer_state.json +0 -0
- training_args.bin +3 -0
README.md
CHANGED
@@ -1,35 +1,21 @@
|
|
1 |
---
|
2 |
-
|
3 |
-
language:
|
4 |
-
- sw
|
5 |
---
|
6 |
-
|
7 |
-
===
|
8 |
|
9 |
-
## How to use
|
10 |
-
```python
|
11 |
-
from peft import AutoPeftModelForCausalLM
|
12 |
-
from transformers import AutoTokenizer
|
13 |
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
|
|
|
|
|
|
|
|
|
|
21 |
|
22 |
-
## Citation
|
23 |
-
```
|
24 |
-
@article{yamaguchi2024empirical,
|
25 |
-
title={An Empirical Study on Cross-lingual Vocabulary Adaptation for Efficient Generative {LLM} Inference},
|
26 |
-
author={Atsuki Yamaguchi and Aline Villavicencio and Nikolaos Aletras},
|
27 |
-
journal={ArXiv},
|
28 |
-
year={2024},
|
29 |
-
volume={abs/2402.10712},
|
30 |
-
url={https://arxiv.org/abs/2402.10712}
|
31 |
-
}
|
32 |
-
```
|
33 |
|
34 |
-
|
35 |
-
For more details, please visit https://github.com/gucci-j/llm-cva
|
|
|
1 |
---
|
2 |
+
library_name: peft
|
|
|
|
|
3 |
---
|
4 |
+
## Training procedure
|
|
|
5 |
|
|
|
|
|
|
|
|
|
6 |
|
7 |
+
The following `bitsandbytes` quantization config was used during training:
|
8 |
+
- quant_method: bitsandbytes
|
9 |
+
- load_in_8bit: True
|
10 |
+
- load_in_4bit: False
|
11 |
+
- llm_int8_threshold: 6.0
|
12 |
+
- llm_int8_skip_modules: None
|
13 |
+
- llm_int8_enable_fp32_cpu_offload: False
|
14 |
+
- llm_int8_has_fp16_weight: False
|
15 |
+
- bnb_4bit_quant_type: fp4
|
16 |
+
- bnb_4bit_use_double_quant: False
|
17 |
+
- bnb_4bit_compute_dtype: float32
|
18 |
+
### Framework versions
|
19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
+
- PEFT 0.5.0
|
|
adapter_config.json
CHANGED
@@ -1,26 +1 @@
|
|
1 |
-
{
|
2 |
-
"auto_mapping": null,
|
3 |
-
"base_model_name_or_path": "atsuki-yamaguchi/bloom-1b1-focus-sw",
|
4 |
-
"bias": "none",
|
5 |
-
"fan_in_fan_out": false,
|
6 |
-
"inference_mode": true,
|
7 |
-
"init_lora_weights": true,
|
8 |
-
"layers_pattern": null,
|
9 |
-
"layers_to_transform": null,
|
10 |
-
"lora_alpha": 32,
|
11 |
-
"lora_dropout": 0.05,
|
12 |
-
"modules_to_save": [
|
13 |
-
"lm_head",
|
14 |
-
"word_embeddings"
|
15 |
-
],
|
16 |
-
"peft_type": "LORA",
|
17 |
-
"r": 8,
|
18 |
-
"revision": null,
|
19 |
-
"target_modules": [
|
20 |
-
"query_key_value",
|
21 |
-
"dense",
|
22 |
-
"dense_h_to_4h",
|
23 |
-
"dense_4h_to_h"
|
24 |
-
],
|
25 |
-
"task_type": "CAUSAL_LM"
|
26 |
-
}
|
|
|
1 |
+
{"auto_mapping": null, "base_model_name_or_path": "atsuki-yamaguchi/bloom-1b1-focus-sw", "bias": "none", "fan_in_fan_out": false, "inference_mode": true, "init_lora_weights": true, "layers_pattern": null, "layers_to_transform": null, "lora_alpha": 32, "lora_dropout": 0.05, "modules_to_save": ["lm_head", "word_embeddings"], "peft_type": "LORA", "r": 8, "revision": null, "target_modules": ["query_key_value", "dense", "dense_h_to_4h", "dense_4h_to_h"], "task_type": "CAUSAL_LM"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
config.json
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
{
|
2 |
-
"_name_or_path": "/
|
3 |
"apply_residual_connection_post_layernorm": false,
|
4 |
"architectures": [
|
5 |
"BloomForCausalLM"
|
@@ -24,9 +24,9 @@
|
|
24 |
"skip_bias_add": true,
|
25 |
"skip_bias_add_qkv": false,
|
26 |
"slow_but_exact": false,
|
27 |
-
"torch_dtype": "
|
28 |
"transformers_version": "4.35.0.dev0",
|
29 |
"unk_token_id": 0,
|
30 |
"use_cache": true,
|
31 |
-
"vocab_size":
|
32 |
}
|
|
|
1 |
{
|
2 |
+
"_name_or_path": "bigscience/bloom-1b1",
|
3 |
"apply_residual_connection_post_layernorm": false,
|
4 |
"architectures": [
|
5 |
"BloomForCausalLM"
|
|
|
24 |
"skip_bias_add": true,
|
25 |
"skip_bias_add_qkv": false,
|
26 |
"slow_but_exact": false,
|
27 |
+
"torch_dtype": "float64",
|
28 |
"transformers_version": "4.35.0.dev0",
|
29 |
"unk_token_id": 0,
|
30 |
"use_cache": true,
|
31 |
+
"vocab_size": 50272
|
32 |
}
|
model.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:596241152abf1a3a5ad3e83088a7231a3902b45d41bb70408bccaceab4e0dacd
|
3 |
+
size 3337442912
|
optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:eccf3e55c1e2d054f193c29d25c0af8c270982a1b8f2aba50ed3aac4cc76838d
|
3 |
+
size 319055994
|
rng_state.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:14dc63b8088e5f154919e2f30dfb59a9c87431aa177fd55c6cbb110da46ba761
|
3 |
+
size 14244
|
scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9b189bad7ffd6901fc2b4d75323e964d16850111ccbafc29a9e9c93d6da4cae4
|
3 |
+
size 1064
|
special_tokens_map.json
CHANGED
@@ -1,23 +1,5 @@
|
|
1 |
{
|
2 |
-
"bos_token":
|
3 |
-
|
4 |
-
|
5 |
-
"normalized": false,
|
6 |
-
"rstrip": false,
|
7 |
-
"single_word": false
|
8 |
-
},
|
9 |
-
"eos_token": {
|
10 |
-
"content": "<|endoftext|>",
|
11 |
-
"lstrip": false,
|
12 |
-
"normalized": false,
|
13 |
-
"rstrip": false,
|
14 |
-
"single_word": false
|
15 |
-
},
|
16 |
-
"unk_token": {
|
17 |
-
"content": "<|endoftext|>",
|
18 |
-
"lstrip": false,
|
19 |
-
"normalized": false,
|
20 |
-
"rstrip": false,
|
21 |
-
"single_word": false
|
22 |
-
}
|
23 |
}
|
|
|
1 |
{
|
2 |
+
"bos_token": "<|endoftext|>",
|
3 |
+
"eos_token": "<|endoftext|>",
|
4 |
+
"unk_token": "<|endoftext|>"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
}
|
trainer_state.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:70280659792e9b9aa23a298f2a63cdd3431eadfb01d1307afc5a9c8827e24531
|
3 |
+
size 4664
|