usholanb commited on
Commit
8212f7f
1 Parent(s): 0c47618

Model save

Browse files
README.md ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ base_model: HuggingFaceH4/zephyr-7b-beta
4
+ tags:
5
+ - generated_from_trainer
6
+ model-index:
7
+ - name: zephyr-7b-sft-lora
8
+ results: []
9
+ ---
10
+
11
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
12
+ should probably proofread and complete it, then remove this comment. -->
13
+
14
+ # zephyr-7b-sft-lora
15
+
16
+ This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on an unknown dataset.
17
+ It achieves the following results on the evaluation set:
18
+ - Loss: 1.7089
19
+
20
+ ## Model description
21
+
22
+ More information needed
23
+
24
+ ## Intended uses & limitations
25
+
26
+ More information needed
27
+
28
+ ## Training and evaluation data
29
+
30
+ More information needed
31
+
32
+ ## Training procedure
33
+
34
+ ### Training hyperparameters
35
+
36
+ The following hyperparameters were used during training:
37
+ - learning_rate: 2e-05
38
+ - train_batch_size: 1
39
+ - eval_batch_size: 1
40
+ - seed: 42
41
+ - distributed_type: multi-GPU
42
+ - gradient_accumulation_steps: 128
43
+ - total_train_batch_size: 128
44
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
45
+ - lr_scheduler_type: cosine
46
+ - num_epochs: 1
47
+
48
+ ### Training results
49
+
50
+ | Training Loss | Epoch | Step | Validation Loss |
51
+ |:-------------:|:-----:|:----:|:---------------:|
52
+ | 1.7691 | 0.95 | 15 | 1.7089 |
53
+
54
+
55
+ ### Framework versions
56
+
57
+ - Transformers 4.35.0
58
+ - Pytorch 2.1.1+cu121
59
+ - Datasets 2.14.6
60
+ - Tokenizers 0.14.1
adapter_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "HuggingFaceH4/zephyr-7b-beta",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.1,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 64,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "q_proj",
20
+ "o_proj",
21
+ "v_proj",
22
+ "k_proj"
23
+ ],
24
+ "task_type": "CAUSAL_LM"
25
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb9068e36c387a151938423aa8410cd3cbb12bae924c09fbfd44747411dd0ae2
3
+ size 218138576
all_results.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.95,
3
+ "eval_loss": 1.708883285522461,
4
+ "eval_runtime": 2326.7582,
5
+ "eval_samples": 1086,
6
+ "eval_samples_per_second": 0.467,
7
+ "eval_steps_per_second": 0.467,
8
+ "train_loss": 1.773037075996399,
9
+ "train_runtime": 7574.4417,
10
+ "train_samples": 2026,
11
+ "train_samples_per_second": 0.267,
12
+ "train_steps_per_second": 0.002
13
+ }
eval_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.95,
3
+ "eval_loss": 1.708883285522461,
4
+ "eval_runtime": 2326.7582,
5
+ "eval_samples": 1086,
6
+ "eval_samples_per_second": 0.467,
7
+ "eval_steps_per_second": 0.467
8
+ }
runs/Dec04_13-02-48_ip-172-16-12-85.ec2.internal/events.out.tfevents.1701695144.ip-172-16-12-85.ec2.internal.1417.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0955b38af18d01baa6c8d7916fa6def0e69f07c480703fb63f668cd1c6483e8
3
+ size 4664
runs/Dec04_13-16-47_ip-172-16-12-85.ec2.internal/events.out.tfevents.1701695829.ip-172-16-12-85.ec2.internal.18135.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7480c2bc436eec0d913ade5c5b733c35ac992c01eedef92c226b47eb3d4bcc6
3
+ size 6050
runs/Dec04_20-17-32_ip-172-16-12-85.ec2.internal/events.out.tfevents.1701721125.ip-172-16-12-85.ec2.internal.26822.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7270155d9ed8e55bc492972e8662a8c089a80b10255d53673f582beeb9330490
3
+ size 4664
runs/Dec04_20-26-31_ip-172-16-12-85.ec2.internal/events.out.tfevents.1701721666.ip-172-16-12-85.ec2.internal.5396.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bcfa76a5a15f0cac53b9613a5db8565644d48404dbafcb819e9840eaa534d6e0
3
+ size 4664
runs/Dec05_12-10-17_ip-172-16-125-111.ec2.internal/events.out.tfevents.1701778372.ip-172-16-125-111.ec2.internal.7639.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adbb0969d4504766f4599bee91ed33d3e26d8959c66c0b2733faba27f6c85d75
3
+ size 5922
runs/Dec05_15-55-33_ip-172-16-125-111.ec2.internal/events.out.tfevents.1701791749.ip-172-16-125-111.ec2.internal.15369.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e42cb338a768ecb7be79d5f7642651a1ad64d45a427cd480947a9643bbe914a
3
+ size 5920
runs/Dec05_15-55-33_ip-172-16-125-111.ec2.internal/events.out.tfevents.1701801650.ip-172-16-125-111.ec2.internal.15369.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05cface16662e929a9e69185bd67cd846a26276a7e05d936bd6301c24a898ca3
3
+ size 354
special_tokens_map.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<unk>",
4
+ "<s>",
5
+ "</s>"
6
+ ],
7
+ "bos_token": {
8
+ "content": "<s>",
9
+ "lstrip": false,
10
+ "normalized": false,
11
+ "rstrip": false,
12
+ "single_word": false
13
+ },
14
+ "eos_token": {
15
+ "content": "</s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "pad_token": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "unk_token": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ }
35
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ }
27
+ },
28
+ "additional_special_tokens": [
29
+ "<unk>",
30
+ "<s>",
31
+ "</s>"
32
+ ],
33
+ "bos_token": "<s>",
34
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
35
+ "clean_up_tokenization_spaces": false,
36
+ "eos_token": "</s>",
37
+ "legacy": true,
38
+ "model_max_length": 2048,
39
+ "pad_token": "</s>",
40
+ "sp_model_kwargs": {},
41
+ "spaces_between_special_tokens": false,
42
+ "tokenizer_class": "LlamaTokenizer",
43
+ "truncation_side": "left",
44
+ "unk_token": "<unk>",
45
+ "use_default_system_prompt": true
46
+ }
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.95,
3
+ "train_loss": 1.773037075996399,
4
+ "train_runtime": 7574.4417,
5
+ "train_samples": 2026,
6
+ "train_samples_per_second": 0.267,
7
+ "train_steps_per_second": 0.002
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.947680157946693,
5
+ "eval_steps": 500,
6
+ "global_step": 15,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.06,
13
+ "learning_rate": 1.9781476007338058e-05,
14
+ "loss": 1.8617,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.32,
19
+ "learning_rate": 1.5000000000000002e-05,
20
+ "loss": 1.7726,
21
+ "step": 5
22
+ },
23
+ {
24
+ "epoch": 0.63,
25
+ "learning_rate": 5.000000000000003e-06,
26
+ "loss": 1.7596,
27
+ "step": 10
28
+ },
29
+ {
30
+ "epoch": 0.95,
31
+ "learning_rate": 0.0,
32
+ "loss": 1.7691,
33
+ "step": 15
34
+ },
35
+ {
36
+ "epoch": 0.95,
37
+ "eval_loss": 1.7088831663131714,
38
+ "eval_runtime": 2327.3352,
39
+ "eval_samples_per_second": 0.467,
40
+ "eval_steps_per_second": 0.467,
41
+ "step": 15
42
+ },
43
+ {
44
+ "epoch": 0.95,
45
+ "step": 15,
46
+ "total_flos": 1.6904795211798938e+17,
47
+ "train_loss": 1.773037075996399,
48
+ "train_runtime": 7574.4417,
49
+ "train_samples_per_second": 0.267,
50
+ "train_steps_per_second": 0.002
51
+ }
52
+ ],
53
+ "logging_steps": 5,
54
+ "max_steps": 15,
55
+ "num_train_epochs": 1,
56
+ "save_steps": 500,
57
+ "total_flos": 1.6904795211798938e+17,
58
+ "trial_name": null,
59
+ "trial_params": null
60
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c7753dc1d8dfe79da0ea7729a40eb3db8021773ed8b02377466b08c98358e1c
3
+ size 4664