venetis commited on
Commit
f560d64
1 Parent(s): b9aa706

Training in progress, step 98

Browse files
adapter_config.json CHANGED
@@ -14,19 +14,22 @@
14
  "lora_dropout": 0.05,
15
  "megatron_config": null,
16
  "megatron_core": "megatron.core",
17
- "modules_to_save": null,
 
 
 
18
  "peft_type": "LORA",
19
  "r": 32,
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "down_proj",
24
- "gate_proj",
25
- "o_proj",
26
- "v_proj",
27
  "up_proj",
 
 
 
28
  "k_proj",
29
- "q_proj"
 
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
 
14
  "lora_dropout": 0.05,
15
  "megatron_config": null,
16
  "megatron_core": "megatron.core",
17
+ "modules_to_save": [
18
+ "embed_tokens",
19
+ "lm_head"
20
+ ],
21
  "peft_type": "LORA",
22
  "r": 32,
23
  "rank_pattern": {},
24
  "revision": null,
25
  "target_modules": [
 
 
 
 
26
  "up_proj",
27
+ "o_proj",
28
+ "gate_proj",
29
+ "q_proj",
30
  "k_proj",
31
+ "down_proj",
32
+ "v_proj"
33
  ],
34
  "task_type": "CAUSAL_LM",
35
  "use_dora": false,
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:58945988f47fec23a3d6648fb86994a9aec471ebc73c59a41f9a1885c2e74147
3
- size 167832688
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c348ce9f8dd88f5d0f81aadac707cf370948de43eeb3a56b9f8a732329f3cec
3
+ size 6471972152
config.json CHANGED
@@ -6,7 +6,7 @@
6
  "attention_bias": false,
7
  "attention_dropout": 0.0,
8
  "bos_token_id": 128000,
9
- "eos_token_id": 128001,
10
  "hidden_act": "silu",
11
  "hidden_size": 4096,
12
  "initializer_range": 0.02,
@@ -24,5 +24,5 @@
24
  "torch_dtype": "bfloat16",
25
  "transformers_version": "4.40.2",
26
  "use_cache": false,
27
- "vocab_size": 128256
28
  }
 
6
  "attention_bias": false,
7
  "attention_dropout": 0.0,
8
  "bos_token_id": 128000,
9
+ "eos_token_id": 128256,
10
  "hidden_act": "silu",
11
  "hidden_size": 4096,
12
  "initializer_range": 0.02,
 
24
  "torch_dtype": "bfloat16",
25
  "transformers_version": "4.40.2",
26
  "use_cache": false,
27
+ "vocab_size": 128258
28
  }
special_tokens_map.json CHANGED
@@ -7,7 +7,7 @@
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "<|end_of_text|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
 
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "<|im_end|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
tokenizer.json CHANGED
@@ -2306,6 +2306,24 @@
2306
  "rstrip": false,
2307
  "normalized": false,
2308
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2309
  }
2310
  ],
2311
  "normalizer": null,
 
2306
  "rstrip": false,
2307
  "normalized": false,
2308
  "special": true
2309
+ },
2310
+ {
2311
+ "id": 128256,
2312
+ "content": "<|im_end|>",
2313
+ "single_word": false,
2314
+ "lstrip": false,
2315
+ "rstrip": false,
2316
+ "normalized": false,
2317
+ "special": false
2318
+ },
2319
+ {
2320
+ "id": 128257,
2321
+ "content": "<|im_start|>",
2322
+ "single_word": false,
2323
+ "lstrip": false,
2324
+ "rstrip": false,
2325
+ "normalized": false,
2326
+ "special": false
2327
  }
2328
  ],
2329
  "normalizer": null,
tokenizer_config.json CHANGED
@@ -2047,12 +2047,28 @@
2047
  "rstrip": false,
2048
  "single_word": false,
2049
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
2054
  "clean_up_tokenization_spaces": true,
2055
- "eos_token": "<|end_of_text|>",
2056
  "model_input_names": [
2057
  "input_ids",
2058
  "attention_mask"
 
2047
  "rstrip": false,
2048
  "single_word": false,
2049
  "special": true
2050
+ },
2051
+ "128256": {
2052
+ "content": "<|im_end|>",
2053
+ "lstrip": false,
2054
+ "normalized": false,
2055
+ "rstrip": false,
2056
+ "single_word": false,
2057
+ "special": false
2058
+ },
2059
+ "128257": {
2060
+ "content": "<|im_start|>",
2061
+ "lstrip": false,
2062
+ "normalized": false,
2063
+ "rstrip": false,
2064
+ "single_word": false,
2065
+ "special": false
2066
  }
2067
  },
2068
  "bos_token": "<|begin_of_text|>",
2069
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
2070
  "clean_up_tokenization_spaces": true,
2071
+ "eos_token": "<|im_end|>",
2072
  "model_input_names": [
2073
  "input_ids",
2074
  "attention_mask"
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8a150e935c33cd3a4b50b76e7477c48fc36c32204c30a7f82fedd479e5a33811
3
  size 5880
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f66d2a36b6f48c04b32105395da62a475fcc59ea88cc27ae0d13084ff2f69f64
3
  size 5880