aipib commited on
Commit
dc71c36
1 Parent(s): f6a8d3e

Upload folder using huggingface_hub

Browse files
README.md ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - merge
4
+ - mergekit
5
+ - lazymergekit
6
+ - kcoopermiller/llm-jp-1.3b-v1.0-aya
7
+ - llm-jp/llm-jp-1.3b-v1.0
8
+ base_model:
9
+ - kcoopermiller/llm-jp-1.3b-v1.0-aya
10
+ - llm-jp/llm-jp-1.3b-v1.0
11
+ ---
12
+
13
+ # llmjp-slerp2
14
+
15
+ llmjp-slerp2 is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
16
+ * [kcoopermiller/llm-jp-1.3b-v1.0-aya](https://huggingface.co/kcoopermiller/llm-jp-1.3b-v1.0-aya)
17
+ * [llm-jp/llm-jp-1.3b-v1.0](https://huggingface.co/llm-jp/llm-jp-1.3b-v1.0)
18
+
19
+ ## 🧩 Configuration
20
+
21
+ ```yaml
22
+ slices:
23
+ - sources:
24
+ - model: kcoopermiller/llm-jp-1.3b-v1.0-aya
25
+ layer_range: [0, 24]
26
+ - model: llm-jp/llm-jp-1.3b-v1.0
27
+ layer_range: [0, 24]
28
+ merge_method: slerp
29
+ base_model: llm-jp/llm-jp-1.3b-v1.0
30
+ parameters:
31
+ t:
32
+ - filter: self_attn
33
+ value: [0, 0.5, 0.3, 0.7, 1]
34
+ - filter: mlp
35
+ value: [1, 0.5, 0.7, 0.3, 0]
36
+ - value: 0.5
37
+ dtype: bfloat16
38
+ ```
39
+
40
+ ## 💻 Usage
41
+
42
+ ```python
43
+ !pip install -qU transformers accelerate
44
+
45
+ from transformers import AutoTokenizer
46
+ import transformers
47
+ import torch
48
+
49
+ model = "aipib/llmjp-slerp2"
50
+ messages = [{"role": "user", "content": "What is a large language model?"}]
51
+
52
+ tokenizer = AutoTokenizer.from_pretrained(model)
53
+ prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
54
+ pipeline = transformers.pipeline(
55
+ "text-generation",
56
+ model=model,
57
+ torch_dtype=torch.float16,
58
+ device_map="auto",
59
+ )
60
+
61
+ outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
62
+ print(outputs[0]["generated_text"])
63
+ ```
config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "llm-jp/llm-jp-1.3b-v1.0",
3
+ "activation_function": "gelu",
4
+ "architectures": [
5
+ "GPT2LMHeadModel"
6
+ ],
7
+ "attn_pdrop": 0.1,
8
+ "bos_token_id": 7,
9
+ "embd_pdrop": 0.1,
10
+ "eos_token_id": 7,
11
+ "gradient_checkpointing": false,
12
+ "initializer_range": 0.02,
13
+ "layer_norm_epsilon": 1e-05,
14
+ "model_type": "gpt2",
15
+ "n_ctx": 1024,
16
+ "n_embd": 2048,
17
+ "n_head": 16,
18
+ "n_inner": 8192,
19
+ "n_layer": 24,
20
+ "n_positions": 2048,
21
+ "reorder_and_upcast_attn": false,
22
+ "resid_pdrop": 0.1,
23
+ "scale_attn_by_inverse_layer_idx": false,
24
+ "scale_attn_weights": true,
25
+ "summary_activation": null,
26
+ "summary_first_dropout": 0.1,
27
+ "summary_proj_to_labels": true,
28
+ "summary_type": "cls_index",
29
+ "summary_use_proj": true,
30
+ "torch_dtype": "bfloat16",
31
+ "transformers_version": "4.41.2",
32
+ "use_cache": true,
33
+ "vocab_size": 50688
34
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ slices:
3
+ - sources:
4
+ - model: kcoopermiller/llm-jp-1.3b-v1.0-aya
5
+ layer_range: [0, 24]
6
+ - model: llm-jp/llm-jp-1.3b-v1.0
7
+ layer_range: [0, 24]
8
+ merge_method: slerp
9
+ base_model: llm-jp/llm-jp-1.3b-v1.0
10
+ parameters:
11
+ t:
12
+ - filter: self_attn
13
+ value: [0, 0.5, 0.3, 0.7, 1]
14
+ - filter: mlp
15
+ value: [1, 0.5, 0.7, 0.3, 0]
16
+ - value: 0.5
17
+ dtype: bfloat16
model-00001-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d92ccd6642c50ac11ea4b18b73520cdfd9d0e42de1630b7222897bfb7e885d64
3
+ size 973623440
model-00002-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b71639e373c2c3438ea60929fd228b3b2b51f33b422f65bc076487e6eea66186
3
+ size 973619232
model-00003-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0838db1b0f5440798abf31c3866c7b9dc6004981477da6ac1c7aa198f558ac69
3
+ size 685999688
model.safetensors.index.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"metadata": {"mergekit_version": "0.0.4.2", "total_size": 2633211904}, "weight_map": {"transformer.h.0.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.0.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.0.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.0.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.0.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.0.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.0.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.0.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.0.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.0.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.0.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.0.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.1.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.1.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.1.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.1.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.1.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.1.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.1.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.1.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.1.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.1.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.1.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.1.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.10.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.10.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.10.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.10.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.10.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.10.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.10.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.10.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.10.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.10.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.10.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.10.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.11.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.11.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.11.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.11.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.11.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.11.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.11.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.11.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.11.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.11.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.11.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.11.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.12.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.12.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.12.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.12.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.12.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.12.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.12.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.12.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.12.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.12.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.12.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.12.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.13.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.13.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.13.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.13.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.13.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.13.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.13.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.13.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.13.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.13.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.13.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.13.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.14.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.14.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.14.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.14.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.14.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.14.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.14.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.14.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.14.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.14.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.14.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.14.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.15.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.15.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.15.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.15.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.15.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.15.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.15.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.15.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.15.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.15.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.15.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.15.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.16.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.16.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.16.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.16.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.16.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.16.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.16.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.16.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.16.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.16.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.16.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.16.mlp.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.17.attn.c_attn.bias": "model-00001-of-00003.safetensors", "transformer.h.17.attn.c_attn.weight": "model-00001-of-00003.safetensors", "transformer.h.17.attn.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.17.attn.c_proj.weight": "model-00001-of-00003.safetensors", "transformer.h.17.ln_1.bias": "model-00001-of-00003.safetensors", "transformer.h.17.ln_1.weight": "model-00001-of-00003.safetensors", "transformer.h.17.ln_2.bias": "model-00001-of-00003.safetensors", "transformer.h.17.ln_2.weight": "model-00001-of-00003.safetensors", "transformer.h.17.mlp.c_fc.bias": "model-00001-of-00003.safetensors", "transformer.h.17.mlp.c_fc.weight": "model-00001-of-00003.safetensors", "transformer.h.17.mlp.c_proj.bias": "model-00001-of-00003.safetensors", "transformer.h.17.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.18.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.18.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.18.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.18.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.18.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.18.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.18.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.18.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.18.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.18.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.18.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.18.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.19.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.19.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.19.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.19.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.19.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.19.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.19.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.19.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.19.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.19.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.19.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.19.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.2.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.2.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.2.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.2.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.2.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.2.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.2.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.2.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.2.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.2.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.2.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.2.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.20.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.20.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.20.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.20.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.20.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.20.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.20.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.20.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.20.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.20.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.20.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.20.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.21.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.21.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.21.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.21.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.21.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.21.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.21.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.21.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.21.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.21.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.21.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.21.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.22.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.22.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.22.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.22.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.22.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.22.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.22.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.22.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.22.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.22.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.22.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.22.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.23.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.23.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.23.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.23.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.23.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.23.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.23.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.23.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.23.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.23.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.23.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.23.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.3.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.3.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.3.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.3.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.3.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.3.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.3.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.3.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.3.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.3.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.3.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.3.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.4.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.4.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.4.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.4.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.4.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.4.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.4.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.4.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.4.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.4.mlp.c_fc.weight": "model-00002-of-00003.safetensors", "transformer.h.4.mlp.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.4.mlp.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.5.attn.c_attn.bias": "model-00002-of-00003.safetensors", "transformer.h.5.attn.c_attn.weight": "model-00002-of-00003.safetensors", "transformer.h.5.attn.c_proj.bias": "model-00002-of-00003.safetensors", "transformer.h.5.attn.c_proj.weight": "model-00002-of-00003.safetensors", "transformer.h.5.ln_1.bias": "model-00002-of-00003.safetensors", "transformer.h.5.ln_1.weight": "model-00002-of-00003.safetensors", "transformer.h.5.ln_2.bias": "model-00002-of-00003.safetensors", "transformer.h.5.ln_2.weight": "model-00002-of-00003.safetensors", "transformer.h.5.mlp.c_fc.bias": "model-00002-of-00003.safetensors", "transformer.h.5.mlp.c_fc.weight": "model-00003-of-00003.safetensors", "transformer.h.5.mlp.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.5.mlp.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.6.attn.c_attn.bias": "model-00003-of-00003.safetensors", "transformer.h.6.attn.c_attn.weight": "model-00003-of-00003.safetensors", "transformer.h.6.attn.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.6.attn.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.6.ln_1.bias": "model-00003-of-00003.safetensors", "transformer.h.6.ln_1.weight": "model-00003-of-00003.safetensors", "transformer.h.6.ln_2.bias": "model-00003-of-00003.safetensors", "transformer.h.6.ln_2.weight": "model-00003-of-00003.safetensors", "transformer.h.6.mlp.c_fc.bias": "model-00003-of-00003.safetensors", "transformer.h.6.mlp.c_fc.weight": "model-00003-of-00003.safetensors", "transformer.h.6.mlp.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.6.mlp.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.7.attn.c_attn.bias": "model-00003-of-00003.safetensors", "transformer.h.7.attn.c_attn.weight": "model-00003-of-00003.safetensors", "transformer.h.7.attn.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.7.attn.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.7.ln_1.bias": "model-00003-of-00003.safetensors", "transformer.h.7.ln_1.weight": "model-00003-of-00003.safetensors", "transformer.h.7.ln_2.bias": "model-00003-of-00003.safetensors", "transformer.h.7.ln_2.weight": "model-00003-of-00003.safetensors", "transformer.h.7.mlp.c_fc.bias": "model-00003-of-00003.safetensors", "transformer.h.7.mlp.c_fc.weight": "model-00003-of-00003.safetensors", "transformer.h.7.mlp.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.7.mlp.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.8.attn.c_attn.bias": "model-00003-of-00003.safetensors", "transformer.h.8.attn.c_attn.weight": "model-00003-of-00003.safetensors", "transformer.h.8.attn.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.8.attn.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.8.ln_1.bias": "model-00003-of-00003.safetensors", "transformer.h.8.ln_1.weight": "model-00003-of-00003.safetensors", "transformer.h.8.ln_2.bias": "model-00003-of-00003.safetensors", "transformer.h.8.ln_2.weight": "model-00003-of-00003.safetensors", "transformer.h.8.mlp.c_fc.bias": "model-00003-of-00003.safetensors", "transformer.h.8.mlp.c_fc.weight": "model-00003-of-00003.safetensors", "transformer.h.8.mlp.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.8.mlp.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.9.attn.c_attn.bias": "model-00003-of-00003.safetensors", "transformer.h.9.attn.c_attn.weight": "model-00003-of-00003.safetensors", "transformer.h.9.attn.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.9.attn.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.h.9.ln_1.bias": "model-00003-of-00003.safetensors", "transformer.h.9.ln_1.weight": "model-00003-of-00003.safetensors", "transformer.h.9.ln_2.bias": "model-00003-of-00003.safetensors", "transformer.h.9.ln_2.weight": "model-00003-of-00003.safetensors", "transformer.h.9.mlp.c_fc.bias": "model-00003-of-00003.safetensors", "transformer.h.9.mlp.c_fc.weight": "model-00003-of-00003.safetensors", "transformer.h.9.mlp.c_proj.bias": "model-00003-of-00003.safetensors", "transformer.h.9.mlp.c_proj.weight": "model-00003-of-00003.safetensors", "transformer.ln_f.bias": "model-00003-of-00003.safetensors", "transformer.ln_f.weight": "model-00003-of-00003.safetensors", "transformer.wpe.weight": "model-00003-of-00003.safetensors", "transformer.wte.weight": "model-00003-of-00003.safetensors"}}
special_tokens_map.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "</s|LLM-jp>"
4
+ ],
5
+ "bos_token": {
6
+ "content": "<s|LLM-jp>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "cls_token": {
13
+ "content": "<CLS|LLM-jp>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false
18
+ },
19
+ "eos_token": {
20
+ "content": "<EOD|LLM-jp>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false
25
+ },
26
+ "mask_token": {
27
+ "content": "<mask|LLM-jp>",
28
+ "lstrip": false,
29
+ "normalized": false,
30
+ "rstrip": false,
31
+ "single_word": false
32
+ },
33
+ "pad_token": {
34
+ "content": "<pad|LLM-jp>",
35
+ "lstrip": false,
36
+ "normalized": false,
37
+ "rstrip": false,
38
+ "single_word": false
39
+ },
40
+ "sep_token": {
41
+ "content": "<SEP|LLM-jp>",
42
+ "lstrip": false,
43
+ "normalized": false,
44
+ "rstrip": false,
45
+ "single_word": false
46
+ },
47
+ "unk_token": {
48
+ "content": "<unk|LLM-jp>",
49
+ "lstrip": false,
50
+ "normalized": false,
51
+ "rstrip": false,
52
+ "single_word": false
53
+ }
54
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk|LLM-jp>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s|LLM-jp>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s|LLM-jp>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<mask|LLM-jp>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<pad|LLM-jp>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "<CLS|LLM-jp>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "6": {
52
+ "content": "<SEP|LLM-jp>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "7": {
60
+ "content": "<EOD|LLM-jp>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ }
67
+ },
68
+ "additional_special_tokens": [
69
+ "</s|LLM-jp>"
70
+ ],
71
+ "bos_token": "<s|LLM-jp>",
72
+ "clean_up_tokenization_spaces": false,
73
+ "cls_token": "<CLS|LLM-jp>",
74
+ "eod_token": "<EOD|LLM-jp>",
75
+ "eos_token": "<EOD|LLM-jp>",
76
+ "extra_ids": 0,
77
+ "mask_token": "<mask|LLM-jp>",
78
+ "model_max_length": 1000000000000000019884624838656,
79
+ "pad_token": "<pad|LLM-jp>",
80
+ "sep_token": "<SEP|LLM-jp>",
81
+ "sp_model_kwargs": {},
82
+ "tokenizer_class": "PreTrainedTokenizerFast",
83
+ "unk_token": "<unk|LLM-jp>"
84
+ }