initial commit

README.md

@@ -1,3 +1,69 @@
 ---
 license: apache-2.0
+language:
+- ja
 ---
+
+## モデル概要
+AWSのtrn1インスタンスを用いて開発した大喜利言語モデルです。  
+事前学習後に大喜利データでFine-tuningしています。
+
+* Architecture: GPT2
+* Vocab size: 44880
+* Model size: 6B params
+* License: [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)
+* Library: [aws-neuron-reference-for-megatron-lm](https://github.com/aws-neuron/aws-neuron-reference-for-megatron-lm)
+
+## 学習データ
+以下のコーパスを使用して、事前学習を行いました。その際のトークン数は477億トークンでした。
+* [C4](https://huggingface.co/datasets/mc4)の日本語データ
+* [CC-100](https://huggingface.co/datasets/cc100)の日本語データ
+* [OSCAR](https://huggingface.co/datasets/oscar)の日本語データ
+* [Wikipedia](https://ja.wikipedia.org/wiki/%E3%83%A1%E3%82%A4%E3%83%B3%E3%83%9A%E3%83%BC%E3%82%B8)の日本語ダンプデータ
+* 自社データ
+
+Fine-tuningは、693万件の大喜利データを用いて行いました。
+
+## 使用方法
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_name = "watashiha/watashiha-gpt-6b"
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
+
+if torch.cuda.is_available():
+    model = model.to("cuda")
+
+text = "お題:ホラー映画の「○○○から逃げろ！」<SEP>回答:"
+token_ids = tokenizer.encode(text, add_special_tokens=False, return_tensors="pt").to(model.device)
+
+output_ids = model.generate(
+    token_ids,
+    do_sample=True,
+    max_new_tokens=32,
+    top_p=0.9,
+    top_k=50,
+    pad_token_id=tokenizer.pad_token_id,
+    eos_token_id=tokenizer.eos_token_id,
+)
+output = tokenizer.decode(output_ids.tolist()[0], skip_special_tokens=True)
+print(output)
+"""お題:ホラー映画の「○○○から逃げろ！」<SEP>回答:怖いもの知らずの大学生"""
+```
+
+## 性能比較
+以下は各モデルを同様の条件でFine-tuningし、出力させたボケをケータイ大喜利レジェンドに4段階で評価してもらった結果です。  
+
+圏外:お題を日本語として理解できていない  
+1本:お題を理解はできているがボケとして成立していない（面白みがない）  
+2本:ボケとして成立している（面白みがある）  
+3本:面白い（一定以上の面白さがある）  
+
+|              | 圏外 | 1本 | 2本 | 3本 |
+|--------------|------|-----|-----|-----|
+| watashiha-gpt-6b           | 77   | 204 | 175 | 44  |
+|[rinna/japanese-gpt-neox-3.6b](https://huggingface.co/rinna/japanese-gpt-neox-3.6b)      | 88   | 194 | 185 | 30  |
+| [stabilityai/japanese-stablelm-base-alpha-7b](https://huggingface.co/stabilityai/japanese-stablelm-base-alpha-7b)  | 96   | 164 | 196 | 43  |
+| [elyza/ELYZA-japanese-Llama-2-7b-fast](https://huggingface.co/elyza/ELYZA-japanese-Llama-2-7b-fast)     | 75  | 197 | 198 | 25  |

added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "<CLS>": 44876,
+  "<EOD>": 44878,
+  "<MASK>": 44879,
+  "<SEP>": 44877
+}

config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "./watashiha-gpt-6b",
+  "activation_function": "gelu",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.0,
+  "bos_token_id": 44879,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 44879,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 4096,
+  "n_head": 32,
+  "n_inner": 16384,
+  "n_layer": 28,
+  "n_positions": 2048,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.0,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.2",
+  "use_cache": true,
+  "vocab_size": 44880
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 44879,
+  "eos_token_id": 44879,
+  "transformers_version": "4.35.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:914ba157d8eeb920da55a9acad79fc49466b05413a223e4557e8a8ae3733b589
+size 4949088944

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d76de14762178279d2f2d17d9f180a950d097139a250387a57cebd0bd3f0ca3
+size 4967382264

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68380eaf2a6204f71e77085ae7ead33a405fe825f457526d5dac4ecd35c7e5d1
+size 1745286544

model.safetensors.index.json

@@ -0,0 +1,347 @@
+{
+  "metadata": {
+    "total_size": 11661721600
+  },
+  "weight_map": {
+    "transformer.h.0.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.0.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.0.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.0.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.0.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.0.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.0.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.0.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.0.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.0.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.0.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.0.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.1.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.1.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.1.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.1.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.1.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.1.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.1.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.1.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.1.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.1.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.1.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.1.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.10.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.10.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.10.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.10.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.10.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.10.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.10.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.10.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.10.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.10.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.10.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.10.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.11.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.11.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.11.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.11.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.11.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.11.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.11.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.11.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.11.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.11.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.11.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.11.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.12.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.12.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.12.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.12.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.12.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.12.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.12.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.12.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.12.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.12.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.12.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.12.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.13.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.13.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.13.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.13.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.13.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.13.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.13.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.13.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.13.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.13.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.13.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.13.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.14.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.14.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.14.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.14.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.14.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.14.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.14.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.14.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.14.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.14.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.14.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.14.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.15.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.15.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.15.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.15.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.15.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.15.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.15.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.15.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.15.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.15.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.15.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.15.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.16.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.16.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.16.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.16.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.16.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.16.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.16.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.16.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.16.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.16.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.16.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.16.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.17.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.17.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.17.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.17.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.17.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.17.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.17.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.17.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.17.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.17.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.17.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.17.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.18.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.18.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.18.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.18.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.18.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.18.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.18.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.18.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.18.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.18.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.18.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.18.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.19.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.19.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.19.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.19.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.19.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.19.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.19.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.19.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.19.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.19.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.19.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.19.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.2.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.2.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.2.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.2.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.2.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.2.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.2.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.2.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.2.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.2.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.2.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.2.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.20.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.20.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.20.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.20.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.20.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.20.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.20.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.20.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.20.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.20.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.20.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.20.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.21.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.21.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.21.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.21.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.21.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.21.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.21.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.21.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.21.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.21.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.21.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.21.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.22.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.22.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.22.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.22.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.22.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.22.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.22.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.22.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.22.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.22.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.22.mlp.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.22.mlp.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.23.attn.c_attn.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.23.attn.c_attn.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.23.attn.c_proj.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.23.attn.c_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.23.ln_1.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.23.ln_1.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.23.ln_2.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.23.ln_2.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.23.mlp.c_fc.bias": "model-00002-of-00003.safetensors",
+    "transformer.h.23.mlp.c_fc.weight": "model-00002-of-00003.safetensors",
+    "transformer.h.23.mlp.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.23.mlp.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.24.attn.c_attn.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.24.attn.c_attn.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.24.attn.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.24.attn.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.24.ln_1.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.24.ln_1.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.24.ln_2.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.24.ln_2.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.24.mlp.c_fc.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.24.mlp.c_fc.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.24.mlp.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.24.mlp.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.25.attn.c_attn.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.25.attn.c_attn.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.25.attn.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.25.attn.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.25.ln_1.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.25.ln_1.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.25.ln_2.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.25.ln_2.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.25.mlp.c_fc.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.25.mlp.c_fc.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.25.mlp.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.25.mlp.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.26.attn.c_attn.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.26.attn.c_attn.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.26.attn.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.26.attn.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.26.ln_1.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.26.ln_1.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.26.ln_2.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.26.ln_2.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.26.mlp.c_fc.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.26.mlp.c_fc.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.26.mlp.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.26.mlp.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.27.attn.c_attn.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.27.attn.c_attn.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.27.attn.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.27.attn.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.27.ln_1.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.27.ln_1.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.27.ln_2.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.27.ln_2.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.27.mlp.c_fc.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.27.mlp.c_fc.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.27.mlp.c_proj.bias": "model-00003-of-00003.safetensors",
+    "transformer.h.27.mlp.c_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.h.3.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.3.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.3.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.3.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.3.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.3.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.3.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.3.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.3.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.3.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.3.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.3.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.4.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.4.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.4.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.4.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.4.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.4.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.4.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.4.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.4.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.4.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.4.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.4.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.5.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.5.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.5.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.5.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.5.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.5.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.5.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.5.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.5.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.5.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.5.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.5.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.6.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.6.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.6.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.6.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.6.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.6.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.6.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.6.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.6.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.6.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.6.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.6.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.7.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.7.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.7.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.7.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.7.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.7.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.7.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.7.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.7.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.7.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.7.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.7.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.8.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.8.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.8.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.8.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.8.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.8.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.8.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.8.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.8.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.8.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.8.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.8.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.9.attn.c_attn.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.9.attn.c_attn.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.9.attn.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.9.attn.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.9.ln_1.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.9.ln_1.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.9.ln_2.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.9.ln_2.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.9.mlp.c_fc.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.9.mlp.c_fc.weight": "model-00001-of-00003.safetensors",
+    "transformer.h.9.mlp.c_proj.bias": "model-00001-of-00003.safetensors",
+    "transformer.h.9.mlp.c_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.ln_f.bias": "model-00003-of-00003.safetensors",
+    "transformer.ln_f.weight": "model-00003-of-00003.safetensors",
+    "transformer.wpe.weight": "model-00001-of-00003.safetensors",
+    "transformer.wte.weight": "model-00001-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,11 @@
+{
+  "additional_special_tokens": [
+    "<CLS>",
+    "<SEP>",
+    "<EOD>",
+    "<MASK>"
+  ],
+  "eos_token": "<EOD>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dbbd4ddbe43941051ed35fd44ff0d9d1c00ed345f7fd4d1969df174110f0609
+size 1044749

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "additional_special_tokens": [
+    "<CLS>",
+    "<SEP>",
+    "<EOD>",
+    "<MASK>"
+  ],
+  "eos_token": "<EOD>",
+  "extra_ids": 0,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "[UNK]"
+}