upload model

README.md

@@ -0,0 +1,62 @@
+---
+license: apache-2.0
+language:
+  - en
+  - ja
+datasets:
+  - izumi-lab/wikipedia-ja-20230720
+  - izumi-lab/wikipedia-en-20230720
+  - izumi-lab/open-text-books
+  - if001/aozorabunko-clean-sin
+  - if001/oscar_2023_filtered
+tags:
+  - ja
+  - japanese
+  - mixtral
+inference: false
+---
+
+275.86Mのmixtralを日本語データセットでpretrainingしたものです
+
+## sample
+
+```
+from transformers import AutoTokenizer, AutoModelForCausalLM
+model = AutoModelForCausalLM.from_pretrained("if001/tiny_mixtral_ja")
+tokenizer = AutoTokenizer.from_pretrained("if001/sentencepiece_ja", trust_remote_code=True)
+
+prompt = "それは九月初旬のある蒸し暑い晩のことであった。私は、Ｄ坂の"
+inputs = tokenizer(prompt, return_tensors="pt")
+ 
+generate_ids = model.generate(
+    inputs.input_ids, 
+    max_length=30,
+    top_k=30,
+    top_p=0.95,
+    temperature=0.6,
+    repetition_penalty=1.2,
+    do_sample=True,
+)
+tokenizer.decode(generate_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False)
+
+>> それは九月初旬のある蒸し暑い晩のことであった。私は、D坂の茶舗を後にして、その路地の角に横丁をあるいて居る、と云うと、丁度其処から、
+```
+
+## dataset
+英語と日本語のデータセットを使用
+
+```
+total tokens: 8.64B
+
+wikipedia_ja:    844.65M  
+wikipedia_en:    3.80B  
+open-text-books: 60.17M  
+oscar:           3.85B  
+aozorabunko:     92.97M  
+```
+
+## tokenizer
+```
+all_special_ids:  [1, 2, 3, 0, 4]
+all_special_tokens:  ['<BOS>', '<EOS>', '<UNK>', '<PAD>', '<MASK>']
+```

config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 2400,
+  "max_position_embeddings": 131072,
+  "model_type": "mixtral",
+  "num_attention_heads": 8,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 4,
+  "num_local_experts": 6,
+  "output_router_logits": true,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.0",
+  "use_cache": true,
+  "vocab_size": 35000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.37.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:797294a9ec3adcf8b2f3af8dd4896d9d4aeb7e2b5e9821e51908d725d5f0e4cf
+size 1103448792

sentencepiece_ja.py

@@ -0,0 +1,70 @@
+import os 
+from typing import Union, List, Optional, Tuple
+
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, AutoTokenizer
+from transformers.utils.hub import cached_file
+class SentencePieceJA(PreTrainedTokenizer):
+    def __init__(self, 
+                 model_path = "./tokenizer.json", 
+                 pad = "<PAD>",
+                 bos = "<BOS>",
+                 eos = "<EOS>",
+                 unk = "<UNK>",
+                 mask = "<MASK>",
+                 **kwargs):
+        from tokenizers import Tokenizer
+        try:             
+            self._tokenizer = Tokenizer.from_file(model_path)
+        except Exception as e:
+            print('exception: ', e)
+            print('load from cache...')
+            model_path = cached_file('if001/sentencepiece_ja', 'tokenizer.json')        
+            self._tokenizer = Tokenizer.from_file(model_path)
+        super().__init__(**kwargs)
+        self.add_special_tokens({
+            'pad_token': pad,
+            'bos_token': bos,
+            'eos_token': eos,
+            'unk_token': unk,
+            'mask_token': mask
+        })
+        self._tokenizer.add_tokens([" ", "　"])
+    
+    def get_vocab(self) -> int:
+        return self._tokenizer.get_vocab()
+    
+    @property
+    def vocab_size(self) -> int:
+        return self._tokenizer.get_vocab_size()
+
+    def _tokenize(self, text, **kwargs):
+        return self._tokenizer.encode(text).tokens
+
+    def _convert_token_to_id(self, token):
+        ids = self._tokenizer.encode(token).ids
+        if len(ids) == 0:
+            return self.unk_token_id
+        return self._tokenizer.encode(token).ids[0]
+        
+    def _convert_id_to_token(self, index: int) -> str:
+        return self._tokenizer.decode([index])
+    
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:        
+        ## 日本語用
+        return "".join(tokens)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        index = 0
+        if os.path.isdir(save_directory):
+            vocab_file = os.path.join(
+                save_directory, (filename_prefix + "-" if filename_prefix else "") + 'vocab.txt'
+            )
+        else:
+            vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory        
+        with open(vocab_file, "w", encoding="utf-8") as writer:
+            for token, token_index in sorted(self.get_vocab().items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    index = token_index
+                writer.write(token + "\n")
+                index += 1
+        return (vocab_file,)

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "bos_token": "<BOS>",
+  "eos_token": "<EOS>",
+  "mask_token": "<MASK>",
+  "pad_token": "<PAD>",
+  "unk_token": "<UNK>"
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token": "<BOS>",  
+  "eos_token": "<EOS>",
+  "mask_token": "<MASK>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<PAD>",  
+  "unk_token": "<UNK>",
+  "clean_up_tokenization_spaces": true,  
+  "tokenizer_class": "SentencePieceJA",
+  "auto_map": {
+    "AutoTokenizer": ["","sentencepiece_ja.SentencePieceJA"]
+  },
+  "transformers_version": " 4.34.1"
+}