tokenizer, config 추가

README.md

@@ -1,3 +1,4 @@
 ---
 license: mit
 ---
+학습중

config.json

@@ -0,0 +1,26 @@
+{
+    "architectures": [
+      "GPTNeoXForCausalLM"
+    ],
+    "attention_probs_dropout_prob": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "pad_token_id": 0,
+    "unk_token_id": 5,
+    "hidden_act": "gelu_fast",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-05,
+    "max_position_embeddings": 512,
+    "model_type": "gpt_neox",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 6,
+    "rotary_emb_base": 10000,
+    "rotary_pct": 0.25,
+    "tie_word_embeddings": false,
+    "transformers_version": "4.24.0",
+    "use_cache": true,
+    "vocab_size": 8000
+  }

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "name_or_path": "tokenizer-8k",
+  "pad_token": null,
+  "special_tokens_map_file": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}