Delete tokenizer

tokenizer/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:da969d56fc3a8fcb72ae57f1484be0a65fc365b6374d98c2460c16a53dceda0a
-size 4400216504

tokenizer/save.py

@@ -1,20 +0,0 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
-from safetensors.torch import save_file
-
-# Load your model and tokenizer
-model_name = "Stefan171/TinyLlama-QuantumQuill-chat-12-05-24"
-model = AutoModelForCausalLM.from_pretrained(model_name)
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-# Define the path to save the safetensors model in the current directory
-safetensors_path = "./model.safetensors"
-
-# Convert model parameters to a dictionary
-state_dict = model.state_dict()
-
-# Save the model parameters in safetensors format
-save_file(state_dict, safetensors_path)
-
-# Save the tokenizer in the current directory
-tokenizer.save_pretrained("./")

tokenizer/special_tokens_map.json

@@ -1,30 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

tokenizer/tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723

tokenizer/tokenizer_config.json

@@ -1,50 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [],
-  "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "cls_token": null,
-  "eos_token": "</s>",
-  "mask_token": null,
-  "model_input_names": [
-    "input_ids",
-    "attention_mask"
-  ],
-  "model_max_length": 16000,
-  "pad_token": "<unk>",
-  "padding_side": "left",
-  "sep_token": null,
-  "split_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "truncation_side": "right",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}

tokenizer/upload.py

@@ -1,26 +0,0 @@
-from huggingface_hub import HfApi
-
-api = HfApi()
-model_id = "Stefan171/TinyLlama-QuantumQuill-chat-12-05-24"
-
-# Path to the safetensors model file
-safetensors_path = "./model.safetensors"
-
-# Upload the safetensors model
-api.upload_file(
-    path_or_fileobj=safetensors_path,
-    path_in_repo="model.safetensors",
-    repo_id=model_id,
-    repo_type="model"
-)
-
-# Path to the tokenizer folder (current directory)
-tokenizer_path = "./"
-
-# Upload the tokenizer files
-api.upload_folder(
-    folder_path=tokenizer_path,
-    path_in_repo="tokenizer",
-    repo_id=model_id,
-    repo_type="model"
-)