Update cognitivess_model/tokenization_cognitivess.py

cognitivess_model/tokenization_cognitivess.py

@@ -1,4 +1,48 @@
-from transformers import LlamaTokenizer
+# cognitivess_model/tokenization_cognitivess.py
 
-class CognitivessTokenizer(LlamaTokenizer):
-    pass
+from transformers import PreTrainedTokenizer
+import json
+
+class CognitivessTokenizer(PreTrainedTokenizer):
+    def __init__(self, vocab_file, merges_file=None, **kwargs):
+        super().__init__(**kwargs)
+        self.vocab_file = vocab_file
+        self.merges_file = merges_file
+        self.load_vocab()
+
+    def load_vocab(self):
+        # Load vocabulary
+        with open(self.vocab_file, 'r') as f:
+            self.vocab = {line.strip(): idx for idx, line in enumerate(f)}
+
+        # Load merges file if exists
+        self.merges = []
+        if self.merges_file:
+            with open(self.merges_file, 'r') as f:
+                self.merges = [line.strip() for line in f]
+
+    def _tokenize(self, text):
+        # Tokenization logic (basic example)
+        tokens = text.split()  # Simple whitespace-based tokenization
+        return tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return [self.vocab.get(token, self.vocab.get('[UNK]')) for token in tokens]
+
+    def convert_ids_to_tokens(self, ids):
+        reverse_vocab = {idx: token for token, idx in self.vocab.items()}
+        return [reverse_vocab.get(idx, '[UNK]') for idx in ids]
+
+    def save_vocabulary(self, save_directory):
+        vocab_path = f"{save_directory}/vocab.txt"
+        with open(vocab_path, 'w') as f:
+            for token in self.vocab:
+                f.write(f"{token}\n")
+
+        if self.merges_file:
+            merges_path = f"{save_directory}/merges.txt"
+            with open(merges_path, 'w') as f:
+                for merge in self.merges:
+                    f.write(f"{merge}\n")
+            return vocab_path, merges_path
+        return vocab_path,