update

app.py

@@ -18,6 +18,7 @@
 - baichuan的单字数量怎么两万多个？
 - OOV
 - feedback位置
+- gpt4, gpt3.5 的overlap tokens 有问题。
 
 
 plots

utils/log_util.py

@@ -3,7 +3,9 @@ import logging
 
 logging.basicConfig(
     format='%(asctime)s - %(filename)s - %(levelname)s - %(process)d - %(thread)d - %(message)s',
+    level=logging.INFO,
     datefmt="%Y-%m-%d %H:%M:%S",
+
 )
 
 logger = logging.getLogger(__name__)

vocab/gpt_35_turbo/__init__.py

@@ -2,6 +2,7 @@
 
 import tiktoken
 from tiktoken import Encoding
+from utils.log_util import logger
 
 tokenizer = tiktoken.encoding_for_model('gpt-3.5-turbo')
 tokenizer.vocab_size = tokenizer.n_vocab
@@ -22,16 +23,21 @@ def convert_ids_to_tokens(self, tokens):
 def get_vocab(self):
     """Returns vocab as a dict"""
     vocab = {}
+    key_error_list = []
+    unicode_decode_error_list = []
     for i in range(self.vocab_size):
         try:
             token_byte = self.convert_ids_to_tokens([i])[0]
             token_str = token_byte.decode("utf-8")
             vocab[token_str] = i
-        except KeyError:
-            print("gpt_35_turbo decode KeyError", i)
-        except UnicodeDecodeError:
-            print("gpt_35_turbo decode UnicodeDecodeError", i, str(token_byte))
+        except KeyError:  # 100256 100261-100275
+            key_error_list.append(i)
+        except UnicodeDecodeError:  # 特别多
+            unicode_decode_error_list.append((i, str(token_byte)))
+
     # vocab.update(self.added_tokens_encoder)
+    logger.info(f"gpt_35_turbo {len(key_error_list)} KeyError: {key_error_list}")
+    logger.info(f"gpt_35_turbo {len(unicode_decode_error_list)} UnicodeDecodeError: {unicode_decode_error_list[:5]}")
     return vocab
 
 

vocab/gpt_4/__init__.py

@@ -22,16 +22,21 @@ def convert_ids_to_tokens(self, tokens):
 def get_vocab(self):
     """Returns vocab as a dict"""
     vocab = {}
+    key_error_list = []
+    unicode_decode_error_list = []
     for i in range(self.vocab_size):
         try:
             token_byte = self.convert_ids_to_tokens([i])[0]
             token_str = token_byte.decode("utf-8")
             vocab[token_str] = i
-        except KeyError:
-            print("gpt_35_turbo decode KeyError", i)
-        except UnicodeDecodeError:
-            print("gpt_35_turbo decode UnicodeDecodeError", i, str(token_byte))
+        except KeyError:  # 100256 100261-100275
+            key_error_list.append(i)
+        except UnicodeDecodeError:  # 特别多
+            unicode_decode_error_list.append((i, str(token_byte)))
+
     # vocab.update(self.added_tokens_encoder)
+    logger.info(f"gpt-4 {len(key_error_list)} KeyError: {key_error_list}")
+    logger.info(f"gpt-4 {len(unicode_decode_error_list)} UnicodeDecodeError: {unicode_decode_error_list[:5]}")
     return vocab