fear: add log

app.py

@@ -7,6 +7,7 @@ import pandas as pd
 
 from utils import pipeline
 from utils.models import list_models
+from loguru import logger
 
 
 def read_data(filepath: str) -> Optional[pd.DataFrame]:
@@ -27,6 +28,7 @@ def process(
         file=None,
 ) -> (None, pd.DataFrame, str):
     try:
+        logger.info(f'Processing {task_name} with {model_name} and {pooling}')
         # load file
         if file:
             df = read_data(file.name)
@@ -51,6 +53,15 @@ def process(
         return None, df.iloc[:10], path
 
     except:
+        error = traceback.format_exc()
+        logger.warning({
+            'error': error,
+            'task_name': task_name,
+            'model_name': model_name,
+            'pooling': pooling,
+            'text': text,
+            'file': file,
+        })
         return {'Info': 'Something wrong', 'Error': traceback.format_exc()}, None, None
 
 

requirements.txt

@@ -8,3 +8,5 @@ sentence-transformers
 openpyxl
 tabulate
 gradio
+loguru
+

utils/models.py

@@ -1,6 +1,7 @@
 from functools import lru_cache
 
 import torch
+from loguru import logger
 from sentence_transformers import SentenceTransformer
 from transformers import AutoTokenizer, AutoModel
 
@@ -19,10 +20,8 @@ list_models = [
 
 class SBert:
     def __init__(self, path):
-        print(f'Loading model from {path} ...')
         self.model = SentenceTransformer(path, device=DEVICE)
-        # from pprint import pprint
-        # pprint(self.model.__dict__)
+        logger.info(f'Load {self.__class__} from {path} ...')
 
     @lru_cache(maxsize=10000)
     def __call__(self, x) -> torch.Tensor:
@@ -34,8 +33,9 @@ class ModelWithPooling:
     def __init__(self, path):
         self.tokenizer = AutoTokenizer.from_pretrained(path)
         self.model = AutoModel.from_pretrained(path)
+        logger.info(f'Load {self.__class__} from {path} ...')
 
-    @lru_cache(maxsize=10000)
+    @lru_cache(maxsize=100)
     @torch.no_grad()
     def __call__(self, text: str, pooling='mean'):
         inputs = self.tokenizer(text, padding=True, truncation=True, return_tensors="pt")