simplify tokenizer bug handling (#5)

- move bos+pad token def to tokenizer_config.json (1d51c7b17e912f8b136517acb52394f63de55f47)


Co-authored-by: Garðar Ingvarsson <gardarjuto@users.noreply.huggingface.co>

handler.py

@@ -33,19 +33,11 @@ class EndpointHandler:
         self.model = AutoModelForCausalLM.from_pretrained(
             path, device_map="auto", torch_dtype=torch.bfloat16
         )
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
         LOGGER.info(f"Inference model loaded from {path}")
         LOGGER.info(f"Model device: {self.model.device}")
 
-        # Fix the pad and bos tokens to avoid bug in the tokenizer
-        pad_token = "<unk>"
-        bos_token = "<|endoftext|>"
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            "AI-Sweden-Models/gpt-sw3-6.7b", pad_token=pad_token, bos_token=bos_token
-        )
-
-    def check_valid_inputs(
-        self, input_a: str, input_b: str, task: int
-    ) -> bool:
+    def check_valid_inputs(self, input_a: str, input_b: str, task: int) -> bool:
         """
         Check if the inputs are valid
         """

tokenizer_config.json

@@ -1,3 +1,5 @@
 {
-  "name_or_path": "AI-Sweden-Models/gpt-sw3-6.7b"
-}
+  "name_or_path": "AI-Sweden-Models/gpt-sw3-6.7b",
+  "bos_token": "<|endoftext|>",
+  "pad_token": "<unk>"
+}