Update deploy_pipeline.py

deploy_pipeline.py

@@ -13,15 +13,14 @@ class TokenizeAndAlignLabelsStep():
 
     # Adapted From : https://huggingface.co/docs/transformers/tasks/token_classification
     def tokenize_and_align_labels(self, examples, tokenizer):
-        
-        tokenized_inputs = tokenizer(examples, padding='max_length', truncation=True, max_length=128)
+                
+        tokenized_inputs = tokenizer(examples, padding='max_length', truncation=True, max_length=128, is_split_into_words=True)
                 
         # Map tokens to their respective word.
         word_ids = tokenized_inputs.word_ids()
 
         previous_word_idx = None
-        
-    
+                
         labels_mask = []        
         
         for word_idx in word_ids:  # Set the special tokens to -100.
@@ -35,28 +34,25 @@ class TokenizeAndAlignLabelsStep():
             
             previous_word_idx = word_idx
                         
-        tokenized_inputs["tokens"] = tokenizer.decode(tokenized_inputs["input_ids"], skip_special_tokens=True)
         tokenized_inputs["labels_mask"] = labels_mask
 
         return tokenized_inputs
 
 
 
-
 class BERT_CRF_Pipeline(Pipeline):
 
     def _sanitize_parameters(self, **kwargs):
         return {}, {}, {}
 
-    def preprocess(self, text):
+    def preprocess(self, inputs):
+        tokens = inputs['tokens']
 
         tokenizer = AutoTokenizer.from_pretrained(
             "neuralmind/bert-base-portuguese-cased", do_lower_case=False)            
+        
+        return TokenizeAndAlignLabelsStep().tokenize_and_align_labels(examples=tokens, tokenizer=tokenizer)
 
-        TokenizeAndAlignLabelsStep().tokenize_and_align_labels(
-            examples=text, tokenizer=tokenizer)
-
-        return TokenizeAndAlignLabelsStep().tokenize_and_align_labels(examples=text, tokenizer=tokenizer)
 
     def _forward(self, tokenizer_results):
 
@@ -79,13 +75,15 @@ class BERT_CRF_Pipeline(Pipeline):
         return outputs
 
     def postprocess(self, model_outputs):
+        
         # From Ner_tags to Ner_labels
         for i, label in enumerate(model_outputs[0]):
             model_outputs[0][i] = self.model.config.id2label[label]
-
+                
         return model_outputs[0]
 
 
+
 def main():    
 
     PIPELINE_REGISTRY.register_pipeline("PT-BERT-Large-CRF-Conll2003-pipeline",