Update generic_ner.py

generic_ner.py

@@ -1,16 +1,14 @@
 from transformers import Pipeline
 import numpy as np
 import torch
+import nltk
+nltk.download('averaged_perceptron_tagger')
 from nltk.chunk import conlltags2tree
 from nltk import pos_tag
 from nltk.tree import Tree
 import string
 import torch.nn.functional as F
-from langdetect import detect
-
-
 import re, string
-import pysbd
 
 
 def tokenize(text):
@@ -202,27 +200,8 @@ class MultitaskTokenClassificationPipeline(Pipeline):
         }
         return preprocess_kwargs, {}, {}
 
-    # def preprocess(self, text, **kwargs):
-    #
-    #     language = detect(text)
-    #     sentences = segment_and_trim_sentences(text, language, 512)
-    #
-    #     tokenized_inputs = self.tokenizer(
-    #         text,
-    #         padding="max_length",
-    #         truncation=True,
-    #         max_length=512,
-    #         return_tensors="pt",
-    #     )
-    #
-    #     text_sentences = [
-    #         tokenize(add_spaces_around_punctuation(sentence)) for sentence in sentences
-    #     ]
-    #     return tokenized_inputs, text_sentences, text
     def preprocess(self, text, **kwargs):
 
-        # sentences = segment_and_trim_sentences(text, language, 512)
-
         tokenized_inputs = self.tokenizer(
             text, padding="max_length", truncation=True, max_length=512
         )