Update Sejarah.py

Sejarah.py

@@ -28,7 +28,9 @@ class Sejarah:
         indexing_pipeline.add_node(component=preprocessor, name="PreProcessor", inputs=["TextConverter"])
         indexing_pipeline.add_node(component=document_store, name="DocumentStore", inputs=["PreProcessor"])
 
-        files_to_index = ["documents/" + f for f in os.listdir("documents")]
+        dir = "/content/drive/Shareddrives/Natural Language Processing/Dataset/txt files"
+
+        files_to_index = [dir+"/" + f for f in os.listdir(dir)]
         indexing_pipeline.run_batch(file_paths=files_to_index)
 
         retriever = BM25Retriever(document_store=document_store)
@@ -50,26 +52,18 @@ class Sejarah:
     def language_converter(self, content, lang, method):
         
         content = content.lower()
-        
-        print(lang)
-        print(method)
-        print(content)
-        
+
         if lang == "en":
             if method == "question":
-                tokenized_text = self.id_en_tokenizer.prepare_seq2seq_batch([content], return_tensors='pt')
-                translation = self.id_en_model.generate(**tokenized_text)
-                content = self.id_en_tokenizer.batch_decode(translation, skip_special_tokens=True)[0]
-
-                print(tokenized_text)
-                print(translation)
-                print(content)
-                
-            else:
                 tokenized_text = self.en_id_tokenizer.prepare_seq2seq_batch([content], return_tensors='pt')
                 translation = self.en_id_model.generate(**tokenized_text)
                 content = self.en_id_tokenizer.batch_decode(translation, skip_special_tokens=True)[0]
 
+            else:
+                tokenized_text = self.id_en_tokenizer.prepare_seq2seq_batch([content], return_tensors='pt')
+                translation = self.id_en_model.generate(**tokenized_text)
+                content = self.id_en_tokenizer.batch_decode(translation, skip_special_tokens=True)[0]
+
         return content
     
     def interface(self, question):
@@ -85,12 +79,6 @@ class Sejarah:
             }
         )
 
-        print("_____Result____")
-        print(language)
-        print(question)
-        print(converted_question)
-        print(result['answers'][0])
-
         answer = self.language_converter(result['answers'][0].answer, language, "answer")
         context = self.language_converter(result['answers'][0].context, language, "answer")
         
@@ -99,5 +87,4 @@ class Sejarah:
     
     def detect_language(self, content):
         lang = langid.classify(content)
-        return lang[0]
-        
+        return lang[0]