Updated training

trainBpeTokenizer.py

@@ -3,6 +3,15 @@ from tokenizers.models import BPE
 from tokenizers.trainers import BpeTrainer
 from tokenizers.pre_tokenizers import ByteLevel
 from tokenizers.processors import TemplateProcessing
+import argparse
+
+
+parser = argparse.ArgumentParser(description='Train BPE Tokenizer.')
+parser.add_argument('dataset_file_path', type=str, help='Path to the dataset file')
+parser.add_argument('output_file_path', type=str, help='Path to file containing trained tokenizer weights')
+
+# Parse the arguments
+args = parser.parse_args()
 
 # Initialize a tokenizer
 tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
@@ -14,7 +23,7 @@ tokenizer.pre_tokenizer = ByteLevel()
 trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
 
 # Path to the file(s) for training the tokenizer
-files = ["/home/saicharan/Downloads/chembl.csv"]
+files = [args.dataset_file_path]
 
 # Train the tokenizer
 tokenizer.train(files, trainer)
@@ -30,4 +39,4 @@ tokenizer.post_processor = TemplateProcessing(
 )
 
 # Save the tokenizer
-tokenizer.save("/home/saicharan/Downloads/chembl_bpe_tokenizer.json")
+tokenizer.save(args.output_file_path)