commit all

app.py

@@ -5,17 +5,15 @@ import pickle
 import torch
 import torch.nn as nn
 from torchtext.transforms import PadTransform
-from torch.utils.data import Dataset, DataLoader
 from torch.nn import functional as F
 from tqdm import tqdm
-from underthesea import word_tokenize, text_normalize
+from underthesea import text_normalize
 
 # Build Vocabulary
-# device = 'cuda' if torch.cuda.is_available() else 'cpu'
 device = "cpu"
 
 # Build Vocabulary
-MAX_LENGTH = 15
+MAX_LENGTH = 20
 class Vocabulary:
     """The Vocabulary class is used to record words, which are used to convert
     text to numbers and vice versa.
@@ -75,14 +73,22 @@ class Vocabulary:
     def preprocessing_sent(self, sent, lang="en"):
         """Preprocessing a sentence (depend on language english or vietnamese)
         @param sent (str)
-        @param lang (str) 
+        @param lang (str)
         """
 
         # Lowercase sentence and remove space at beginning and ending
         sent = sent.lower().strip()
 
+        # Replace HTML charecterist
+        sent = re.sub("'", "'", sent)
+        sent = re.sub(""", '"', sent)
+        sent = re.sub("[", "[", sent)
+        sent = re.sub("]", "]", sent)
+
         # Remove unnecessary space
         sent = re.sub("(?<=\w)\.", " .", sent)
+
+        # Normalizing the distance between tokens (word and punctuation)
         sent = re.sub("(?<=\w),", " ,", sent)
         sent = re.sub("(?<=\w)\?", " ?", sent)
         sent = re.sub("(?<=\w)\!", " !", sent)
@@ -93,10 +99,12 @@ class Vocabulary:
             sent = re.sub("what's", "what is", sent)
             sent = re.sub("who's", "who is", sent)
             sent = re.sub("which's", "which is", sent)
+            sent = re.sub("who's", "who is", sent)
+            sent = re.sub("here's", "here is", sent)
+            sent = re.sub("there's", "there is", sent)
+            sent = re.sub("it's", "it is", sent)
 
             sent = re.sub("i'm", "i am", sent)
-            # Dont know to preprocess with possessive case
-            sent = re.sub("it's", "it is", sent)
             sent = re.sub("'re ", " are ", sent)
             sent = re.sub("'ve ", " have ", sent)
             sent = re.sub("'ll ", " will ", sent)
@@ -115,7 +123,8 @@ class Vocabulary:
         else:
             # Package underthesea.text_normalize support to normalize vietnamese
             sent = text_normalize(sent)
-
+        if not sent.endswith(('.', '!', '?')):
+            sent = sent + ' .'
         return sent.strip()
 
     def tokenize_corpus(self, corpus, disable=False):
@@ -165,40 +174,51 @@ class Vocabulary:
         return corpus
 
 
-with open("vocab_source.pkl", "rb") as file:
+with open("vocab_source_final.pkl", "rb") as file:
     VOCAB_SOURCE = pickle.load(file)
-with open("vocab_target.pkl", "rb") as file:
+with open("vocab_target_final.pkl", "rb") as file:
     VOCAB_TARGET = pickle.load(file)
 
 input_embedding = torch.zeros((len(VOCAB_SOURCE), 100))
 output_embedding = torch.zeros((len(VOCAB_TARGET), 100))
 
 
-def create_input_emb_layer():
-    num_embeddings, embedding_dim = input_embedding.size()
+def create_input_emb_layer(pretrained = False):
+    if not pretrained:
+        weights_matrix = torch.zeros((len(VOCAB_SOURCE), 100))
+    else:
+        weights_matrix = input_embedding
+    num_embeddings, embedding_dim = weights_matrix.size()
     emb_layer = nn.Embedding(num_embeddings, embedding_dim)
+    emb_layer.weight.data = weights_matrix
     emb_layer.weight.requires_grad = False
+
     return emb_layer, embedding_dim
 
-def create_output_emb_layer():
-    num_embeddings, embedding_dim = output_embedding.size()
+def create_output_emb_layer(pretrained = False):
+    if not pretrained:
+        weights_matrix = torch.zeros((len(VOCAB_TARGET), 100))
+    else:
+        weights_matrix = output_embedding
+    num_embeddings, embedding_dim = weights_matrix.size()
     emb_layer = nn.Embedding(num_embeddings, embedding_dim)
+    emb_layer.weight.data = weights_matrix
     emb_layer.weight.requires_grad = False
+
     return emb_layer, embedding_dim
 
 
-class EncoderRNN(nn.Module):
+class EncoderAtt(nn.Module):
     def __init__(self, input_dim, hidden_dim, dropout = 0.1):
         """ Encoder RNN
         @param input_dim (int): size of vocab_souce
         @param hidden_dim (int)
         @param dropout (float): dropout ratio of layer drop out
         """
-        super(EncoderRNN, self).__init__()
+        super(EncoderAtt, self).__init__()
         self.hidden_dim = hidden_dim
-        #self.embedding = nn.Embedding(input_dim, hidden_dim)
-        # Đổi thành input embedding
-        self.embedding, self.embedding_dim = create_input_emb_layer()
+        # Using pretrained Embedding
+        self.embedding, self.embedding_dim = create_input_emb_layer(True)
         self.gru = nn.GRU(self.embedding_dim, hidden_dim, batch_first=True)
         self.dropout = nn.Dropout(dropout)
 
@@ -207,7 +227,6 @@ class EncoderRNN(nn.Module):
         output, hidden = self.gru(embedded)
         return output, hidden
 
-
 class BahdanauAttention(nn.Module):
     def __init__(self, hidden_size):
         """ Bahdanau Attention
@@ -227,20 +246,21 @@ class BahdanauAttention(nn.Module):
 
         return context, weights
 
-class AttnDecoderRNN(nn.Module):
-    def __init__(self, hidden_size, output_size, dropout_p=0.1):
+class DecoderAtt(nn.Module):
+    def __init__(self, hidden_size, output_size, dropout=0.1):
         """ Decoder RNN using Attention
         @param hidden_size (int)
         @param output_size (int): size of vocab_target
         @param dropout (float): dropout ratio of layer drop out
         """
-        super(AttnDecoderRNN, self).__init__()
-        self.embedding, self.embedding_dim = create_output_emb_layer()
+        super(DecoderAtt, self).__init__()
+        # Using pretrained Embedding
+        self.embedding, self.embedding_dim = create_output_emb_layer(True)
         self.fc = nn.Linear(self.embedding_dim, hidden_size)
         self.attention = BahdanauAttention(hidden_size)
         self.gru = nn.GRU(2 * hidden_size, hidden_size, batch_first=True)
         self.out = nn.Linear(hidden_size, output_size)
-        self.dropout = nn.Dropout(dropout_p)
+        self.dropout = nn.Dropout(dropout)
 
     def forward(self, encoder_outputs, encoder_hidden, target_tensor=None):
         batch_size = encoder_outputs.size(0)
@@ -293,13 +313,13 @@ OUTPUT_DIM = len(VOCAB_TARGET)
 HID_DIM = 512
 
 # Load our Model Translation
-ENCODER = EncoderRNN(INPUT_DIM, HID_DIM)
-ENCODER.load_state_dict(torch.load("hid512_encoder_att_epoch_20.pt"))
-DECODER = AttnDecoderRNN(HID_DIM, OUTPUT_DIM)
-DECODER.load_state_dict(torch.load("hid512_decoder_att_epoch_20.pt"))
+ENCODER = EncoderAtt(INPUT_DIM, HID_DIM)
+#ENCODER.load_state_dict(torch.load("hid512_encoder_att_epoch_20.pt"), map_location=torch.device('cpu'))
+DECODER = DecoderAtt(HID_DIM, OUTPUT_DIM)
+#DECODER.load_state_dict(torch.load("hid512_decoder_att_epoch_20.pt"), map_location=torch.device('cpu'))
 
 
-def evaluate(encoder, decoder, sentence, vocab_source, vocab_target, disable=False):
+def evaluate_final_model(encoder, decoder, sentence, vocab_source, vocab_target, disable=False):
     encoder.eval()
     decoder.eval()
     with torch.no_grad():
@@ -326,12 +346,14 @@ def evaluate(encoder, decoder, sentence, vocab_source, vocab_target, disable=Fal
     return decoded_words, decoder_attn
 
 
-def my_translate_model(sentence):
-    output_words, _ = evaluate(
-        ENCODER, DECODER, sentence, VOCAB_SOURCE, VOCAB_TARGET, disable=True
-    )
+def my_translation(sentence):
+    output_words, _ = evaluate_final_model(sentence, ENCODER, DECODER, VOCAB_SOURCE, VOCAB_TARGET, disable= True)
+    output_words = output_words.remove("<pad>")
+    output_words = output_words.remove("<unk>")
+    output_words = output_words.remove("<sos>")
+    output_words = output_words.remove("<eos>")
 
-    return " ".join(output_words[1:-1]).capitalize()
+    return ' '.join(output_words[1:-1]).capitalize()
 
 
 def envit5_translation(text):
@@ -339,44 +361,44 @@ def envit5_translation(text):
         text,
         max_length=512,
         early_stopping=True,
-    )[0][
-        "translation_text"
-    ][3:]
+    )[0]["translation_text"][3:]
     return res
 
 
 def translation(text):
-    output1 = my_translate_model(text)
+    if not text.endswith(('.', '!', '?')):
+        text = text + '.'
+    #output1 = my_translation(text)
+    output1 = "Something"
     output2 = envit5_translation(text)
-    #output3 = finetune_BERT(text)
 
     return (output1, output2)
 
+if __name__ == "__main__":
+    examples = [["Hello guys", "Input"], 
+                ["Xin chào các bạn", "Output"]]
+
+    demo = gr.Interface(
+        theme = gr.themes.Base(),
+        fn=translation,
+        title="Co Gai Mo Duong",
+        description="""
+        ## Machine Translation: English to Vietnamese
+        """,
+        examples=examples,
+        inputs=[
+            gr.Textbox(
+                lines=5, placeholder="Enter text", label="Input"
+            )
+        ],
+        outputs=[
+            gr.Textbox(
+                "text", label="Our Machine Translation"
+            ),
+            gr.Textbox(
+                "text", label="VietAI Machine Translation"
+            )
+        ]
+    )
 
-examples = [["Input: Hello guys"], 
-            ["Output: Xin chào các bạn"]]
-
-demo = gr.Interface(
-    theme = gr.themes.Base(),
-    fn=translation,
-    title="Co Gai Mo Duong",
-    description="""
-    ## Machine Translation: English to Vietnamese
-    """,
-    examples=examples,
-    inputs=[
-        gr.Textbox(
-            lines=5, placeholder="Enter text", label="Input"
-        )
-    ],
-    outputs=[
-        gr.Textbox(
-            "text", label="Our Machine Translation"
-        ),
-        gr.Textbox(
-            "text", label="VietAI Machine Translation"
-        )
-    ]
-)
-
-demo.launch(share = True)
+    demo.launch()

vocab_source.pkl → vocab_source_final.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf38f3daacf3feb3b80cba2069210d5ac3b770c232233178f42434b709bba360
-size 659103
+oid sha256:470ea7a4a120e9c2274db2ad7f5b68241eb1cea444881852245013ef91f69106
+size 682848

vocab_target.pkl → vocab_target_final.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac7bd376478b2b3bbcfbeeccd5ced630340b95d3da5eab8d7c1c9e01d74b50d2
-size 228271
+oid sha256:853cf7ecce86d078c1a8cf81b867f55454d1b7bf21679832fea8391711198c6f
+size 250477