index

.gitattributes

@@ -34,3 +34,22 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 index/en-es_metadata_ref.pkl filter=lfs diff=lfs merge=lfs -text
+index/en-es_output_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-zh_metadata_ref.pkl filter=lfs diff=lfs merge=lfs -text
+index/en-zh_output_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-ar_input_tokens.index filter=lfs diff=lfs merge=lfs -text
+index/en-ar_output_tokens.index filter=lfs diff=lfs merge=lfs -text
+index/en-fr_metadata_ref.pkl filter=lfs diff=lfs merge=lfs -text
+index/en-fr_input_tokens.index filter=lfs diff=lfs merge=lfs -text
+index/en-fr_input_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-fr_output_tokens.index filter=lfs diff=lfs merge=lfs -text
+index/en-zh_input_tokens.index filter=lfs diff=lfs merge=lfs -text
+index/en-ar_input_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-ar_metadata_ref.pkl filter=lfs diff=lfs merge=lfs -text
+index/en-ar_output_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-fr_output_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-zh_input_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-zh_output_tokens.index filter=lfs diff=lfs merge=lfs -text
+index/en-es_input_tokens.index filter=lfs diff=lfs merge=lfs -text
+index/en-es_input_words.index filter=lfs diff=lfs merge=lfs -text
+index/en-es_output_tokens.index filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+local_index/*
+app_local.py

app.py

@@ -23,23 +23,24 @@ from transformers import AutoTokenizer, MarianTokenizer, AutoModel, AutoModelFor
 model_es = "Helsinki-NLP/opus-mt-en-es"
 model_fr = "Helsinki-NLP/opus-mt-en-fr"
 model_zh = "Helsinki-NLP/opus-mt-en-zh"
-model_sw = "Helsinki-NLP/opus-mt-en-sw"
+model_ar = "Helsinki-NLP/opus-mt-en-ar"
 
 tokenizer_es = AutoTokenizer.from_pretrained(model_es)
 tokenizer_fr = AutoTokenizer.from_pretrained(model_fr)
 tokenizer_zh = AutoTokenizer.from_pretrained(model_zh)
-tokenizer_sw = AutoTokenizer.from_pretrained(model_sw)
+tokenizer_ar = AutoTokenizer.from_pretrained(model_ar)
 
 model_tr_es = MarianMTModel.from_pretrained(model_es)
 model_tr_fr = MarianMTModel.from_pretrained(model_fr)
 model_tr_zh = MarianMTModel.from_pretrained(model_zh)
-model_tr_sw = MarianMTModel.from_pretrained(model_sw)
+model_tr_ar = MarianMTModel.from_pretrained(model_ar)
 
 from faiss import write_index, read_index
 import pickle 
 
 def load_index(model): 
-	with open('index/'+ model + '_metadata_ref.pkl', 'rb') as f:
+	# with open('index/'+ model + '_metadata_ref.pkl', 'rb') as f:
+	with open('local_index/'+ model + '_metadata_ref.pkl', 'rb') as f:
 		loaded_dict = pickle.load(f)
 	for type in ['tokens','words']:
 		for kind in ['input', 'output']: 
@@ -54,25 +55,28 @@ dict_models = {
 	'en-es': model_es,
 	'en-fr': model_fr,
 	'en-zh': model_zh,
-	'en-sw': model_sw,
+	'en-ar': model_ar,
 }
 
 dict_models_tr = {
 	'en-es': model_tr_es,
 	'en-fr': model_tr_fr,
 	'en-zh': model_tr_zh,
-	'en-sw': model_tr_sw,
+	'en-ar': model_tr_ar,
 }
 
 dict_tokenizer_tr = {
 	'en-es': tokenizer_es,
 	'en-fr': tokenizer_fr,
 	'en-zh': tokenizer_zh,
-	'en-sw': tokenizer_sw,
+	'en-ar': tokenizer_ar,
 }
-
+# dict_reference_faiss = {'en-es':[]}
 dict_reference_faiss = {
 	'en-es': load_index('en-es'),
+	'en-ar': load_index('en-ar'),
+	'en-fr': load_index('en-fr'),
+	'en-zh': load_index('en-zh'),
 }
 
 # print("dict", dict_reference_faiss['en-es']['input']['tokens'][1])
@@ -698,11 +702,11 @@ html_embd = """
  """
 
 html_tok_target ="""
- <div id="d3_tok_target">... tokenization visualization ...</div> 
+ <div id="d3_tok_target"> </div> 
  """
 
 html_embd_target= """
- <div id="d3_embd_target">... token embeddings visualization ...</div> 
+ <div id="d3_embd_target"> </div> 
  <div id="d3_graph_output_words" class="d3_graph words"></div>
 	<div id="d3_graph_output_tokens" class="d3_graph tokens"></div>
   <div id="similar_output_words" class=""></div>
@@ -823,7 +827,7 @@ with gr.Blocks(js="plotsjs.js") as demo:
 		"""
 		1. Select the language pair for the translation 
 		""")
-	radio_c = gr.Radio(choices=['en-zh', 'en-es', 'en-fr', 'en-sw'], value="en-es", label= '', container=False)
+	radio_c = gr.Radio(choices=['en-zh', 'en-es', 'en-fr', 'en-ar'], value="en-es", label= '', container=False)
 	gr.Markdown(
 		"""
 		2. Source text to translate

index/en-ar_input_tokens.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:705af4d667addbc721da76b3099be10fcc437eeea0d30445751e1d7edbd2af19
+size 754691

index/en-ar_input_words.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cac0ee3490002a39df24b27a6a7f570a60333d7f431c2f7aae2c4b1694d64948
+size 756747

index/en-ar_metadata_ref.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37e6a98d587055a03daeceac1959cedd5ad793cf0fec6f1b377b1dcb5caa2c5f
+size 28805589

index/en-ar_output_tokens.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd71fb94d70342945b2abdb658d6b04401c8cb2f98238d40919baecbd8f61781
+size 1019915

index/en-ar_output_words.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:798941013c236a3be852512ddc8ff280969fee61f47336244e03da94d2eb8d52
+size 892443

index/en-es_metadata_ref.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:545ee0536a47df856bb92082ef49ff5f3ac15d80c04b03c03fe8ebd8a089a356
-size 2042890
+oid sha256:0f766d5a212ded9981ff504628d9ecadb567872e5244297582a3bc4ad0e3b774
+size 25129757

index/en-fr_input_tokens.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e253777db5a5cc1eb76da856178d841f1a2814e11c7e7d4ac24be817e4983638
+size 764971

index/en-fr_input_words.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:961b08022a7cea12c1dfbb8db0a2812e833980b0fcbd8cdf0455ee70f5ae778d
+size 756747

index/en-fr_metadata_ref.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab1301d87b26b6af5d2cd54ad891ae53b817bfda08a0315f60964ae0bd57e6c5
+size 27041562

index/en-fr_output_tokens.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b205bacecb6e4c8791e7c9820de772501ed974a19d365608646a69fe5cde3ff
+size 910947

index/en-fr_output_words.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c90bf7ac638494a23efed842d3128f113f6ed279413b3362747892d7e9913e77
+size 873939

index/en-zh_input_tokens.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098a6ed7021dd99b7dbeaf6174373081c73fe9e16375b9de79e56275997c035d
+size 754691

index/en-zh_input_words.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:881ef345b1d0c689be2c98539beeacc0516ff35c571ac2f2eb75a8bb67fb5330
+size 756747

index/en-zh_metadata_ref.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:223617175b67c586d07e10577e3e7168875eec66f7a660d1a3ada40bc05f864c
+size 22426143

index/en-zh_output_tokens.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05a9caebc080364fec8685a5dd4f90bdc6f89a6dd2e39d8e649a0f3969e02639
+size 880107

index/en-zh_output_words.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1293142f7b1639bd7ec7c618d12ca100ddb6ef91d5e20920fa62dacc8f5ee20d
+size 242747