Upload 5 files

.gitattributes

@@ -2,34 +2,26 @@
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,13 @@
 ---
-title: MTDemo
-emoji: 🏃
-colorFrom: yellow
-colorTo: purple
+title: Nllb Translation Demo
+emoji: 👀
+colorFrom: indigo
+colorTo: green
 sdk: gradio
-sdk_version: 4.22.0
+sdk_version: 3.0.26
 app_file: app.py
 pinned: false
+duplicated_from: Geonmo/nllb-translation-demo
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -8,13 +8,30 @@ from flores200_codes import flores_codes
 
 def load_models():
     # build model and tokenizer
-    model_name_dict = {
-                'nllb-1.3B': 'facebook/nllb-200-1.3B',
-        #'nllb-distilled-600M': 'facebook/nllb-200-distilled-600M',
+    model_name_dict = {#'nllb-distilled-600M': 'facebook/nllb-200-distilled-600M',
+                  'nllb-1.3B': 'facebook/nllb-200-1.3B',
+                  #'nllb-distilled-1.3B': 'facebook/nllb-200-distilled-1.3B',
+                  #'nllb-3.3B': 'facebook/nllb-200-3.3B',
+                  }
+
+    model_dict = {}
+
+    for call_name, real_name in model_name_dict.items():
+        print('\tLoading model: %s' % call_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(real_name)
+        tokenizer = AutoTokenizer.from_pretrained(real_name)
+        model_dict[call_name+'_model'] = model
+        model_dict[call_name+'_tokenizer'] = tokenizer
+
+    return model_dict
+
+
+def load_models():
+    # build model and tokenizer
+    model_name_dict = {'nllb-distilled-600M': 'facebook/nllb-200-distilled-600M',
                   #'nllb-1.3B': 'facebook/nllb-200-1.3B',
                   #'nllb-distilled-1.3B': 'facebook/nllb-200-distilled-1.3B',
                   #'nllb-3.3B': 'facebook/nllb-200-3.3B',
-                  # 'nllb-distilled-600M': 'facebook/nllb-200-distilled-600M', 
                   }
 
     model_dict = {}
@@ -31,7 +48,7 @@ def load_models():
 
 def translation(source, target, text):
     if len(model_dict) == 2:
-        model_name = 'nllb-3.3B'
+        model_name = 'nllb-distilled-600M'
 
     start_time = time.time()
     source = flores_codes[source]
@@ -53,6 +70,7 @@ def translation(source, target, text):
     return result
 
 
+
 if __name__ == '__main__':
     print('\tinit models')
 
@@ -64,18 +82,18 @@ if __name__ == '__main__':
     lang_codes = list(flores_codes.keys())
     #inputs = [gr.inputs.Radio(['nllb-distilled-600M', 'nllb-1.3B', 'nllb-distilled-1.3B'], label='NLLB Model'),
     inputs = [gr.inputs.Dropdown(lang_codes, default='English', label='Source'),
-              gr.inputs.Dropdown(lang_codes, default='Hindi', label='Target'),
+              gr.inputs.Dropdown(lang_codes, default='Korean', label='Target'),
               gr.inputs.Textbox(lines=5, label="Input text"),
               ]
 
     outputs = gr.outputs.JSON()
 
-    title = "Machine Translation Demo"
+    title = "NLLB distilled 1.3B demo"
 
     demo_status = "Demo is running on CPU"
-    description = f"{demo_status}"
+    description = f"Details: https://github.com/facebookresearch/fairseq/tree/nllb. {demo_status}"
     examples = [
-    ['English', 'Hindi', 'Hi. nice to meet you']
+    ['English', 'Korean', 'Hi. nice to meet you']
     ]
 
     gr.Interface(translation,
@@ -84,3 +102,5 @@ if __name__ == '__main__':
                  title=title,
                  description=description,
                  ).launch()
+
+

flores200_codes.py

@@ -1,36 +1,207 @@
 codes_as_string = '''Acehnese (Arabic script)	ace_Arab
+Acehnese (Latin script)	ace_Latn
+Mesopotamian Arabic	acm_Arab
+Ta’izzi-Adeni Arabic	acq_Arab
+Tunisian Arabic	aeb_Arab
+Afrikaans	afr_Latn
+South Levantine Arabic	ajp_Arab
+Akan	aka_Latn
+Amharic	amh_Ethi
+North Levantine Arabic	apc_Arab
+Modern Standard Arabic	arb_Arab
+Modern Standard Arabic (Romanized)	arb_Latn
+Najdi Arabic	ars_Arab
+Moroccan Arabic	ary_Arab
+Egyptian Arabic	arz_Arab
 Assamese	asm_Beng
+Asturian	ast_Latn
 Awadhi	awa_Deva
+Central Aymara	ayr_Latn
+South Azerbaijani	azb_Arab
+North Azerbaijani	azj_Latn
+Bashkir	bak_Cyrl
+Bambara	bam_Latn
+Balinese	ban_Latn
+Belarusian	bel_Cyrl
+Bemba	bem_Latn
 Bengali	ben_Beng
 Bhojpuri	bho_Deva
+Banjar (Arabic script)	bjn_Arab
+Banjar (Latin script)	bjn_Latn
+Standard Tibetan	bod_Tibt
+Bosnian	bos_Latn
+Buginese	bug_Latn
+Bulgarian	bul_Cyrl
+Catalan	cat_Latn
+Cebuano	ceb_Latn
+Czech	ces_Latn
+Chokwe	cjk_Latn
+Central Kurdish	ckb_Arab
+Crimean Tatar	crh_Latn
+Welsh	cym_Latn
+Danish	dan_Latn
+German	deu_Latn
+Southwestern Dinka	dik_Latn
+Dyula	dyu_Latn
 Dzongkha	dzo_Tibt
+Greek	ell_Grek
 English	eng_Latn
+Esperanto	epo_Latn
+Estonian	est_Latn
+Basque	eus_Latn
+Ewe	ewe_Latn
+Faroese	fao_Latn
 Fijian	fij_Latn
+Finnish	fin_Latn
+Fon	fon_Latn
+French	fra_Latn
+Friulian	fur_Latn
+Nigerian Fulfulde	fuv_Latn
+Scottish Gaelic	gla_Latn
+Irish	gle_Latn
+Galician	glg_Latn
+Guarani	grn_Latn
 Gujarati	guj_Gujr
+Haitian Creole	hat_Latn
+Hausa	hau_Latn
+Hebrew	heb_Hebr
 Hindi	hin_Deva
 Chhattisgarhi	hne_Deva
+Croatian	hrv_Latn
+Hungarian	hun_Latn
+Armenian	hye_Armn
+Igbo	ibo_Latn
+Ilocano	ilo_Latn
+Indonesian	ind_Latn
+Icelandic	isl_Latn
+Italian	ita_Latn
+Javanese	jav_Latn
+Japanese	jpn_Jpan
+Kabyle	kab_Latn
+Jingpho	kac_Latn
+Kamba	kam_Latn
 Kannada	kan_Knda
 Kashmiri (Arabic script)	kas_Arab
 Kashmiri (Devanagari script)	kas_Deva
+Georgian	kat_Geor
+Central Kanuri (Arabic script)	knc_Arab
+Central Kanuri (Latin script)	knc_Latn
+Kazakh	kaz_Cyrl
+Kabiyè	kbp_Latn
+Kabuverdianu	kea_Latn
+Khmer	khm_Khmr
+Kikuyu	kik_Latn
+Kinyarwanda	kin_Latn
+Kyrgyz	kir_Cyrl
+Kimbundu	kmb_Latn
+Northern Kurdish	kmr_Latn
+Kikongo	kon_Latn
+Korean	kor_Hang
+Lao	lao_Laoo
+Ligurian	lij_Latn
+Limburgish	lim_Latn
+Lingala	lin_Latn
+Lithuanian	lit_Latn
+Lombard	lmo_Latn
+Latgalian	ltg_Latn
+Luxembourgish	ltz_Latn
+Luba-Kasai	lua_Latn
+Ganda	lug_Latn
+Luo	luo_Latn
 Mizo	lus_Latn
+Standard Latvian	lvs_Latn
 Magahi	mag_Deva
 Maithili	mai_Deva
 Malayalam	mal_Mlym
 Marathi	mar_Deva
+Minangkabau (Arabic script)	min_Arab
+Minangkabau (Latin script)	min_Latn
+Macedonian	mkd_Cyrl
+Plateau Malagasy	plt_Latn
+Maltese	mlt_Latn
 Meitei (Bengali script)	mni_Beng
+Halh Mongolian	khk_Cyrl
+Mossi	mos_Latn
+Maori	mri_Latn
 Burmese	mya_Mymr
+Dutch	nld_Latn
+Norwegian Nynorsk	nno_Latn
+Norwegian Bokmål	nob_Latn
 Nepali	npi_Deva
+Northern Sotho	nso_Latn
+Nuer	nus_Latn
+Nyanja	nya_Latn
+Occitan	oci_Latn
+West Central Oromo	gaz_Latn
 Odia	ory_Orya
+Pangasinan	pag_Latn
 Eastern Panjabi	pan_Guru
+Papiamento	pap_Latn
 Western Persian	pes_Arab
+Polish	pol_Latn
+Portuguese	por_Latn
+Dari	prs_Arab
+Southern Pashto	pbt_Arab
+Ayacucho Quechua	quy_Latn
+Romanian	ron_Latn
+Rundi	run_Latn
+Russian	rus_Cyrl
+Sango	sag_Latn
 Sanskrit	san_Deva
 Santali	sat_Olck
+Sicilian	scn_Latn
+Shan	shn_Mymr
 Sinhala	sin_Sinh
+Slovak	slk_Latn
+Slovenian	slv_Latn
+Samoan	smo_Latn
+Shona	sna_Latn
 Sindhi	snd_Arab
+Somali	som_Latn
+Southern Sotho	sot_Latn
+Spanish	spa_Latn
+Tosk Albanian	als_Latn
+Sardinian	srd_Latn
+Serbian	srp_Cyrl
+Swati	ssw_Latn
+Sundanese	sun_Latn
+Swedish	swe_Latn
+Swahili	swh_Latn
+Silesian	szl_Latn
 Tamil	tam_Taml
+Tatar	tat_Cyrl
 Telugu	tel_Telu
+Tajik	tgk_Cyrl
+Tagalog	tgl_Latn
+Thai	tha_Thai
+Tigrinya	tir_Ethi
+Tamasheq (Latin script)	taq_Latn
+Tamasheq (Tifinagh script)	taq_Tfng
+Tok Pisin	tpi_Latn
+Tswana	tsn_Latn
+Tsonga	tso_Latn
+Turkmen	tuk_Latn
+Tumbuka	tum_Latn
+Turkish	tur_Latn
+Twi	twi_Latn
+Central Atlas Tamazight	tzm_Tfng
+Uyghur	uig_Arab
+Ukrainian	ukr_Cyrl
+Umbundu	umb_Latn
 Urdu	urd_Arab
-Vietnamese	vie_Latn'''
+Northern Uzbek	uzn_Latn
+Venetian	vec_Latn
+Vietnamese	vie_Latn
+Waray	war_Latn
+Wolof	wol_Latn
+Xhosa	xho_Latn
+Eastern Yiddish	ydd_Hebr
+Yoruba	yor_Latn
+Yue Chinese	yue_Hant
+Chinese (Simplified)	zho_Hans
+Chinese (Traditional)	zho_Hant
+Standard Malay	zsm_Latn
+Zulu	zul_Latn'''
 
 codes_as_string = codes_as_string.split('\n')
 

requirements.txt

@@ -1,3 +1,7 @@
 git+https://github.com/huggingface/transformers
-gradio
-torch
+
+gradio==3.8
+
+torch
+
+httpx==0.24.1