Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,28 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

main.py

@@ -0,0 +1,3 @@
+import subprocess
+
+subprocess.run("uvicorn modules.app:app --host 0.0.0.0 --port 7860", shell=True)

modules/STT & TTS/all_langs.tsv

@@ -0,0 +1,1198 @@
+abi Abidji
+abk Abkhaz
+abp Ayta, Abellen
+aca Achagua
+acd Gikyode
+ace Aceh
+acf Lesser Antillean French Creole
+ach Acholi
+acn Achang
+acr Achi
+acu Achuar-Shiwiar
+ade Adele
+adh Jopadhola
+adj Adioukrou
+adx Tibetan, Amdo
+aeu Akeu
+afr Afrikaans
+agd Agarabi
+agg Angor
+agn Agutaynen
+agr Awajún
+agu Awakateko
+agx Aghul
+aha Ahanta
+ahk Akha
+aia Arosi
+aka Akan
+akb Batak Angkola
+ake Akawaio
+akp Siwu
+alj Alangan
+alp Alune
+alt Altai, Southern
+alz Alur
+ame Yanesha’
+amf Hamer-Banna
+amh Amharic
+ami Amis
+amk Ambai
+ann Obolo
+any Anyin
+aoz Uab Meto
+apb Sa’a
+apr Arop-Lokep
+ara Arabic
+arl Arabela
+asa Asu
+asg Cishingini
+asm Assamese
+ast Asturian
+ata Pele-Ata
+atb Zaiwa
+atg Ivbie North-Okpela-Arhe
+ati Attié
+atq Aralle-Tabulahan
+ava Avar
+avn Avatime
+avu Avokaya
+awa Awadhi
+awb Awa
+ayo Ayoreo
+ayr Aymara, Central
+ayz Mai Brat
+azb Azerbaijani, South
+azg Amuzgo, San Pedro Amuzgos
+azj-script_cyrillic Azerbaijani, North
+azj-script_latin Azerbaijani, North
+azz Nahuatl, Highland Puebla
+bak Bashkort
+bam Bamanankan
+ban Bali
+bao Waimaha
+bas Basaa
+bav Vengo
+bba Baatonum
+bbb Barai
+bbc Batak Toba
+bbo Konabéré
+bcc-script_arabic Balochi, Southern
+bcc-script_latin Balochi, Southern
+bcl Bikol, Central
+bcw Bana
+bdg Bonggi
+bdh Baka
+bdq Bahnar
+bdu Oroko
+bdv Bodo Parja
+beh Biali
+bel Belarusian
+bem Bemba
+ben Bengali
+bep Behoa
+bex Jur Modo
+bfa Bari
+bfo Birifor, Malba
+bfy Bagheli
+bfz Pahari, Mahasu
+bgc Haryanvi
+bgq Bagri
+bgr Chin, Bawm
+bgt Bughotu
+bgw Bhatri
+bha Bharia
+bht Bhattiyali
+bhz Bada
+bib Bisa
+bim Bimoba
+bis Bislama
+biv Birifor, Southern
+bjr Binumarien
+bjv Bedjond
+bjw Bakwé
+bjz Baruga
+bkd Binukid
+bkv Bekwarra
+blh Kuwaa
+blt Tai Dam
+blx Ayta, Mag-Indi
+blz Balantak
+bmq Bomu
+bmr Muinane
+bmu Somba-Siawari
+bmv Bum
+bng Benga
+bno Bantoanon
+bnp Bola
+boa Bora
+bod Tibetan, Central
+boj Anjam
+bom Berom
+bor Borôro
+bos Bosnian
+bov Tuwuli
+box Buamu
+bpr Blaan, Koronadal
+bps Blaan, Sarangani
+bqc Boko
+bqi Bakhtiâri
+bqj Bandial
+bqp Bisã
+bre Breton
+bru Bru, Eastern
+bsc Oniyan
+bsq Bassa
+bss Akoose
+btd Batak Dairi
+bts Batak Simalungun
+btt Bete-Bendi
+btx Batak Karo
+bud Ntcham
+bul Bulgarian
+bus Bokobaru
+bvc Baelelea
+bvz Bauzi
+bwq Bobo Madaré, Southern
+bwu Buli
+byr Yipma
+bzh Buang, Mapos
+bzi Bisu
+bzj Belize English Creole
+caa Ch’orti’
+cab Garifuna
+cac-dialect_sanmateoixtatan Chuj
+cac-dialect_sansebastiancoatan Chuj
+cak-dialect_central Kaqchikel
+cak-dialect_santamariadejesus Kaqchikel
+cak-dialect_santodomingoxenacoj Kaqchikel
+cak-dialect_southcentral Kaqchikel
+cak-dialect_western Kaqchikel
+cak-dialect_yepocapa Kaqchikel
+cap Chipaya
+car Carib
+cas Tsimané
+cat Catalan
+cax Chiquitano
+cbc Carapana
+cbi Chachi
+cbr Kakataibo-Kashibo
+cbs Kashinawa
+cbt Shawi
+cbu Kandozi-Chapra
+cbv Cacua
+cce Chopi
+cco Chinantec, Comaltepec
+cdj Churahi
+ceb Cebuano
+ceg Chamacoco
+cek Chin, Eastern Khumi
+ces Czech
+cfm Chin, Falam
+cgc Kagayanen
+che Chechen
+chf Chontal, Tabasco
+chv Chuvash
+chz Chinantec, Ozumacín
+cjo Ashéninka, Pajonal
+cjp Cabécar
+cjs Shor
+ckb Kurdish, Central
+cko Anufo
+ckt Chukchi
+cla Ron
+cle Chinantec, Lealao
+cly Chatino, Eastern Highland
+cme Cerma
+cmn-script_simplified Chinese, Mandarin
+cmo-script_khmer Mnong, Central
+cmo-script_latin Mnong, Central
+cmr Mro-Khimi
+cnh Chin, Hakha
+cni Asháninka
+cnl Chinantec, Lalana
+cnt Chinantec, Tepetotutla
+coe Koreguaje
+cof Tsafiki
+cok Cora, Santa Teresa
+con Cofán
+cot Caquinte
+cou Wamey
+cpa Chinantec, Palantla
+cpb Ashéninka, Ucayali-Yurúa
+cpu Ashéninka, Pichis
+crh Crimean Tatar
+crk-script_latin Cree, Plains
+crk-script_syllabics Cree, Plains
+crn Cora, El Nayar
+crq Chorote, Iyo’wujwa
+crs Seychelles French Creole
+crt Chorote, Iyojwa’ja
+csk Jola-Kasa
+cso Chinantec, Sochiapam
+ctd Chin, Tedim
+ctg Chittagonian
+cto Embera Catío
+ctu Chol
+cuc Chinantec, Usila
+cui Cuiba
+cuk Kuna, San Blas
+cul Kulina
+cwa Kabwa
+cwe Kwere
+cwt Kuwaataay
+cya Chatino, Nopala
+cym Welsh
+daa Dangaléat
+dah Gwahatike
+dan Danish
+dar Dargwa
+dbj Ida’an
+dbq Daba
+ddn Dendi
+ded Dedua
+des Desano
+deu German, Standard
+dga Dagaare, Southern
+dgi Dagara, Northern
+dgk Dagba
+dgo Dogri
+dgr Tlicho
+dhi Dhimal
+did Didinga
+dig Chidigo
+dik Dinka, Southwestern
+dip Dinka, Northeastern
+div Maldivian
+djk Aukan
+dnj-dialect_blowowest Dan
+dnj-dialect_gweetaawueast Dan
+dnt Dani, Mid Grand Valley
+dnw Dani, Western
+dop Lukpa
+dos Dogosé
+dsh Daasanach
+dso Desiya
+dtp Kadazan Dusun
+dts Dogon, Toro So
+dug Chiduruma
+dwr Dawro
+dyi Sénoufo, Djimini
+dyo Jola-Fonyi
+dyu Jula
+dzo Dzongkha
+eip Lik
+eka Ekajuk
+ell Greek
+emp Emberá, Northern
+enb Markweeta
+eng English
+enx Enxet
+epo Esperanto
+ese Ese Ejja
+ess Yupik, Saint Lawrence Island
+est Estonian
+eus Basque
+evn Evenki
+ewe Éwé
+eza Ezaa
+fal Fali, South
+fao Faroese
+far Fataleka
+fas Persian
+fij Fijian
+fin Finnish
+flr Fuliiru
+fmu Muria, Far Western
+fon Fon
+fra French
+frd Fordata
+fry Frisian
+ful Fulah
+gag-script_cyrillic Gagauz
+gag-script_latin Gagauz
+gai Mbore
+gam Kandawo
+gau Gadaba, Mudhili
+gbi Galela
+gbk Gaddi
+gbm Garhwali
+gbo Grebo, Northern
+gde Gude
+geb Kire
+gej Gen
+gil Kiribati
+gjn Gonja
+gkn Gokana
+gld Nanai
+gle Irish
+glg Galician
+glk Gilaki
+gmv Gamo
+gna Kaansa
+gnd Zulgo-Gemzek
+gng Ngangam
+gof-script_latin Gofa
+gog Gogo
+gor Gorontalo
+gqr Gor
+grc Greek, Ancient
+gri Ghari
+grn Guarani
+grt Garo
+gso Gbaya, Southwest
+gub Guajajára
+guc Wayuu
+gud Dida, Yocoboué
+guh Guahibo
+guj Gujarati
+guk Gumuz
+gum Misak
+guo Guayabero
+guq Aché
+guu Yanomamö
+gux Gourmanchéma
+gvc Wanano
+gvl Gulay
+gwi Gwich’in
+gwr Gwere
+gym Ngäbere
+gyr Guarayu
+had Hatam
+hag Hanga
+hak Chinese, Hakka
+hap Hupla
+hat Haitian Creole
+hau Hausa
+hay Haya
+heb Hebrew
+heh Hehe
+hif Hindi, Fiji
+hig Kamwe
+hil Hiligaynon
+hin Hindi
+hlb Halbi
+hlt Chin, Matu
+hne Chhattisgarhi
+hnn Hanunoo
+hns Hindustani, Sarnami
+hoc Ho
+hoy Holiya
+hrv Croatian
+hsb Sorbian, Upper
+hto Witoto, Minika
+hub Wampís
+hui Huli
+hun Hungarian
+hus-dialect_centralveracruz Huastec
+hus-dialect_westernpotosino Huastec
+huu Witoto, Murui
+huv Huave, San Mateo del Mar
+hvn Hawu
+hwc Hawaii Pidgin
+hye Armenian
+hyw Armenian, Western
+iba Iban
+ibo Igbo
+icr Islander English Creole
+idd Ede Idaca
+ifa Ifugao, Amganad
+ifb Ifugao, Batad
+ife Ifè
+ifk Ifugao, Tuwali
+ifu Ifugao, Mayoyao
+ify Kallahan, Keley-i
+ign Ignaciano
+ikk Ika
+ilb Ila
+ilo Ilocano
+imo Imbongu
+ina Interlingua (International Auxiliary Language Association)
+inb Inga
+ind Indonesian
+iou Tuma-Irumu
+ipi Ipili
+iqw Ikwo
+iri Rigwe
+irk Iraqw
+isl Icelandic
+ita Italian
+itl Itelmen
+itv Itawit
+ixl-dialect_sangasparchajul Ixil
+ixl-dialect_sanjuancotzal Ixil
+ixl-dialect_santamarianebaj Ixil
+izr Izere
+izz Izii
+jac Jakalteko
+jam Jamaican English Creole
+jav Javanese
+jbu Jukun Takum
+jen Dza
+jic Tol
+jiv Shuar
+jmc Machame
+jmd Yamdena
+jpn Japanese
+jun Juang
+juy Juray
+jvn Javanese, Suriname
+kaa Karakalpak
+kab Amazigh
+kac Jingpho
+kak Kalanguya
+kam Kamba
+kan Kannada
+kao Xaasongaxango
+kaq Capanahua
+kat Georgian
+kay Kamayurá
+kaz Kazakh
+kbo Keliko
+kbp Kabiyè
+kbq Kamano
+kbr Kafa
+kby Kanuri, Manga
+kca Khanty
+kcg Tyap
+kdc Kutu
+kde Makonde
+kdh Tem
+kdi Kumam
+kdj Ng’akarimojong
+kdl Tsikimba
+kdn Kunda
+kdt Kuay
+kea Kabuverdianu
+kek Q’eqchi’
+ken Kenyang
+keo Kakwa
+ker Kera
+key Kupia
+kez Kukele
+kfb Kolami, Northwestern
+kff-script_telugu Koya
+kfw Naga, Kharam
+kfx Pahari, Kullu
+khg Tibetan, Khams
+khm Khmer
+khq Songhay, Koyra Chiini
+kia Kim
+kij Kilivila
+kik Gikuyu
+kin Kinyarwanda
+kir Kyrgyz
+kjb Q’anjob’al
+kje Kisar
+kjg Khmu
+kjh Khakas
+kki Kagulu
+kkj Kako
+kle Kulung
+klu Klao
+klv Maskelynes
+klw Tado
+kma Konni
+kmd Kalinga, Majukayang
+kml Kalinga, Tanudan
+kmr-script_arabic Kurdish, Northern
+kmr-script_cyrillic Kurdish, Northern
+kmr-script_latin Kurdish, Northern
+kmu Kanite
+knb Kalinga, Lubuagan
+kne Kankanaey
+knf Mankanya
+knj Akateko
+knk Kuranko
+kno Kono
+kog Kogi
+kor Korean
+kpq Korupun-Sela
+kps Tehit
+kpv Komi-Zyrian
+kpy Koryak
+kpz Kupsapiiny
+kqe Kalagan
+kqp Kimré
+kqr Kimaragang
+kqy Koorete
+krc Karachay-Balkar
+kri Krio
+krj Kinaray-a
+krl Karelian
+krr Krung
+krs Gbaya
+kru Kurux
+ksb Shambala
+ksr Borong
+kss Kisi, Southern
+ktb Kambaata
+ktj Krumen, Plapo
+kub Kutep
+kue Kuman
+kum Kumyk
+kus Kusaal
+kvn Kuna, Border
+kvw Wersing
+kwd Kwaio
+kwf Kwara’ae
+kwi Awa-Cuaiquer
+kxc Konso
+kxf Kawyaw
+kxm Khmer, Northern
+kxv Kuvi
+kyb Kalinga, Butbut
+kyc Kyaka
+kyf Kouya
+kyg Keyagana
+kyo Klon
+kyq Kenga
+kyu Kayah, Western
+kyz Kayabí
+kzf Kaili, Da’a
+lac Lacandon
+laj Lango
+lam Lamba
+lao Lao
+las Lama
+lat Latin
+lav Latvian
+law Lauje
+lbj Ladakhi
+lbw Tolaki
+lcp Lawa, Western
+lee Lyélé
+lef Lelemi
+lem Nomaande
+lew Kaili, Ledo
+lex Luang
+lgg Lugbara
+lgl Wala
+lhu Lahu
+lia Limba, West-Central
+lid Nyindrou
+lif Limbu
+lin Lingala
+lip Sekpele
+lis Lisu
+lit Lithuanian
+lje Rampi
+ljp Lampung Api
+llg Lole
+lln Lele
+lme Pévé
+lnd Lundayeh
+lns Lamnso’
+lob Lobi
+lok Loko
+lom Loma
+lon Lomwe, Malawi
+loq Lobala
+lsi Lacid
+lsm Saamya-Gwe
+ltz Luxembourgish
+luc Aringa
+lug Ganda
+luo Dholuo
+lwo Luwo
+lww Lewo
+lzz Laz
+maa-dialect_sanantonio Mazatec, San Jerónimo Tecóatl
+maa-dialect_sanjeronimo Mazatec, San Jerónimo Tecóatl
+mad Madura
+mag Magahi
+mah Marshallese
+mai Maithili
+maj Mazatec, Jalapa de Díaz
+mak Makasar
+mal Malayalam
+mam-dialect_central Mam
+mam-dialect_northern Mam
+mam-dialect_southern Mam
+mam-dialect_western Mam
+maq Mazatec, Chiquihuitlán
+mar Marathi
+maw Mampruli
+maz Mazahua, Central
+mbb Manobo, Western Bukidnon
+mbc Macushi
+mbh Mangseng
+mbj Nadëb
+mbt Manobo, Matigsalug
+mbu Mbula-Bwazza
+mbz Mixtec, Amoltepec
+mca Maka
+mcb Matsigenka
+mcd Sharanahua
+mco Mixe, Coatlán
+mcp Makaa
+mcq Ese
+mcu Mambila, Cameroon
+mda Mada
+mdf Moksha
+mdv Mixtec, Santa Lucía Monteverde
+mdy Male
+med Melpa
+mee Mengen
+mej Meyah
+men Mende
+meq Merey
+met Mato
+mev Maan
+mfe Morisyen
+mfh Matal
+mfi Wandala
+mfk Mofu, North
+mfq Moba
+mfy Mayo
+mfz Mabaan
+mgd Moru
+mge Mango
+mgh Makhuwa-Meetto
+mgo Meta’
+mhi Ma’di
+mhr Mari, Meadow
+mhu Digaro-Mishmi
+mhx Lhao Vo
+mhy Ma’anyan
+mib Mixtec, Atatlahuca
+mie Mixtec, Ocotepec
+mif Mofu-Gudur
+mih Mixtec, Chayuco
+mil Mixtec, Peñoles
+mim Mixtec, Alacatlatzala
+min Minangkabau
+mio Mixtec, Pinotepa Nacional
+mip Mixtec, Apasco-Apoala
+miq Mískito
+mit Mixtec, Southern Puebla
+miy Mixtec, Ayutla
+miz Mixtec, Coatzospan
+mjl Mandeali
+mjv Mannan
+mkd Macedonian
+mkl Mokole
+mkn Malay, Kupang
+mlg Malagasy
+mlt Maltese
+mmg Ambrym, North
+mnb Muna
+mnf Mundani
+mnk Mandinka
+mnw Mon
+mnx Sougb
+moa Mwan
+mog Mongondow
+mon Mongolian
+mop Maya, Mopán
+mor Moro
+mos Mòoré
+mox Molima
+moz Mukulu
+mpg Marba
+mpm Mixtec, Yosondúa
+mpp Migabac
+mpx Misima-Panaeati
+mqb Mbuko
+mqf Momuna
+mqj Mamasa
+mqn Moronene
+mri Maori
+mrw Maranao
+msy Aruamu
+mtd Mualang
+mtj Moskona
+mto Mixe, Totontepec
+muh Mündü
+mup Malvi
+mur Murle
+muv Muthuvan
+muy Muyang
+mvp Duri
+mwq Chin, Müün
+mwv Mentawai
+mxb Mixtec, Tezoatlán
+mxq Mixe, Juquila
+mxt Mixtec, Jamiltepec
+mxv Mixtec, Metlatónoc
+mya Burmese
+myb Mbay
+myk Sénoufo, Mamara
+myl Moma
+myv Erzya
+myx Masaaba
+myy Macuna
+mza Mixtec, Santa María Zacatepec
+mzi Mazatec, Ixcatlán
+mzj Manya
+mzk Mambila, Nigeria
+mzm Mumuye
+mzw Deg
+nab Nambikuára, Southern
+nag Nagamese
+nan Chinese, Min Nan
+nas Naasioi
+naw Nawuri
+nca Iyo
+nch Nahuatl, Central Huasteca
+ncj Nahuatl, Northern Puebla
+ncl Nahuatl, Michoacán
+ncu Chumburung
+ndj Ndamba
+ndp Kebu
+ndv Ndut
+ndy Lutos
+ndz Ndogo
+neb Toura
+new Newar
+nfa Dhao
+nfr Nafaanra
+nga Ngbaka
+ngl Lomwe
+ngp Ngulu
+ngu Nahuatl, Guerrero
+nhe Nahuatl, Eastern Huasteca
+nhi Nahuatl, Zacatlán-Ahuacatlán-Tepetzintla
+nhu Noone
+nhw Nahuatl, Western Huasteca
+nhx Nahuatl, Isthmus-Mecayapan
+nhy Nahuatl, Northern Oaxaca
+nia Nias
+nij Ngaju
+nim Nilamba
+nin Ninzo
+nko Nkonya
+nlc Nalca
+nld Dutch
+nlg Gela
+nlk Yali, Ninia
+nmz Nawdm
+nnb Nande
+nno Norwegian Nynorsk
+nnq Ngindo
+nnw Nuni, Southern
+noa Woun Meu
+nob Norwegian Bokmål
+nod Thai, Northern
+nog Nogai
+not Nomatsigenga
+npi Nepali
+npl Nahuatl, Southeastern Puebla
+npy Napu
+nso Sotho, Northern
+nst Naga, Tangshang
+nsu Nahuatl, Sierra Negra
+ntm Nateni
+ntr Delo
+nuj Nyole
+nus Nuer
+nuz Nahuatl, Tlamacazapa
+nwb Nyabwa
+nxq Naxi
+nya Chichewa
+nyf Kigiryama
+nyn Nyankore
+nyo Nyoro
+nyy Nyakyusa-Ngonde
+nzi Nzema
+obo Manobo, Obo
+oci Occitan
+ojb-script_latin Ojibwa, Northwestern
+ojb-script_syllabics Ojibwa, Northwestern
+oku Oku
+old Mochi
+omw Tairora, South
+onb Lingao
+ood Tohono O’odham
+orm Oromo
+ory Odia
+oss Ossetic
+ote Otomi, Mezquital
+otq Otomi, Querétaro
+ozm Koonzime
+pab Parecís
+pad Paumarí
+pag Pangasinan
+pam Kapampangan
+pan Punjabi, Eastern
+pao Paiute, Northern
+pap Papiamentu
+pau Palauan
+pbb Nasa
+pbc Patamona
+pbi Parkwa
+pce Palaung, Ruching
+pcm Pidgin, Nigerian
+peg Pengo
+pez Penan, Eastern
+pib Yine
+pil Yom
+pir Piratapuyo
+pis Pijin
+pjt Pitjantjatjara
+pkb Kipfokomo
+pls Popoloca, San Marcos Tlacoyalco
+plw Palawano, Brooke’s Point
+pmf Pamona
+pny Pinyin
+poh-dialect_eastern Poqomchi’
+poh-dialect_western Poqomchi’
+poi Popoluca, Highland
+pol Polish
+por Portuguese
+poy Pogolo
+ppk Uma
+pps Popoloca, San Luís Temalacayuca
+prf Paranan
+prk Wa, Parauk
+prt Prai
+pse Malay, Central
+pss Kaulong
+ptu Bambam
+pui Puinave
+pus Pushto
+pwg Gapapaiwa
+pww Karen, Pwo Northern
+pxm Mixe, Quetzaltepec
+qub Quechua, Huallaga
+quc-dialect_central K’iche’
+quc-dialect_east K’iche’
+quc-dialect_north K’iche’
+quf Quechua, Lambayeque
+quh Quechua, South Bolivian
+qul Quechua, North Bolivian
+quw Quichua, Tena Lowland
+quy Quechua, Ayacucho
+quz Quechua, Cusco
+qvc Quechua, Cajamarca
+qve Quechua, Eastern Apurímac
+qvh Quechua, Huamalíes-Dos de Mayo Huánuco
+qvm Quechua, Margos-Yarowilca-Lauricocha
+qvn Quechua, North Junín
+qvo Quichua, Napo
+qvs Quechua, San Martín
+qvw Quechua, Huaylla Wanca
+qvz Quichua, Northern Pastaza
+qwh Quechua, Huaylas Ancash
+qxh Quechua, Panao
+qxl Quichua, Salasaca Highland
+qxn Quechua, Northern Conchucos Ancash
+qxo Quechua, Southern Conchucos
+qxr Quichua, Cañar Highland
+rah Rabha
+rai Ramoaaina
+rap Rapa Nui
+rav Sampang
+raw Rawang
+rej Rejang
+rel Rendille
+rgu Rikou
+rhg Rohingya
+rif-script_arabic Tarifit
+rif-script_latin Tarifit
+ril Riang Lang
+rim Nyaturu
+rjs Rajbanshi
+rkt Rangpuri
+rmc-script_cyrillic Romani, Carpathian
+rmc-script_latin Romani, Carpathian
+rmo Romani, Sinte
+rmy-script_cyrillic Romani, Vlax
+rmy-script_latin Romani, Vlax
+rng Ronga
+rnl Ranglong
+roh-dialect_sursilv Romansh
+roh-dialect_vallader Romansh
+rol Romblomanon
+ron Romanian
+rop Kriol
+rro Waima
+rub Gungu
+ruf Luguru
+rug Roviana
+run Rundi
+rus Russian
+sab Buglere
+sag Sango
+sah Yakut
+saj Sahu
+saq Samburu
+sas Sasak
+sat Santhali
+sba Ngambay
+sbd Samo, Southern
+sbl Sambal, Botolan
+sbp Sangu
+sch Sakachep
+sck Sadri
+sda Toraja-Sa’dan
+sea Semai
+seh Sena
+ses Songhay, Koyraboro Senni
+sey Paicoca
+sgb Ayta, Mag-antsi
+sgj Surgujia
+sgw Sebat Bet Gurage
+shi Tachelhit
+shk Shilluk
+shn Shan
+sho Shanga
+shp Shipibo-Conibo
+sid Sidamo
+sig Paasaal
+sil Sisaala, Tumulung
+sja Epena
+sjm Mapun
+sld Sissala
+slk Slovak
+slu Selaru
+slv Slovene
+sml Sama, Central
+smo Samoan
+sna Shona
+snd Sindhi
+sne Bidayuh, Bau
+snn Siona
+snp Siane
+snw Selee
+som Somali
+soy Miyobe
+spa Spanish
+spp Sénoufo, Supyire
+spy Sabaot
+sqi Albanian
+sri Siriano
+srm Saramaccan
+srn Sranan Tongo
+srp-script_cyrillic Serbian
+srp-script_latin Serbian
+srx Sirmauri
+stn Owa
+stp Tepehuan, Southeastern
+suc Subanon, Western
+suk Sukuma
+sun Sunda
+sur Mwaghavul
+sus Susu
+suv Puroik
+suz Sunwar
+swe Swedish
+swh Swahili
+sxb Suba
+sxn Sangir
+sya Siang
+syl Sylheti
+sza Semelai
+tac Tarahumara, Western
+taj Tamang, Eastern
+tam Tamil
+tao Yami
+tap Taabwa
+taq Tamasheq
+tat Tatar
+tav Tatuyo
+tbc Takia
+tbg Tairora, North
+tbk Tagbanwa, Calamian
+tbl Tboli
+tby Tabaru
+tbz Ditammari
+tca Ticuna
+tcc Datooga
+tcs Torres Strait Creole
+tcz Chin, Thado
+tdj Tajio
+ted Krumen, Tepo
+tee Tepehua, Huehuetla
+tel Telugu
+tem Themne
+teo Ateso
+ter Terêna
+tes Tengger
+tew Tewa
+tex Tennet
+tfr Teribe
+tgj Tagin
+tgk Tajik
+tgl Tagalog
+tgo Sudest
+tgp Tangoa
+tha Thai
+thk Kitharaka
+thl Tharu, Dangaura
+tih Murut, Timugon
+tik Tikar
+tir Tigrigna
+tkr Tsakhur
+tlb Tobelo
+tlj Talinga-Bwisi
+tly Talysh
+tmc Tumak
+tmf Toba-Maskoy
+tna Tacana
+tng Tobanga
+tnk Kwamera
+tnn Tanna, North
+tnp Whitesands
+tnr Ménik
+tnt Tontemboan
+tob Toba
+toc Totonac, Coyutla
+toh Tonga
+tom Tombulu
+tos Totonac, Highland
+tpi Tok Pisin
+tpm Tampulma
+tpp Tepehua, Pisaflores
+tpt Tepehua, Tlachichilco
+trc Triqui, Copala
+tri Trió
+trn Trinitario
+trs Triqui, Chicahuaxtla
+tso Tsonga
+tsz Purepecha
+ttc Tektiteko
+tte Bwanabwana
+ttq-script_tifinagh Tamajaq, Tawallammat
+tue Tuyuca
+tuf Tunebo, Central
+tuk-script_arabic Turkmen
+tuk-script_latin Turkmen
+tuo Tucano
+tur Turkish
+tvw Sedoa
+twb Tawbuid
+twe Teiwa
+twu Termanu
+txa Tombonuo
+txq Tii
+txu Kayapó
+tye Kyanga
+tzh-dialect_bachajon Tzeltal
+tzh-dialect_tenejapa Tzeltal
+tzj-dialect_eastern Tz’utujil
+tzj-dialect_western Tz’utujil
+tzo-dialect_chamula Tzotzil
+tzo-dialect_chenalho Tzotzil
+ubl Bikol, Buhi’non
+ubu Umbu-Ungu
+udm Udmurt
+udu Uduk
+uig-script_arabic Uyghur
+uig-script_cyrillic Uyghur
+ukr Ukrainian
+umb Umbundu
+unr Mundari
+upv Uripiv-Wala-Rano-Atchin
+ura Urarina
+urb Kaapor
+urd-script_arabic Urdu
+urd-script_devanagari Urdu
+urd-script_latin Urdu
+urk Urak Lawoi’
+urt Urat
+ury Orya
+usp Uspanteko
+uzb-script_cyrillic Uzbek
+uzb-script_latin Uzbek
+vag Vagla
+vid Vidunda
+vie Vietnamese
+vif Vili
+vmw Makhuwa
+vmy Mazatec, Ayautla
+vot Vod
+vun Vunjo
+vut Vute
+wal-script_ethiopic Wolaytta
+wal-script_latin Wolaytta
+wap Wapishana
+war Waray-Waray
+waw Waiwai
+way Wayana
+wba Warao
+wlo Wolio
+wlx Wali
+wmw Mwani
+wob Wè Northern
+wol Wolof
+wsg Gondi, Adilabad
+wwa Waama
+xal Kalmyk-Oirat
+xdy Malayic Dayak
+xed Hdi
+xer Xerénte
+xho Xhosa
+xmm Malay, Manado
+xnj Chingoni
+xnr Kangri
+xog Soga
+xon Konkomba
+xrb Karaboro, Eastern
+xsb Sambal
+xsm Kasem
+xsr Sherpa
+xsu Sanumá
+xta Mixtec, Alcozauca
+xtd Mixtec, Diuxi-Tilantongo
+xte Ketengban
+xtm Mixtec, Magdalena Peñasco
+xtn Mixtec, Northern Tlaxiaco
+xua Kurumba, Alu
+xuo Kuo
+yaa Yaminahua
+yad Yagua
+yal Yalunka
+yam Yamba
+yao Yao
+yas Nugunu
+yat Yambeta
+yaz Lokaa
+yba Yala
+ybb Yemba
+ycl Lolopo
+ycn Yucuna
+yea Ravula
+yka Yakan
+yli Yali, Angguruk
+yor Yoruba
+yre Yaouré
+yua Maya, Yucatec
+yue-script_traditional Chinese, Yue
+yuz Yuracare
+yva Yawa
+zaa Zapotec, Sierra de Juárez
+zab Zapotec, Western Tlacolula Valley
+zac Zapotec, Ocotlán
+zad Zapotec, Cajonos
+zae Zapotec, Yareni
+zai Zapotec, Isthmus
+zam Zapotec, Miahuatlán
+zao Zapotec, Ozolotepec
+zaq Zapotec, Aloápam
+zar Zapotec, Rincón
+zas Zapotec, Santo Domingo Albarradas
+zav Zapotec, Yatzachi
+zaw Zapotec, Mitla
+zca Zapotec, Coatecas Altas
+zga Kinga
+zim Mesme
+ziw Zigula
+zlm Malay
+zmz Mbandja
+zne Zande
+zos Zoque, Francisco León
+zpc Zapotec, Choapan
+zpg Zapotec, Guevea de Humboldt
+zpi Zapotec, Santa María Quiegolani
+zpl Zapotec, Lachixío
+zpm Zapotec, Mixtepec
+zpo Zapotec, Amatlán
+zpt Zapotec, San Vicente Coatlán
+zpu Zapotec, Yalálag
+zpz Zapotec, Texmelucan
+ztq Zapotec, Quioquitani-Quierí
+zty Zapotec, Yatee
+zul Zulu
+zyb Zhuang, Yongbei
+zyp Chin, Zyphe
+zza Zaza

modules/app.py

@@ -0,0 +1,171 @@
+'''
+        Created By Lewis Kamau Kimaru 
+        Sema translator api backend
+        January 2024
+        Docker deployment
+'''
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import HTMLResponse
+import gradio as gr
+import ctranslate2
+import sentencepiece as spm
+import fasttext
+import uvicorn
+import pytz
+from datetime import datetime
+import os
+
+app = FastAPI()
+
+fasttext.FastText.eprint = lambda x: None
+
+# Get time of request
+
+def get_time():
+    nairobi_timezone = pytz.timezone('Africa/Nairobi')
+    current_time_nairobi = datetime.now(nairobi_timezone)
+    
+    curr_day = current_time_nairobi.strftime('%A')
+    curr_date = current_time_nairobi.strftime('%Y-%m-%d')
+    curr_time = current_time_nairobi.strftime('%H:%M:%S')
+    
+    full_date = f"{curr_day} | {curr_date} | {curr_time}"
+    return full_date, curr_time
+    
+# Load the model and tokenizer ..... only once!
+beam_size = 1  # change to a smaller value for faster inference
+device = "cpu"  # or "cuda"
+
+# Language Prediction model
+print("\nimporting Language Prediction model")
+lang_model_file = "lid218e.bin"
+lang_model_full_path = os.path.join(os.path.dirname(__file__), lang_model_file)
+lang_model = fasttext.load_model(lang_model_full_path)
+
+
+# Load the source SentencePiece model
+print("\nimporting SentencePiece model")
+sp_model_file = "spm.model"
+sp_model_full_path = os.path.join(os.path.dirname(__file__), sp_model_file)
+sp = spm.SentencePieceProcessor()
+sp.load(sp_model_full_path)
+
+# Import The Translator model
+print("\nimporting Translator model")
+ct_model_file = "sematrans-3.3B"
+ct_model_full_path = os.path.join(os.path.dirname(__file__), ct_model_file)
+translator = ctranslate2.Translator(ct_model_full_path, device)
+
+print('\nDone importing models\n')
+
+    
+def translate_detect(userinput: str, target_lang: str):
+    source_sents = [userinput]
+    source_sents = [sent.strip() for sent in source_sents]
+    target_prefix = [[target_lang]] * len(source_sents)
+
+    # Predict the source language
+    predictions = lang_model.predict(source_sents[0], k=1)
+    source_lang = predictions[0][0].replace('__label__', '')
+
+    # Subword the source sentences
+    source_sents_subworded = sp.encode(source_sents, out_type=str)
+    source_sents_subworded = [[source_lang] + sent + ["</s>"] for sent in source_sents_subworded]
+
+    # Translate the source sentences
+    translations = translator.translate_batch(
+        source_sents_subworded,
+        batch_type="tokens",
+        max_batch_size=2024,
+        beam_size=beam_size,
+        target_prefix=target_prefix,
+    )
+    translations = [translation[0]['tokens'] for translation in translations]
+
+    # Desubword the target sentences
+    translations_desubword = sp.decode(translations)
+    translations_desubword = [sent[len(target_lang):] for sent in translations_desubword]
+
+    # Return the source language and the translated text
+    return source_lang, translations_desubword
+
+def translate_enter(userinput: str, source_lang: str, target_lang: str):  
+  source_sents = [userinput]
+  source_sents = [sent.strip() for sent in source_sents]
+  target_prefix = [[target_lang]] * len(source_sents)
+
+  # Subword the source sentences
+  source_sents_subworded = sp.encode(source_sents, out_type=str)
+  source_sents_subworded = [[source_lang] + sent + ["</s>"] for sent in source_sents_subworded]
+
+  # Translate the source sentences
+  translations = translator.translate_batch(source_sents_subworded, batch_type="tokens", max_batch_size=2024, beam_size=beam_size, target_prefix=target_prefix)
+  translations = [translation[0]['tokens'] for translation in translations]
+
+  # Desubword the target sentences
+  translations_desubword = sp.decode(translations)
+  translations_desubword = [sent[len(target_lang):] for sent in translations_desubword]
+
+  # Return the source language and the translated text
+  return translations_desubword[0]
+
+
+@app.get("/")
+async def read_root():
+    gradio_interface = """
+    <html>
+    <meta name="viewport" content="width=device-width, height=device-height, initial-scale=1.0">
+    <head>
+        <title>Sema</title>
+    </head>
+    <frameset>
+        <frame src=https://kamau1-semaapi-frontend.hf.space/?embedded=true'>
+    </frameset>
+    </html>
+    """
+    return HTMLResponse(content=gradio_interface)
+
+
+@app.post("/translate_detect/")
+async def translate_detect_endpoint(request: Request):
+    datad = await request.json()
+    userinputd = datad.get("userinput")
+    target_langd = datad.get("target_lang")
+    dfull_date = get_time()[0]
+    print(f"\nrequest: {dfull_date}\nTarget Language; {target_langd}, User Input: {userinputd}\n")
+
+    if not userinputd or not target_langd:
+        raise HTTPException(status_code=422, detail="Both 'userinput' and 'target_lang' are required.")
+
+    source_langd, translated_text_d = translate_detect(userinputd, target_langd)
+    dcurrent_time = get_time()[1]
+    print(f"\nresponse: {dcurrent_time}; ... Source_language: {source_langd}, Translated Text: {translated_text_d}\n\n")
+    return {
+        "source_language": source_langd,
+        "translated_text": translated_text_d[0],
+    }
+
+
+@app.post("/translate_enter/")
+async def translate_enter_endpoint(request: Request):
+    datae = await request.json()
+    userinpute = datae.get("userinput")
+    source_lange = datae.get("source_lang")
+    target_lange = datae.get("target_lang")
+    efull_date = get_time()[0]
+    print(f"\nrequest: {efull_date}\nSource_language; {source_lange}, Target Language; {target_lange}, User Input: {userinpute}\n")
+
+    if not userinpute or not target_lange:
+        raise HTTPException(status_code=422, detail="'userinput' 'sourc_lang'and 'target_lang' are required.")
+
+    translated_text_e = translate_enter(userinpute, source_lange, target_lange)
+    ecurrent_time = get_time()[1]
+    print(f"\nresponse: {ecurrent_time}; ... Translated Text: {translated_text_e}\n\n")
+    return {
+        "translated_text": translated_text_e,
+    }
+
+
+print("\nAPI starting .......\n")

modules/flores200_codes.py

@@ -0,0 +1,211 @@
+codes_as_string = '''Acehnese (Arabic script)	ace_Arab
+Acehnese (Latin script)	ace_Latn
+Mesopotamian Arabic	acm_Arab
+Ta’izzi-Adeni Arabic	acq_Arab
+Tunisian Arabic	aeb_Arab
+Afrikaans	afr_Latn
+South Levantine Arabic	ajp_Arab
+Akan	aka_Latn
+Amharic	amh_Ethi
+North Levantine Arabic	apc_Arab
+Modern Standard Arabic	arb_Arab
+Modern Standard Arabic (Romanized)	arb_Latn
+Najdi Arabic	ars_Arab
+Moroccan Arabic	ary_Arab
+Egyptian Arabic	arz_Arab
+Assamese	asm_Beng
+Asturian	ast_Latn
+Awadhi	awa_Deva
+Central Aymara	ayr_Latn
+South Azerbaijani	azb_Arab
+North Azerbaijani	azj_Latn
+Bashkir	bak_Cyrl
+Bambara	bam_Latn
+Balinese	ban_Latn
+Belarusian	bel_Cyrl
+Bemba	bem_Latn
+Bengali	ben_Beng
+Bhojpuri	bho_Deva
+Banjar (Arabic script)	bjn_Arab
+Banjar (Latin script)	bjn_Latn
+Standard Tibetan	bod_Tibt
+Bosnian	bos_Latn
+Buginese	bug_Latn
+Bulgarian	bul_Cyrl
+Catalan	cat_Latn
+Cebuano	ceb_Latn
+Czech	ces_Latn
+Chokwe	cjk_Latn
+Central Kurdish	ckb_Arab
+Crimean Tatar	crh_Latn
+Welsh	cym_Latn
+Danish	dan_Latn
+German	deu_Latn
+Southwestern Dinka	dik_Latn
+Dyula	dyu_Latn
+Dzongkha	dzo_Tibt
+Greek	ell_Grek
+English	eng_Latn
+Esperanto	epo_Latn
+Estonian	est_Latn
+Basque	eus_Latn
+Ewe	ewe_Latn
+Faroese	fao_Latn
+Fijian	fij_Latn
+Finnish	fin_Latn
+Fon	fon_Latn
+French	fra_Latn
+Friulian	fur_Latn
+Nigerian Fulfulde	fuv_Latn
+Scottish Gaelic	gla_Latn
+Irish	gle_Latn
+Galician	glg_Latn
+Guarani	grn_Latn
+Gujarati	guj_Gujr
+Haitian Creole	hat_Latn
+Hausa	hau_Latn
+Hebrew	heb_Hebr
+Hindi	hin_Deva
+Chhattisgarhi	hne_Deva
+Croatian	hrv_Latn
+Hungarian	hun_Latn
+Armenian	hye_Armn
+Igbo	ibo_Latn
+Ilocano	ilo_Latn
+Indonesian	ind_Latn
+Icelandic	isl_Latn
+Italian	ita_Latn
+Javanese	jav_Latn
+Japanese	jpn_Jpan
+Kabyle	kab_Latn
+Jingpho	kac_Latn
+Kamba	kam_Latn
+Kannada	kan_Knda
+Kashmiri (Arabic script)	kas_Arab
+Kashmiri (Devanagari script)	kas_Deva
+Georgian	kat_Geor
+Central Kanuri (Arabic script)	knc_Arab
+Central Kanuri (Latin script)	knc_Latn
+Kazakh	kaz_Cyrl
+Kabiyè	kbp_Latn
+Kabuverdianu	kea_Latn
+Khmer	khm_Khmr
+Kikuyu	kik_Latn
+Kinyarwanda	kin_Latn
+Kyrgyz	kir_Cyrl
+Kimbundu	kmb_Latn
+Northern Kurdish	kmr_Latn
+Kikongo	kon_Latn
+Korean	kor_Hang
+Lao	lao_Laoo
+Ligurian	lij_Latn
+Limburgish	lim_Latn
+Lingala	lin_Latn
+Lithuanian	lit_Latn
+Lombard	lmo_Latn
+Latgalian	ltg_Latn
+Luxembourgish	ltz_Latn
+Luba-Kasai	lua_Latn
+Ganda	lug_Latn
+Luo	luo_Latn
+Mizo	lus_Latn
+Standard Latvian	lvs_Latn
+Magahi	mag_Deva
+Maithili	mai_Deva
+Malayalam	mal_Mlym
+Marathi	mar_Deva
+Minangkabau (Arabic script)	min_Arab
+Minangkabau (Latin script)	min_Latn
+Macedonian	mkd_Cyrl
+Plateau Malagasy	plt_Latn
+Maltese	mlt_Latn
+Meitei (Bengali script)	mni_Beng
+Halh Mongolian	khk_Cyrl
+Mossi	mos_Latn
+Maori	mri_Latn
+Burmese	mya_Mymr
+Dutch	nld_Latn
+Norwegian Nynorsk	nno_Latn
+Norwegian Bokmål	nob_Latn
+Nepali	npi_Deva
+Northern Sotho	nso_Latn
+Nuer	nus_Latn
+Nyanja	nya_Latn
+Occitan	oci_Latn
+West Central Oromo	gaz_Latn
+Odia	ory_Orya
+Pangasinan	pag_Latn
+Eastern Panjabi	pan_Guru
+Papiamento	pap_Latn
+Western Persian	pes_Arab
+Polish	pol_Latn
+Portuguese	por_Latn
+Dari	prs_Arab
+Southern Pashto	pbt_Arab
+Ayacucho Quechua	quy_Latn
+Romanian	ron_Latn
+Rundi	run_Latn
+Russian	rus_Cyrl
+Sango	sag_Latn
+Sanskrit	san_Deva
+Santali	sat_Olck
+Sicilian	scn_Latn
+Shan	shn_Mymr
+Sinhala	sin_Sinh
+Slovak	slk_Latn
+Slovenian	slv_Latn
+Samoan	smo_Latn
+Shona	sna_Latn
+Sindhi	snd_Arab
+Somali	som_Latn
+Southern Sotho	sot_Latn
+Spanish	spa_Latn
+Tosk Albanian	als_Latn
+Sardinian	srd_Latn
+Serbian	srp_Cyrl
+Swati	ssw_Latn
+Sundanese	sun_Latn
+Swedish	swe_Latn
+Swahili	swh_Latn
+Silesian	szl_Latn
+Tamil	tam_Taml
+Tatar	tat_Cyrl
+Telugu	tel_Telu
+Tajik	tgk_Cyrl
+Tagalog	tgl_Latn
+Thai	tha_Thai
+Tigrinya	tir_Ethi
+Tamasheq (Latin script)	taq_Latn
+Tamasheq (Tifinagh script)	taq_Tfng
+Tok Pisin	tpi_Latn
+Tswana	tsn_Latn
+Tsonga	tso_Latn
+Turkmen	tuk_Latn
+Tumbuka	tum_Latn
+Turkish	tur_Latn
+Twi	twi_Latn
+Central Atlas Tamazight	tzm_Tfng
+Uyghur	uig_Arab
+Ukrainian	ukr_Cyrl
+Umbundu	umb_Latn
+Urdu	urd_Arab
+Northern Uzbek	uzn_Latn
+Venetian	vec_Latn
+Vietnamese	vie_Latn
+Waray	war_Latn
+Wolof	wol_Latn
+Xhosa	xho_Latn
+Eastern Yiddish	ydd_Hebr
+Yoruba	yor_Latn
+Yue Chinese	yue_Hant
+Chinese (Simplified)	zho_Hans
+Chinese (Traditional)	zho_Hant
+Standard Malay	zsm_Latn
+Zulu	zul_Latn'''
+
+codes_as_string = codes_as_string.split('\n')
+
+flores_codes = {}
+for code in codes_as_string:
+    lang, lang_code = code.split('\t')
+    flores_codes[lang] = lang_code

modules/lid218e.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ded5749a2ad79ae9ab7c9190c7c8b97ff20d54ad8b9527ffa50107238fc7f6a
+size 1176355829

modules/sematrans-1.2B/config.json

@@ -0,0 +1,8 @@
+{
+  "add_source_bos": false,
+  "add_source_eos": false,
+  "bos_token": "<s>",
+  "decoder_start_token": "</s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

modules/sematrans-1.2B/model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd041a40b02d6e3343b174e84ad75c2a058c3a60951dca6bbb4aa9334e7b9ddf
+size 1381827064

modules/sematrans-3.3B/config.json

@@ -0,0 +1,8 @@
+{
+  "add_source_bos": false,
+  "add_source_eos": false,
+  "bos_token": "<s>",
+  "decoder_start_token": "</s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

modules/sematrans-3.3B/model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cacb7f6f36b75d73549ab9d99d0bb95270ef65dabea07204adc5648c68274ae8
+size 3363404280

modules/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a
+size 4852054

modules/translator_all_langs.tsv

@@ -0,0 +1,204 @@
+Acehnese-(Arabic script) ace_Arab
+Acehnese-(Latin script) ace_Latn
+Mesopotamian-(Arabic) acm_Arab
+Ta’izzi-Adeni-(Arabic) acq_Arab
+Tunisian-(Arabic) aeb_Arab
+Afrikaans afr_Latn
+South_Levantine_Arabic ajp_Arab
+Akan aka_Latn
+Amharic amh_Ethi
+North_Levantine_Arabic apc_Arab
+Modern_Standard_Arabic arb_Arab
+Modern_Standard_Arabic_(Romanized) arb_Latn
+Najdi_Arabic ars_Arab
+Moroccan_Arabic ary_Arab
+Egyptian_Arabic arz_Arab
+Assamese asm_Beng
+Asturian ast_Latn
+Awadhi awa_Deva
+Central_Aymara ayr_Latn
+South_Azerbaijani azb_Arab
+North_Azerbaijani azj_Latn
+Bashkir bak_Cyrl
+Bambara bam_Latn
+Balinese ban_Latn
+Belarusian bel_Cyrl
+Bemba bem_Latn
+Bengali ben_Beng
+Bhojpuri bho_Deva
+Banjar_(Arabic script) bjn_Arab
+Banjar_(Latin script) bjn_Latn
+Standard_Tibetan bod_Tibt
+Bosnian bos_Latn
+Buginese bug_Latn
+Bulgarian bul_Cyrl
+Catalan cat_Latn
+Cebuano ceb_Latn
+Czech ces_Latn
+Chokwe cjk_Latn
+Central_Kurdish ckb_Arab
+Crimean_Tatar crh_Latn
+Welsh cym_Latn
+Danish dan_Latn
+German deu_Latn
+Southwestern_Dinka dik_Latn
+Dyula dyu_Latn
+Dzongkha dzo_Tibt
+Greek ell_Grek
+English eng_Latn
+Esperanto epo_Latn
+Estonian est_Latn
+Basque eus_Latn
+Ewe ewe_Latn
+Faroese fao_Latn
+Fijian fij_Latn
+Finnish fin_Latn
+Fon fon_Latn
+French fra_Latn
+Friulian fur_Latn
+Nigerian_Fulfulde fuv_Latn
+Scottish_Gaelic gla_Latn
+Irish gle_Latn
+Galician glg_Latn
+Guarani grn_Latn
+Gujarati guj_Gujr
+Haitian_Creole hat_Latn
+Hausa hau_Latn
+Hebrew heb_Hebr
+Hindi hin_Deva
+Chhattisgarhi hne_Deva
+Croatian hrv_Latn
+Hungarian hun_Latn
+Armenian hye_Armn
+Igbo ibo_Latn
+Ilocano ilo_Latn
+Indonesian ind_Latn
+Icelandic isl_Latn
+Italian ita_Latn
+Javanese jav_Latn
+Japanese jpn_Jpan
+Kabyle kab_Latn
+Jingpho kac_Latn
+Kamba kam_Latn
+Kannada kan_Knda
+Kashmiri_(Arabic script) kas_Arab
+Kashmiri_(Devanagari script) kas_Deva
+Georgian kat_Geor
+Central_Kanuri_(Arabic script) knc_Arab
+Central_Kanuri_(Latin script) knc_Latn
+Kazakh kaz_Cyrl
+Kabiyè kbp_Latn
+Kabuverdianu kea_Latn
+Khmer khm_Khmr
+Kikuyu kik_Latn
+Kinyarwanda kin_Latn
+Kyrgyz kir_Cyrl
+Kimbundu kmb_Latn
+Northern_Kurdish kmr_Latn
+Kikongo kon_Latn
+Korean kor_Hang
+Lao lao_Laoo
+Ligurian lij_Latn
+Limburgish lim_Latn
+Lingala lin_Latn
+Lithuanian lit_Latn
+Lombard lmo_Latn
+Latgalian ltg_Latn
+Luxembourgish ltz_Latn
+Luba-Kasai lua_Latn
+Ganda lug_Latn
+Luo luo_Latn
+Mizo lus_Latn
+Standard-Latvian lvs_Latn
+Magahi mag_Deva
+Maithili mai_Deva
+Malayalam mal_Mlym
+Marathi mar_Deva
+Minangkabau-(Arabic script) min_Arab
+Minangkabau-(Latin script) min_Latn
+Macedonian mkd_Cyrl
+Plateau-Malagasy plt_Latn
+Maltese mlt_Latn
+Meitei-(Bengali script) mni_Beng
+Halh-Mongolian khk_Cyrl
+Mossi mos_Latn
+Maori mri_Latn
+Burmese mya_Mymr
+Dutch nld_Latn
+Norwegian-Nynorsk nno_Latn
+Norwegian-Bokmål nob_Latn
+Nepali npi_Deva
+Northern-Sotho nso_Latn
+Nuer nus_Latn
+Nyanja nya_Latn
+Occitan oci_Latn
+West-Central-Oromo gaz_Latn
+Odia ory_Orya
+Pangasinan pag_Latn
+Eastern-Panjabi pan_Guru
+Papiamento pap_Latn
+Western-Persian pes_Arab
+Polish pol_Latn
+Portuguese por_Latn
+Dari prs_Arab
+Southern-Pashto pbt_Arab
+Ayacucho-Quechua quy_Latn
+Romanian ron_Latn
+Rundi run_Latn
+Russian rus_Cyrl
+Sango sag_Latn
+Sanskrit san_Deva
+Santali sat_Olck
+Sicilian scn_Latn
+Shan shn_Mymr
+Sinhala sin_Sinh
+Slovak slk_Latn
+Slovenian slv_Latn
+Samoan smo_Latn
+Shona sna_Latn
+Sindhi snd_Arab
+Somali som_Latn
+Southern-Sotho sot_Latn
+Spanish spa_Latn
+Tosk-Albanian als_Latn
+Sardinian srd_Latn
+Serbian srp_Cyrl
+Swati ssw_Latn
+Sundanese sun_Latn
+Swedish swe_Latn
+Swahili swh_Latn
+Silesian szl_Latn
+Tamil tam_Taml
+Tatar tat_Cyrl
+Telugu tel_Telu
+Tajik tgk_Cyrl
+Tagalog tgl_Latn
+Thai tha_Thai
+Tigrinya tir_Ethi
+Tamasheq-(Latin script) taq_Latn
+Tamasheq-(Tifinagh script) taq_Tfng
+Tok-Pisin tpi_Latn
+Tswana tsn_Latn
+Tsonga tso_Latn
+Turkmen tuk_Latn
+Tumbuka tum_Latn
+Turkish tur_Latn
+Twi twi_Latn
+Central-Atlas-Tamazight tzm_Tfng
+Uyghur uig_Arab
+Ukrainian ukr_Cyrl
+Umbundu umb_Latn
+Urdu urd_Arab
+Northern-Uzbek uzn_Latn
+Venetian vec_Latn
+Vietnamese vie_Latn
+Waray war_Latn
+Wolof wol_Latn
+Xhosa xho_Latn
+Eastern-Yiddish ydd_Hebr
+Yoruba yor_Latn
+Yue Chinese yue_Hant
+Chinese-(Simplified) zho_Hans
+Chinese-(Traditional) zho_Hant
+Standard-Malay zsm_Latn
+Zulu zul_Latn

requirements.txt

@@ -0,0 +1,13 @@
+huggingface_hub
+ctranslate2 
+sentencepiece 
+accelerate 
+bitsandbytes 
+transformers 
+requests 
+pypi-json 
+pytz
+fasttext
+gradio
+fastapi 
+uvicorn[standard]==0.17.*

static/index.html

@@ -0,0 +1,52 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width">
+    <title>Sema Translator</title>
+    <link rel="stylesheet" href="static/style.css" />
+    <py-script src="modules/flores200_codes.py"></py-script>
+</head>
+<body>
+    <div class="Header">
+        <h1>Sema Translator</h1>
+        <p>Unlock the Power of Global Communication with Sema Translator! Seamlessly bridging language barriers. With support for over 200 languages, Sema Translator opens up a realm of possibilities for building truly global applications.</p>
+        
+    </div>
+    
+    <div class="LanguageDropdown" style="text-align:center">
+            <h2>Select a Language:</h2>
+            <select id="languageSelect">
+                {"".join(f"<option value='{code}'>{lang} - {code}</option>" for lang, code in flores_codes.items())}
+            </select>
+    </div>
+    
+    <div class="instructions">
+        <p>Use the following python code to access the api endpoint</p>
+        <pre style="text-align: left;">
+import requests
+
+url = "{public_url}/translate/"
+data = {
+    "userinput": "rũcinĩ rwega, niwokĩra wega?",
+    "target_lang": "eng_Latn",
+}
+
+response = requests.post(url, json=data)
+result = response.json()
+
+print(result)
+
+source_language = result['source_language']
+print("Source Language:", source_language)
+
+translation = result['translated_text']
+print("Translated text:", translation)
+        </pre>
+    </div>
+    
+    <div class="footer">
+        <h1>Created by Lewis Kamau Kiamru</h1>
+    </div>
+</body>
+</html>

static/style.css

@@ -0,0 +1,13 @@
+        .Header {
+            text-align: center;
+        }
+        .LanguageDropdown {
+            text-align: center;
+        }
+        .instructions {
+            background-color: #056e33;
+            padding: 20px;
+        }
+        .footer {
+            text-align: center;
+        }