Spelling

app.py

@@ -7,6 +7,53 @@ tdk1 = pipeline('text-generation', model='notexist/tttf', tokenizer=tokenizer1)
 tokenizer2 = AutoTokenizer.from_pretrained("notexist/ttte")
 tdk2 = pipeline('text-generation', model='notexist/ttte', tokenizer=tokenizer2)
 
+
+# Speller borrowed from http://turkceddi.blogspot.com/
+lower_vowel = {'a', 'â', 'e', 'ê', 'ı', 'î', 'i', 'o', 'ô', 'ö', 'u', 'û', 'ü'}
+SPELL_SLICER = (('001000', 5), ('000100', 5), ('01000', 4), ('00100', 4), ('00010', 4), ('1000', 3), ('0100', 3),
+                ('0011', 3), ('0010', 3), ('011', 2), ('010', 2), ('100', 2), ('10', 1), ('11', 1))
+
+
+def to_lower(word):
+    tolower_text = (word.replace('İ', 'i'))
+    tolower_text = (tolower_text.replace('I', 'ı'))
+    return tolower_text.lower()
+
+
+def wordtoten(word: str):
+    wtt = ''
+
+    for ch in word:
+        if ch in lower_vowel:
+            wtt += '1'
+        else:
+            wtt += '0'
+    return wtt
+
+def spellword(word: str):
+    word = to_lower(word)
+    syllable_list = []
+    tenword = wordtoten(word)
+    len_spell = tenword.count('1')
+
+    for i in range(tenword.count('1')):
+        for x, y in SPELL_SLICER:
+            if tenword.startswith(x):
+                syllable_list.append(word[:y])
+                word = word[y:]
+                tenword = tenword[y:]
+                break
+
+    if tenword == '0':
+        syllable_list[-1] = syllable_list[-1] + word
+    elif word:
+        syllable_list.append(word)
+
+    if len(syllable_list) != len_spell:
+        return False
+
+    return "$".join(syllable_list)
+
 def predict(name, sl, topk, topp):
     if name == "":
         x1 = tdk1(f"<|endoftext|>",
@@ -28,9 +75,9 @@ def predict(name, sl, topk, topp):
         )[0]["generated_text"]
         
         if "[TEXT]" not in x2:
-            return x1[len(f"<|endoftext|>"):]   
+            return x1[len(f"<|endoftext|>"):]+"\n\n"+spellword(new_name)
         else:
-            return x1[len(f"<|endoftext|>"):]+"\n\n"+x2[len(f"<|endoftext|>{new_name}\n\n"):].replace("[TEXT]", " "+new_name+" ")
+            return x1[len(f"<|endoftext|>"):]+"\n\n"+spellword(new_name)+"\n\n"+x2[len(f"<|endoftext|>{new_name}\n\n"):].replace("[TEXT]", " "+new_name+" ")
     else:
         x1 = tdk1(f"<|endoftext|>{name}\n\n",
             do_sample=True, 
@@ -50,9 +97,9 @@ def predict(name, sl, topk, topp):
         )[0]["generated_text"]
 
         if "[TEXT]" not in x2:
-            return x1[len(f"<|endoftext|>{name}\n\n"):]
+            return x1[len(f"<|endoftext|>{name}\n\n"):]+"\n\n"+spellword(name)
         else:
-            return x1[len(f"<|endoftext|>{name}\n\n"):]+"\n\n"+x2[len(f"<|endoftext|>{name}\n\n"):].replace("[TEXT]", " "+name+" ")
+            return x1[len(f"<|endoftext|>{name}\n\n"):]+"\n\n"+spellword(name)+"\n\n"+x2[len(f"<|endoftext|>{name}\n\n"):].replace("[TEXT]", " "+name+" ")