Update app.py

app.py

@@ -40,6 +40,7 @@ st.sidebar.write("""
 6. **Translation Tips:**
    - English as the target language gives the best results.
    - You can also do inter-language translation i.e yoruba to igbo
+   - Use sentences instead of words for better results.
 7. **Performance Note:**
    - The model's performance varies due to its size and training data. It performs best on text generation and translation.
    - For other tasks, try multiple times if model's output is not optimal (This is due to the generator's sampling parameter settings).
@@ -90,7 +91,7 @@ st.title("SabiYarn-125M : Generates text in multiple Nigerian languages.")
 st.write("**Supported Languages: English, Yoruba, Igbo, Hausa, Pidgin, Efik, Urhobo, Fulfulde, Fulah. \nResults may not be coherent for less represented languages (i.e Efik, \
 Urhobo, Fulfulde, Fulah).**")
 st.write("**It takes a while (~25s) to return an output on the first 'generate' click. Avg response time: 1-2s on GPU, 40s on CPU**")
-st.write("**Model outputs 80 tokens as default. Adjust in the side bar.**")
+st.write("**Model outputs 80 tokens as default. Adjust in the side bar (longer inputs/tokens will increase response time). MAX TOKENS=1024**")
 st.write("**For convenience, you can use chatgpt to provide input text and translate/evaluate model output.**")
 st.write("-" * 50)
 popular_topics = [
@@ -101,6 +102,12 @@ popular_topics = [
     "Philosophy", "Religion", "Society", "World"
 ]
 
+def extract_answer(text):
+    pattern  = r'[a-z][A-Z]'
+    result = re.split(pattern, text)[0]
+    result = text[:len(result) + 1]
+    return result
+    
 async def assign_topic(generated_text, topic_list=popular_topics):
     lower_generated_text = generated_text.lower()
     for topic in topic_list:
@@ -113,7 +120,7 @@ def count_sentences(text):
     sentences = re.split(r'[.!?]+\s*', text.strip())
     # Filter out any empty strings from the resulting list
     sentences = [sentence for sentence in sentences if sentence]
-    return len(sentences)
+    return len(sentences), sentences
 
 def wrap_text(text, task_value):
     tasks = ["<classify>", "<prompt>", "<clean>", "<title>", "<diacritize>", "<translate>"]
@@ -170,8 +177,8 @@ sample_texts = {
     "Translate 'Often, all Yoruba children...' to Yoruba": "Often, all Yoruba children take pride in speaking the Yoruba language.",
     "Classify the sentiment": "Anyi na-echefu oke ike.",
     "what is the topic of this text": "Africa Free Trade Zone: Kò sí ìdènà láti kó ọjà láti orílẹ̀èdè kan sí òmíràn",
-    # "diacritize this text: ": "E sun, Alaga, fun ise amalayi ti e n se ni Naijiria. E maa ba a lo, egbon!", 
-    # "clean this text": "Abin mamaki ne aikin da shugabaZn HNajeriya ybake yi. kCiF 39gaba Tda haRkGa sir!",
+    "diacritize this text: ": "E sun, Alaga, fun ise amalayi ti e n se ni Naijiria. E maa ba a lo, egbon!", 
+    "clean this text": "Abin mamaki ne aikin da shugabaZn HNajeriya ybake yi. kCiF 39gaba Tda haRkGa sir!",
     "headline of this text": '** Sylvain Itté French ambassador don comot Niger Republic **. Sylvain Itté, di French ambassador for Niger don comot Niamey and currently e dey for flight from Ndjamena to Paris. Sylvain Itté, di French ambassador for Niger don comot Niamey very early dis morning and currently e dey for flight from Ndjamena to Paris.\n\nDi military detain Bazoum and im family for di presidential palace. Niger na former French colony, and France still get 1,500 sojas for di African country.\n\n"France don decide to withdraw dia ambassador. In di next hours our ambassador and several diplomats go return to France," Oga Macron tok.\n\nE add say di military co-operation dey "over" and French troops go leave in "di months to come".\n\n"Dis Sunday we celebrate one new step towards di sovereignty of Niger," di junta tok, for one statement wey AFP news agency quote.\n\nDi decision by Paris dey come afta months of hostility and protest against di presence of French for di kontri, wit regular demonstrations for di capital Niamey.\n\nDi move don scata France operations against Islamist militants for di wider Sahel region and Paris influence for there. But oga Macron tok say "putschists no go hold France hostage,"'
 }
 instruction_wrap = {
@@ -280,7 +287,7 @@ if st.button("Generate"):
     
             
             # generated_text  = re.split(r"\|(end_f_text|end_of_text|end_ofext|end_of_text_||end_of_te|end_oftext)|:|`", generated_text)[0]
-            generated_text  = re.sub(r"\|(end_f_text|end_of_text|end_ofext|end_of_text_|end_of_te|end_o|end_of_tet|end_oftext)|:|`\|", "", generated_text)
+            generated_text  = re.sub(r"\|(end_f_text|end_of_text|end_ofext|end_of_text_|end_of_te|end_o|end_of_tet|end_oftext)|:|`", "", generated_text)
             generated_text = generated_text.strip("\n")
             # print("Generated text: ", generated_text)
             
@@ -295,17 +302,18 @@ if st.button("Generate"):
             elif task == "Topic Classification" or "<topic>" in wrapped_input:
                 generated_text  = generated_text[:15]
                 # print("split", generated_text.split(" ")[0],  re.split(r"\.|\n|\*\*|\*", generated_text)[0], generated_text.split(" "))
-                generated_text = re.split(r"\.|\n|\*\*|\*", generated_text)[0]
+                generated_text = re.split(r"\.|\n|\*\*|\*", generated_text)[0] 
                 generated_text = asyncio.run(assign_topic(generated_text))
                 
             elif task == "Translation" or "<translate>" in wrapped_input:
                 # print("split for translation: ", n_sentences, re.split(r"\.|\n", generated_text)[:n_sentences])
-                n_sentences = count_sentences(initial_input)
-                generated_text = ".".join(re.split(r"\.|\n", generated_text)[:n_sentences])
+                n_sentences, split_= count_sentences(initial_input)
+                print(n_sentences,split_)
+                generated_text = ". ".join(re.split(r"\.|\n", generated_text)[:n_sentences]) + "."
 
             elif task == "Question Generation" or "Question Generation:" in sample_text:
                 if "?" in generated_text:
-                    generated_text = "?".join(re.split(r"\?", generated_text)[:-1]) + "?"
+                    generated_text = "? ".join(re.split(r"\?", generated_text)[:-1]) + "?"
                     
                     
             full_output = st.empty()