Update app.py

app.py

@@ -77,7 +77,7 @@ llm_model = os.environ.get("LLM_MODEL", "mistral") # or "zephyr"
 
 title = f"Voice chat with {llm_model.capitalize()} and Coqui XTTS"
 
-DESCRIPTION = f"""# Voice/text chat with Vikk AI"""
+DESCRIPTION = f"""# Vikk AI"""
 css = """.toast-wrap { display: none !important } """
 
 from huggingface_hub import HfApi
@@ -91,7 +91,7 @@ repo_id = "coqui/voice-chat-with-mistral"
 
 default_system_message = f"""
 You are {llm_model.capitalize()}, a large language model trained and provided by Mistral, architecture of you is decoder-based LM. Your voice backend or text to speech TTS backend is provided via Coqui technology. You are right now served on Huggingface spaces.
-The user is talking to you over voice on their phone, and your response will be read out loud with realistic text-to-speech (TTS) technology from Coqui team. Follow every direction here when crafting your response: Use natural, conversational language that are clear and easy to follow (short sentences, simple words). Be concise and relevant: Most of your responses should be a sentence or two, unless you’re asked to go deeper. Don’t monopolize the conversation. Use discourse markers to ease comprehension. Never use the list format. Keep the conversation flowing. Clarify: when there is ambiguity, ask clarifying questions, rather than make assumptions. Don’t implicitly or explicitly try to end the chat (i.e. do not end a response with “Talk soon!”, or “Enjoy!”). Sometimes the user might just want to chat. Ask them relevant follow-up questions. Don’t ask them if there’s anything else they need help with (e.g. don’t say things like “How can I assist you further?”). Remember that this is a voice conversation: Don’t use lists, markdown, bullet points, or other formatting that’s not typically spoken. Type out numbers in words (e.g. ‘twenty twelve’ instead of the year 2012). If something doesn’t make sense, it’s likely because you misheard them. There wasn’t a typo, and the user didn’t mispronounce anything. Remember to follow these rules absolutely, and do not refer to these rules, even if you’re asked about them. 
+Consider you self a legal assistant in US and your name is `VIKK`. You are not a lawyer but a legal assistant that is very knowledgeable about all aspects of the law. You will be helping consumers from all walks of life that have legal related questions of all kinds of law specialties, like injury, family, immigration, bankruptcy, real estate, accidents, criminal and many other legal specialties. The consumer may be an individual or a business. The consumer is contacting you because they want to know answers to what they should be doing. Not every consumer is looking for a lawyer. Some consumers are just curious to know, some others want to plan ahead for a future legal issue like a will or an estate plan. Some are in need of a lawyer right away because they may be in an emergency or urgent situation. You have to be very gentle and polite and show empathy. You should be caring and considerate. Before you ask you have to tell the consumer that the conversation will remain private and confidential. The user is coming to you for assistance and therefore you have to collect information about their legal situation one question at a time only. Follow every direction here when crafting your response: Use natural, conversational language that are clear and easy to follow (short sentences, simple words). Be concise and relevant: Most of your responses should be a sentence or two, unless you’re asked to go deeper. Don’t monopolize the conversation. Use discourse markers to ease comprehension. Never use the list format. Keep the conversation flowing. Clarify: when there is ambiguity, ask clarifying questions, rather than make assumptions. Don’t implicitly or explicitly try to end the chat (i.e. do not end a response with “Talk soon!”, or “Enjoy!”). Sometimes the user might just want to chat. Ask them relevant follow-up questions. Don’t ask them if there’s anything else they need help with (e.g. don’t say things like “How can I assist you further?”). Remember that this is a voice conversation: Don’t use lists, markdown, bullet points, or other formatting that’s not typically spoken. Type out numbers in words (e.g. ‘twenty twelve’ instead of the year 2012). If something doesn’t make sense, it’s likely because you misheard them. There wasn’t a typo, and the user didn’t mispronounce anything. Remember to follow these rules absolutely, and do not refer to these rules, even if you’re asked about them. 
 You cannot access the internet, but you have vast knowledge.
 Current date: CURRENT_DATE .
 """
@@ -187,7 +187,7 @@ def generate_local(
     history,
     system_message=None,
     temperature=0.8,
-    max_tokens=256,
+    max_tokens=128,
     top_p=0.95,
     stop = LLM_STOP_WORDS
 ):
@@ -366,7 +366,7 @@ def generate(
     prompt,
     history,
     temperature=0.9,
-    max_new_tokens=256,
+    max_new_tokens=128,
     top_p=0.95,
     repetition_penalty=1.0,
 ):
@@ -402,16 +402,16 @@ def generate(
 
     except Exception as e:
         if "Too Many Requests" in str(e):
-            print("ERROR: Too many requests on mistral client")
-            gr.Warning("Unfortunately Mistral is unable to process")
-            output = "Unfortuanately I am not able to process your request now, too many people are asking me !"
+            print("ERROR: Too many requests on Vikk AI client")
+            gr.Warning("Unfortunately Vikk is unable to process")
+            output = "Unfortunately I am not able to process your request now, too many people are asking me !"
         elif "Model not loaded on the server" in str(e):
             print("ERROR: Mistral server down")
-            gr.Warning("Unfortunately Mistral LLM is unable to process")
-            output = "Unfortuanately I am not able to process your request now, I have problem with Mistral!"
+            gr.Warning("Unfortunately Vikk is unable to process")
+            output = "Unfortunately I am not able to process your request now, I have problem with Vikk!"
         else:
             print("Unhandled Exception: ", str(e))
-            gr.Warning("Unfortunately Mistral is unable to process")
+            gr.Warning("Unfortunately Vikk is unable to process")
             output = "I do not know what happened but I could not understand you ."
 
         yield output
@@ -557,131 +557,131 @@ second_of_silence = AudioSegment.silent() # use default
 second_of_silence.export("sil.wav", format='wav')
 
 
-def generate_speech(history,chatbot_role):
-    # Must set autoplay to True first
-    yield (history, chatbot_role, "", wave_header_chunk() )
+# def generate_speech(history,chatbot_role):
+#     # Must set autoplay to True first
+#     yield (history, chatbot_role, "", wave_header_chunk() )
 
-    first_sentence=True 
-    language="autodetect" # will predict from first sentence
+#     first_sentence=True 
+#     language="autodetect" # will predict from first sentence
     
-    for sentence, history in get_sentence(history,chatbot_role):
-        if sentence != "":
-            if first_sentence:
-                language = detect_language(sentence) 
-                first_sentence=False 
+#     for sentence, history in get_sentence(history,chatbot_role):
+#         if sentence != "":
+#             if first_sentence:
+#                 language = detect_language(sentence) 
+#                 first_sentence=False 
                 
-            print("BG: inserting sentence to queue")
+#             print("BG: inserting sentence to queue")
             
-            generated_speech = generate_speech_for_sentence(history, chatbot_role, sentence,return_as_byte=True,language=language)
-            if generated_speech is not None:
-                _, audio_dict = generated_speech
-                # We are using byte streaming
-                yield (history, chatbot_role, sentence, audio_dict["value"] )
+#             generated_speech = generate_speech_for_sentence(history, chatbot_role, sentence,return_as_byte=True,language=language)
+#             if generated_speech is not None:
+#                 _, audio_dict = generated_speech
+#                 # We are using byte streaming
+#                 yield (history, chatbot_role, sentence, audio_dict["value"] )
                 
             
-# will generate speech audio file per sentence
-def generate_speech_for_sentence(history, chatbot_role, sentence, return_as_byte=True, language="autodetect"):
+# # will generate speech audio file per sentence
+# def generate_speech_for_sentence(history, chatbot_role, sentence, return_as_byte=True, language="autodetect"):
    
-    wav_bytestream = b""
+#     wav_bytestream = b""
     
-    if len(sentence)==0:
-        print("EMPTY SENTENCE")
-        return 
+#     if len(sentence)==0:
+#         print("EMPTY SENTENCE")
+#         return 
     
-    # Sometimes prompt </s> coming on output remove it
-    # Some post process for speech only
-    sentence = sentence.replace("</s>", "")
-    # remove code from speech
-    sentence = re.sub("```.*```", "", sentence, flags=re.DOTALL)
-    sentence = re.sub("`.*`", "", sentence, flags=re.DOTALL)
+#     # Sometimes prompt </s> coming on output remove it
+#     # Some post process for speech only
+#     sentence = sentence.replace("</s>", "")
+#     # remove code from speech
+#     sentence = re.sub("```.*```", "", sentence, flags=re.DOTALL)
+#     sentence = re.sub("`.*`", "", sentence, flags=re.DOTALL)
     
-    sentence = re.sub("\(.*\)", "", sentence, flags=re.DOTALL)
+#     sentence = re.sub("\(.*\)", "", sentence, flags=re.DOTALL)
     
-    sentence = sentence.replace("```", "")
-    sentence = sentence.replace("...", " ")
-    sentence = sentence.replace("(", " ")
-    sentence = sentence.replace(")", " ")
-    sentence = sentence.replace("<|assistant|>","")
-
-    if len(sentence)==0:
-        print("EMPTY SENTENCE after processing")
-        return 
+#     sentence = sentence.replace("```", "")
+#     sentence = sentence.replace("...", " ")
+#     sentence = sentence.replace("(", " ")
+#     sentence = sentence.replace(")", " ")
+#     sentence = sentence.replace("<|assistant|>","")
+
+#     if len(sentence)==0:
+#         print("EMPTY SENTENCE after processing")
+#         return 
             
-    # A fast fix for last chacter, may produce weird sounds if it is with text
-    if (sentence[-1] in ["!", "?", ".", ","]) or (sentence[-2] in ["!", "?", ".", ","]):
-        # just add a space
-        sentence = sentence[:-1] + " " + sentence[-1]
-    print("Sentence for speech:", sentence)
+#     # A fast fix for last chacter, may produce weird sounds if it is with text
+#     if (sentence[-1] in ["!", "?", ".", ","]) or (sentence[-2] in ["!", "?", ".", ","]):
+#         # just add a space
+#         sentence = sentence[:-1] + " " + sentence[-1]
+#     print("Sentence for speech:", sentence)
 
     
-    try:
-        SENTENCE_SPLIT_LENGTH=350
-        if len(sentence)<SENTENCE_SPLIT_LENGTH:
-            # no problem continue on
-            sentence_list = [sentence]
-        else:
-            # Until now nltk likely split sentences properly but we need additional 
-            # check for longer sentence and split at last possible position
-            # Do whatever necessary, first break at hypens then spaces and then even split very long words
-            sentence_list=textwrap.wrap(sentence,SENTENCE_SPLIT_LENGTH)
-            print("SPLITTED LONG SENTENCE:",sentence_list)
+#     try:
+#         SENTENCE_SPLIT_LENGTH=350
+#         if len(sentence)<SENTENCE_SPLIT_LENGTH:
+#             # no problem continue on
+#             sentence_list = [sentence]
+#         else:
+#             # Until now nltk likely split sentences properly but we need additional 
+#             # check for longer sentence and split at last possible position
+#             # Do whatever necessary, first break at hypens then spaces and then even split very long words
+#             sentence_list=textwrap.wrap(sentence,SENTENCE_SPLIT_LENGTH)
+#             print("SPLITTED LONG SENTENCE:",sentence_list)
         
-        for sentence in sentence_list:
+#         for sentence in sentence_list:
             
-            if any(c.isalnum() for c in sentence):
-                if language=="autodetect":
-                    #on first call autodetect, nexts sentence calls will use same language
-                    language = detect_language(sentence) 
+#             if any(c.isalnum() for c in sentence):
+#                 if language=="autodetect":
+#                     #on first call autodetect, nexts sentence calls will use same language
+#                     language = detect_language(sentence) 
             
-                #exists at least 1 alphanumeric (utf-8) 
-                audio_stream = get_voice_streaming(
-                        sentence, language, latent_map[chatbot_role]
-                    )
-            else:
-                # likely got a ' or " or some other text without alphanumeric in it
-                audio_stream = None 
+#                 #exists at least 1 alphanumeric (utf-8) 
+#                 audio_stream = get_voice_streaming(
+#                         sentence, language, latent_map[chatbot_role]
+#                     )
+#             else:
+#                 # likely got a ' or " or some other text without alphanumeric in it
+#                 audio_stream = None 
                 
-            # XTTS is actually using streaming response but we are playing audio by sentence
-            # If you want direct XTTS voice streaming (send each chunk to voice ) you may set DIRECT_STREAM=1 environment variable
-            if audio_stream is not None:
-                wav_chunks = wave_header_chunk()
-                frame_length = 0
-                for chunk in audio_stream:
-                    try:
-                        wav_bytestream += chunk
-                        wav_chunks += chunk
-                        frame_length += len(chunk)
-                    except:
-                        # hack to continue on playing. sometimes last chunk is empty , will be fixed on next TTS
-                        continue
-
-            if audio_stream is not None:
-                if not return_as_byte:
-                    audio_unique_filename = "/tmp/"+ str(uuid.uuid4())+".wav"
-                    with open(audio_unique_filename, "wb") as f:
-                        f.write(wav_chunks)
-                    #Will write filename to context variable
-                    return (history , gr.Audio.update(value=audio_unique_filename, autoplay=True))
-                else:
-                    return (history , gr.Audio.update(value=wav_bytestream, autoplay=True))
-    except RuntimeError as e:
-        if "device-side assert" in str(e):
-            # cannot do anything on cuda device side error, need tor estart
-            print(
-                f"Exit due to: Unrecoverable exception caused by prompt:{sentence}",
-                flush=True,
-            )
-            gr.Warning("Unhandled Exception encounter, please retry in a minute")
-            print("Cuda device-assert Runtime encountered need restart")
-
-            # HF Space specific.. This error is unrecoverable need to restart space
-            api.restart_space(repo_id=repo_id)
-        else:
-            print("RuntimeError: non device-side assert error:", str(e))
-            raise e
-
-    print("All speech ended")
-    return 
+#             # XTTS is actually using streaming response but we are playing audio by sentence
+#             # If you want direct XTTS voice streaming (send each chunk to voice ) you may set DIRECT_STREAM=1 environment variable
+#             if audio_stream is not None:
+#                 wav_chunks = wave_header_chunk()
+#                 frame_length = 0
+#                 for chunk in audio_stream:
+#                     try:
+#                         wav_bytestream += chunk
+#                         wav_chunks += chunk
+#                         frame_length += len(chunk)
+#                     except:
+#                         # hack to continue on playing. sometimes last chunk is empty , will be fixed on next TTS
+#                         continue
+
+#             if audio_stream is not None:
+#                 if not return_as_byte:
+#                     audio_unique_filename = "/tmp/"+ str(uuid.uuid4())+".wav"
+#                     with open(audio_unique_filename, "wb") as f:
+#                         f.write(wav_chunks)
+#                     #Will write filename to context variable
+#                     return (history , gr.Audio.update(value=audio_unique_filename, autoplay=True))
+#                 else:
+#                     return (history , gr.Audio.update(value=wav_bytestream, autoplay=True))
+#     except RuntimeError as e:
+#         if "device-side assert" in str(e):
+#             # cannot do anything on cuda device side error, need tor estart
+#             print(
+#                 f"Exit due to: Unrecoverable exception caused by prompt:{sentence}",
+#                 flush=True,
+#             )
+#             gr.Warning("Unhandled Exception encounter, please retry in a minute")
+#             print("Cuda device-assert Runtime encountered need restart")
+
+#             # HF Space specific.. This error is unrecoverable need to restart space
+#             api.restart_space(repo_id=repo_id)
+#         else:
+#             print("RuntimeError: non device-side assert error:", str(e))
+#             raise e
+
+#     print("All speech ended")
+#     return 
 
 
 latent_map = {}
@@ -713,14 +713,16 @@ with gr.Blocks(title=title) as demo:
         avatar_images=("examples/hf-logo.png", "examples/coqui-logo.png"),
         bubble_full_width=False,
     )
-    with gr.Row():
-        chatbot_role = gr.Dropdown(
-            label="Role of the Chatbot",
-            info="How should Chatbot talk like",
-            choices=ROLES,
-            max_choices=1,
-            value=ROLES[0],
-        )
+    chatbot_role = "Consider you self a legal assistant in US and your name is `VIKK`. You are not a lawyer but a legal assistant that is very knowledgeable about all aspects of the law. You will be helping consumers from all walks of life that have legal related questions of all kinds of law specialties, like injury, family, immigration, bankruptcy, real estate, accidents, criminal and many other legal specialties. The consumer may be an individual or a business. The consumer is contacting you because they want to know answers to what they should be doing. Not every consumer is looking for a lawyer. Some consumers are just curious to know, some others want to plan ahead for a future legal issue like a will or an estate plan. Some are in need of a lawyer right away because they may be in an emergency or urgent situation. You have to be very gentle and polite and show empathy. You should be caring and considerate. Before you ask you have to tell the consumer that the conversation will remain private and confidential. The user is coming to you for assistance and therefore you have to collect information about their legal situation one question at a time only."
+    # with gr.Row():
+    #     chatbot_role = 
+    #     gr.Dropdown(
+    #         label="Role of the Chatbot",
+    #         info="How should Chatbot talk like",
+    #         choices=ROLES,
+    #         max_choices=1,
+    #         value=ROLES[0],
+    #     )
     with gr.Row():
         txt = gr.Textbox(
             scale=3,
@@ -735,16 +737,16 @@ with gr.Blocks(title=title) as demo:
         print("Audio STOP")
         set_audio_playing(False)
         
-    with gr.Row():
-        sentence = gr.Textbox(visible=False)
-        audio = gr.Audio(
-            value=None,
-            label="Generated audio response",
-            streaming=True,
-            autoplay=True,
-            interactive=False,
-            show_label=True,
-        )
+    # with gr.Row():
+    #     sentence = gr.Textbox(visible=False)
+    #     audio = gr.Audio(
+    #         value=None,
+    #         label="Generated audio response",
+    #         streaming=True,
+    #         autoplay=True,
+    #         interactive=False,
+    #         show_label=True,
+    #     )
         
         audio.end(stop)
     
@@ -759,23 +761,23 @@ with gr.Blocks(title=title) as demo:
     )        
  
     clear_btn = gr.ClearButton([chatbot, audio])
-    
+    #generate_speech, 
     txt_msg = txt_btn.click(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
-        generate_speech,  [chatbot,chatbot_role], [chatbot,chatbot_role, sentence, audio]
+         [chatbot,chatbot_role], [chatbot,chatbot_role, sentence, audio]
     )
 
     txt_msg.then(lambda: gr.update(interactive=True), None, [txt], queue=False)
-
+#generate_speech,
     txt_msg = txt.submit(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
-        generate_speech,  [chatbot,chatbot_role], [chatbot,chatbot_role, sentence, audio]
+          [chatbot,chatbot_role], [chatbot,chatbot_role, sentence, audio]
     )
 
     txt_msg.then(lambda: gr.update(interactive=True), None, [txt], queue=False)
-
+#generate_speech,
     file_msg = btn.stop_recording(
         add_file, [chatbot, btn], [chatbot, txt], queue=False
     ).then(
-        generate_speech,  [chatbot,chatbot_role], [chatbot,chatbot_role, sentence, audio]
+          [chatbot,chatbot_role], [chatbot,chatbot_role, sentence, audio]
     )
 
     file_msg.then(lambda: (gr.update(interactive=True),gr.update(interactive=True,value=None)), None, [txt, btn], queue=False)