respond_to_question_with_llama

app.py

@@ -38,6 +38,22 @@ def respond_to_question(transcript, question):
     
     return response
 
+@spaces.GPU
+def respond_to_question_llama(transcript, question):
+    from huggingface_hub import InferenceClient
+
+    client = InferenceClient(
+        "meta-llama/Meta-Llama-3.1-8B-Instruct",
+        token=os.environ["HUGGINGFACEHUB_API_KEY"],
+    )
+
+    response = client.chat_completion(
+        messages=[{"role": "user", "content": f"Transcript: {transcript}\n\nUser: {question}}],
+        max_tokens=500,
+    ).choices[0].content
+
+    return response
+
 @spaces.GPU
 def audio_transcribe(inputs):
     if inputs is None:
@@ -63,7 +79,7 @@ with gr.Blocks() as transcriberUI:
 
     def ask_question_callback(transcription,question):
         if ask_question:
-            response = respond_to_question(transcription, question)
+            response = respond_to_question_llama(transcription, question)
             response_output.value = response
         else:
             response_output.value = "No question asked"