routers/bunnkatu/mysite/interpreter.py

import asyncio
from groq import Groq

async def completion(message: str, history, c=None, d=None):
    client = Groq(api_key=os.getenv("api_key"))
    messages = []
    recent_messages = history[-20:]
    for conversation in recent_messages:
        user_message = conversation[0]
        user_entry = {"role": "user", "content": user_message}
        messages.append(user_entry)
        assistant_message = conversation[1]
        assistant_entry = {"role": "assistant", "content": assistant_message}
        messages.append(assistant_entry)

    user_entry = {"role": "user", "content": message}
    messages.append(user_entry)
    system_prompt = {"role": "system", "content": "あなたは日本語の優秀なアシスタントです。"}
    messages.insert(0, system_prompt)
    async with asyncio.timeout(30):
        try:
            stream = client.chat.completions.create(
                model="llama3-8b-8192",
                messages=messages,
                temperature=1,
                max_tokens=1024,
                top_p=1,
                stream=True,
                stop=None,
            )
            all_result = ""
            for chunk in stream:
                current_content = chunk.choices[0].delta.content or ""
                all_result += current_content
                yield current_content
            yield all_result
        except asyncio.TimeoutError:
            raise HTTPException(status_code=504, detail="Stream timed out")