File size: 1,152 Bytes
019c8ae |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
import asyncio
import json
import aiohttp
async def fastai_stream(user, model="llama3-8b", system="Answer as concisely as possible."):
env_type = "tp16405b" if "405b" in model else "tp16"
data = {'body': {'messages': [{'role': 'system', 'content': system}, {'role': 'user', 'content': user}], 'stream': True, 'model': model}, 'env_type': env_type}
async with aiohttp.ClientSession() as session:
async with session.post('https://fast.snova.ai/api/completion', headers={'content-type': 'application/json'}, json=data) as response:
async for line in response.content:
line = line.decode('utf-8').strip()
if line.startswith('data:'):
try:
data = json.loads(line[len('data: '):])
content = data.get("choices", [{}])[0].get("delta", {}).get("content", '')
if content:
yield f"data: {json.dumps({'response': content})}\n\n"
except json.JSONDecodeError:
if line[len('data: '):] == '[DONE]':
break
|