Spaces:
Runtime error
Runtime error
| # ---------------------------------------- IMPORTS ---------------------------------------- # | |
| # this module allows us to get the headers from the get_headers function | |
| from .headers.get_headers import get_headers | |
| # this module allows us to get a random user agent | |
| from fake_useragent import UserAgent | |
| # this module allows us to generate random numbers | |
| import secrets | |
| # to make requests | |
| import requests | |
| # type hints | |
| from typing import Generator, Any, Dict, List | |
| # ---------------------------------------- FUNCTIONS ---------------------------------------- # | |
| def non_streamed_format(model: str, content: str) -> Dict[str, Any]: | |
| return { | |
| "object": "chat.completion", | |
| "model": f"{model}", | |
| "choices": [{ | |
| "index": 0, | |
| "message": { | |
| "role": "assistant", | |
| "content": f"{content}", | |
| }, | |
| "finish_reason": "stop", | |
| }], | |
| } | |
| # ---------------------------------------- API CLASS ---------------------------------------- # | |
| class Api(object): | |
| # initialize the class and set some variables here | |
| def __init__(self) -> None: | |
| """initialize the class""" | |
| self.url: str = "https://api.deepinfra.com/v1/openai/chat/completions" | |
| self.headers = get_headers(UserAgent().random, secrets.randbelow(500)) # get random headers | |
| self.session = requests.Session() # create a session | |
| # codec for encoding and decoding | |
| self.codec = 'utf-8' | |
| def get_models(self) -> Dict[str, List[Dict[str, str]]]: | |
| """get all models""" | |
| return {'data': [ | |
| {"id": "meta-llama/Meta-Llama-3-70B-Instruct", "context": 8192}, | |
| {"id": "meta-llama/Meta-Llama-3-8B-Instruct", "context": 8192}, | |
| {"id": "codellama/CodeLlama-34b-Instruct-hf", "context": 4096}, | |
| {"id": "jondurbin/airoboros-l2-70b-gpt4-1.4.1", "context": 4096}, | |
| {"id": "mistralai/Mistral-7B-Instruct-v0.2", "context": 32768}, | |
| {"id": "mistralai/Mistral-7B-Instruct-v0.3", "context": 32768}, | |
| {"id": "mistralai/Mixtral-8x7B-Instruct-v0.1", "context": 32768}, | |
| {"id": "mistralai/Mixtral-8x22B-Instruct-v0.1", "context": 65536}, | |
| {"id": "cognitivecomputations/dolphin-2.6-mixtral-8x7b", "context": 32768}, | |
| {"id": "lizpreciatior/lzlv_70b_fp16_hf", "context": 32768}, | |
| {"id": "deepinfra/airoboros-70b", "context": 4096}, | |
| {"id": "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", "context": 65536}, | |
| {"id": "microsoft/WizardLM-2-8x22B", "context": 65536}, | |
| ]} | |
| def chat( | |
| self, | |
| messages: List[Dict[str, str]], | |
| model: str, | |
| stream: bool = True, | |
| temperature: int = 0.7, | |
| max_tokens: int = 150, | |
| top_p: float = 1.0, | |
| top_k: int = 50, | |
| presence_penalty: float = 0.0, | |
| frequency_penalty: float = 0.0 | |
| ) -> Generator[str, Any, Any] | Dict[str, Any]: | |
| """ | |
| Chat with the DeepInfra models. | |
| :param messages: list of messages | |
| :param model: model name | |
| :param stream: stream the response | |
| :param temperature: temperature | |
| :param max_tokens: max tokens | |
| :param top_p: top p | |
| :param top_k: top k | |
| :param presence_penalty: presence penalty | |
| :param frequency_penalty: frequency penalty | |
| :return: generator or dict | |
| """ | |
| # compile the data | |
| data = { | |
| 'messages': messages, | |
| 'model': model, | |
| 'stream': stream, | |
| 'temperature': temperature, | |
| 'max_tokens': max_tokens, | |
| 'top_p': top_p, | |
| 'top_k': top_k, | |
| 'presence_penalty': presence_penalty, | |
| 'frequency_penalty': frequency_penalty, | |
| } | |
| # make a post request to the api | |
| response = self.session.post(self.url, headers=self.headers, json=data, stream=stream) | |
| # raise for status | |
| response.raise_for_status() | |
| # if stream is False | |
| if not stream: | |
| return non_streamed_format(model, response.json()['choices'][0]['message']['content']) | |
| return response.iter_lines() | |
| # Path: assets/source/api.py | |