so back uwu


app.py

@@ -5,34 +5,35 @@ import os
 import datetime
 from requests.exceptions import RequestException
 
+api_keys_env = os.environ.get('API_KEYS')
+if api_keys_env:
+    API_KEYS = [key.strip() for key in api_keys_env.strip().split('\n') if key.strip()]
+else:
+    raise ValueError("all keez ded go kys")
 API_URL = os.environ.get('API_URL')
-API_KEY = os.environ.get('API_KEY')
-
-headers = {
-    "Authorization": f"Bearer {API_KEY}",
-    "Content-Type": "application/json",
-    'Referer': os.environ.get('REFERRER_URL')
-}
-
-# debug switches
-USER_LOGGING_ENABLED = False
-RESPONSE_LOGGING_ENABLED = True
 
 DEFAULT_PARAMS = {
-    "temperature": 0.8,
-    "top_p": 0.95,
-    "top_k": 40,
+    "temperature": 1.0,
+    "top_p": 1,
+    "top_k": 0,
+    "min_p": 0,
+    "top_a": 0.1,
     "frequency_penalty": 0,
     "presence_penalty": 0,
     "repetition_penalty": 1.1,
     "max_tokens": 512
 }
 
+# debug switches
+USER_LOGGING_ENABLED = False
+RESPONSE_LOGGING_ENABLED = True
+
 def get_timestamp():
     return datetime.datetime.now().strftime("%H:%M:%S")
 
-def predict(message, history, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
-    history_format = [{"role": "system", "content": system_prompt}]
+def predict(message, history, system_prompt, temperature, top_p, top_k, min_p, top_a,
+            frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
+    history_format = [{"role": "system", "content": system_prompt}] if system_prompt else []
     for human, assistant in history:
         history_format.append({"role": "user", "content": human})
         if assistant:
@@ -47,6 +48,8 @@ def predict(message, history, system_prompt, temperature, top_p, top_k, frequenc
         "temperature": temperature,
         "top_p": top_p,
         "top_k": top_k,
+        "min_p": min_p,
+        "top_a": top_a,
         "frequency_penalty": frequency_penalty,
         "presence_penalty": presence_penalty,
         "repetition_penalty": repetition_penalty,
@@ -60,48 +63,76 @@ def predict(message, history, system_prompt, temperature, top_p, top_k, frequenc
             print(f"{param}={value}")
 
     data = {
-        "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
+        "model": "meta-llama/llama-3.1-405b-instruct:free",
         "messages": history_format,
         "stream": True,
         "temperature": temperature,
         "top_p": top_p,
         "top_k": top_k,
+        "min_p": min_p,
+        "top_a": top_a,
         "frequency_penalty": frequency_penalty,
         "presence_penalty": presence_penalty,
         "repetition_penalty": repetition_penalty,
         "max_tokens": max_tokens
     }
 
-    try:
-        with requests.post(API_URL, headers=headers, data=json.dumps(data), stream=True) as response:
-            partial_message = ""
-            for line in response.iter_lines():
-                if stop_flag[0]:
-                    response.close()
-                    break
-                if line:
-                    line = line.decode('utf-8')
-                    if RESPONSE_LOGGING_ENABLED:
-                        print(f"API Response: {line}")
-                    if line.startswith("data: "):
-                        if line.strip() == "data: [DONE]":
+    api_key_index = 0
+    retries = 0
+    max_retries = len(API_KEYS)
+
+    while retries < max_retries:
+        current_api_key = API_KEYS[api_key_index]
+
+        headers = {
+            "Authorization": f"Bearer {current_api_key}",
+            "Content-Type": "application/json"
+        }
+
+        try:
+            with requests.post(API_URL, headers=headers, data=json.dumps(data), stream=True) as response:
+                if response.status_code == 200:
+                    partial_message = ""
+                    for line in response.iter_lines():
+                        if stop_flag[0]:
+                            response.close()
                             break
-                        try:
-                            json_data = json.loads(line[6:])
-                            if 'choices' in json_data and json_data['choices']:
-                                content = json_data['choices'][0]['delta'].get('content', '')
-                                if content:
-                                    partial_message += content
-                                    yield partial_message
-                        except json.JSONDecodeError:
-                            continue  
-
-        if partial_message:
-            yield partial_message
-
-    except RequestException as e:
-        print(f"Request error: {e}")
-        yield f"An error occurred: {str(e)}"
+                        if line:
+                            line = line.decode('utf-8')
+                            if RESPONSE_LOGGING_ENABLED:
+                                print(f"API Response: {line}")
+                            if line.startswith("data: "):
+                                if line.strip() == "data: [DONE]":
+                                    break
+                                try:
+                                    json_data = json.loads(line[6:])
+                                    if 'choices' in json_data and json_data['choices']:
+                                        delta = json_data['choices'][0]['delta']
+                                        content = delta.get('content', '')
+                                        if content:
+                                            partial_message += content
+                                            yield partial_message
+                                except json.JSONDecodeError:
+                                    continue  
+                    if partial_message:
+                        yield partial_message
+                    # successful response, break out of retry loop
+                    break
+                elif response.status_code == 429:
+                    print("rate limit hit, cycling keys...")
+                    retries += 1
+                    api_key_index = (api_key_index + 1) % len(API_KEYS)
+                    continue
+                else:
+                    # theothershits
+                    error_message = f"Error: Received status code {response.status_code} - {response.text}"
+                    print(error_message)
+                    yield f"An error occurred: {error_message}"
+                    break
+        except RequestException as e:
+            print(f"Request error: {e}")
+            yield f"An error occurred: {str(e)}"
+            break
 
 def import_chat(custom_format_string):
     try:
@@ -147,7 +178,7 @@ with gr.Blocks(theme='gradio/monochrome') as demo:
     with gr.Row():
         with gr.Column(scale=2):
             chatbot = gr.Chatbot(value=[])
-            msg = gr.Textbox(label="Message (70B for now. The provider might bug out at random. The space may restart frequently)")
+            msg = gr.Textbox(label="Message")
             with gr.Row():
                 clear = gr.Button("Clear")
                 regenerate = gr.Button("Regenerate")
@@ -161,25 +192,29 @@ with gr.Blocks(theme='gradio/monochrome') as demo:
 
         with gr.Column(scale=1):
             system_prompt = gr.Textbox("", label="System Prompt", lines=5)
-            temperature = gr.Slider(0, 2, value=0.8, step=0.01, label="Temperature")
-            top_p = gr.Slider(0, 1, value=0.95, step=0.01, label="Top P")
-            top_k = gr.Slider(1, 500, value=40, step=1, label="Top K")
-            frequency_penalty = gr.Slider(-2, 2, value=0, step=0.1, label="Frequency Penalty")
-            presence_penalty = gr.Slider(-2, 2, value=0, step=0.1, label="Presence Penalty")
-            repetition_penalty = gr.Slider(0.01, 5, value=1.1, step=0.01, label="Repetition Penalty")
-            max_tokens = gr.Slider(1, 4096, value=512, step=1, label="Max Output (max_tokens)")
+            temperature = gr.Slider(0, 2, value=DEFAULT_PARAMS["temperature"], step=0.01, label="Temperature")
+            top_p = gr.Slider(0, 1, value=DEFAULT_PARAMS["top_p"], step=0.01, label="Top P")
+            top_k = gr.Slider(0, 500, value=DEFAULT_PARAMS["top_k"], step=1, label="Top K")
+            min_p = gr.Slider(0, 1, value=DEFAULT_PARAMS["min_p"], step=0.01, label="Min P")
+            top_a = gr.Slider(0, 1, value=DEFAULT_PARAMS["top_a"], step=0.01, label="Top A")
+            frequency_penalty = gr.Slider(-2, 2, value=DEFAULT_PARAMS["frequency_penalty"], step=0.1, label="Frequency Penalty")
+            presence_penalty = gr.Slider(-2, 2, value=DEFAULT_PARAMS["presence_penalty"], step=0.1, label="Presence Penalty")
+            repetition_penalty = gr.Slider(0.01, 5, value=DEFAULT_PARAMS["repetition_penalty"], step=0.01, label="Repetition Penalty")
+            max_tokens = gr.Slider(1, 4096, value=DEFAULT_PARAMS["max_tokens"], step=1, label="Max Output (max_tokens)")
 
     def user(user_message, history):
         history = history or []
         return "", history + [[user_message, None]]
 
-    def bot(history, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
+    def bot(history, system_prompt, temperature, top_p, top_k, min_p, top_a,
+            frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
         stop_flag[0] = False
         history = history or []
         if not history:
             return history
         user_message = history[-1][0]
-        bot_message = predict(user_message, history[:-1], system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag)
+        bot_message = predict(user_message, history[:-1], system_prompt, temperature, top_p, top_k, min_p, top_a,
+                              frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag)
         history[-1][1] = ""
         for chunk in bot_message:
             if stop_flag[0]:
@@ -188,11 +223,13 @@ with gr.Blocks(theme='gradio/monochrome') as demo:
             history[-1][1] = chunk
             yield history
 
-    def regenerate_response(history, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
+    def regenerate_response(history, system_prompt, temperature, top_p, top_k, min_p, top_a,
+                            frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
         if history and len(history) > 0:
             last_user_message = history[-1][0]
             history[-1][1] = None  
-            for new_history in bot(history, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
+            for new_history in bot(history, system_prompt, temperature, top_p, top_k, min_p, top_a,
+                                   frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag):
                 yield new_history
         else:
             yield []
@@ -202,14 +239,16 @@ with gr.Blocks(theme='gradio/monochrome') as demo:
         return imported_history, imported_system_prompt
 
     msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
-        bot, [chatbot, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag], chatbot
+        bot, [chatbot, system_prompt, temperature, top_p, top_k, min_p, top_a,
+              frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag], chatbot
     )
 
     clear.click(lambda: None, None, chatbot, queue=False)
 
     regenerate.click(
         regenerate_response,
-        [chatbot, system_prompt, temperature, top_p, top_k, frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag],
+        [chatbot, system_prompt, temperature, top_p, top_k, min_p, top_a,
+         frequency_penalty, presence_penalty, repetition_penalty, max_tokens, stop_flag],
         chatbot
     )