Added temporary decoding kwargs"

agent.py

@@ -107,6 +107,18 @@ new_chat_template = """{{- bos_token }}
     {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
 {%- endif %}"""
 
+DECODE_ARGS = dict(
+    max_tokens=300,
+    temperature=1.5,
+    top_p=0.2,
+    frequency_penalty=0.3,
+    presence_penalty=0.5,
+    seed=42,
+    mirostat_tau=0.3,
+    mirostat_eta=0.0001,
+)
+MAX_TOKENS_INCREMENT = 50
+
 class Naomi:
     def __init__(self, **kwargs):
         # init dataclasses
@@ -129,11 +141,17 @@ class Naomi:
 
     def respond(self, user_input: dict, **kwargs):
         """ Called during stream. """
+        max_tokens = DECODE_ARGS['max_tokens']
+        DECODE_ARGS['max_tokens'] = max_tokens + MAX_TOKENS_INCREMENT
+
+        if kwargs:
+            DECODE_ARGS.update(kwargs)
+
         # user msg handling
         format_user_input = self.model.tokenizer_.hf_tokenizer.apply_chat_template([user_input], tokenize=False, add_generation_prompt=False)
         self.chat_history += format_user_input
         # agent msg results + clean
-        response = self.model(self.chat_history, **kwargs)
+        response = self.model(self.chat_history, **DECODE_ARGS)
         output = "".join(response['choices'][0]['text'].split('\n\n')[1:])
         # update history
         self.chat_history += self.model.tokenizer_.hf_tokenizer.apply_chat_template([{'role': 'assistant', 'content': output}], tokenize=False, add_generation_prompt=False)