Manually build chat template

app.py

@@ -5,6 +5,9 @@ from threading import Thread
 import torch
 
 MODEL = "m-a-p/OpenCodeInterpreter-DS-33B"
+"bos_token": "<|begin_of_text|>",
+CHAT_TEMPLATE = "{{ bos_token }}{% for message in messages %}{% 
+if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}{% elif message['role'] == 'assistant' %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}{% else %}{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}"
 
 system_message = "You are a computer programmer that can translate python code to C++ in order to improve performance"
 
@@ -20,6 +23,19 @@ def messages_for(python):
         {"role": "user", "content": user_prompt_for(python)}
     ]
 
+def apply_chat_template(messages):
+    bos_token = "<｜begin▁of▁sentence｜>"
+    result = bos_token
+    for message in messages:
+        if message['role'] == 'user':
+            result += f"<|start_header_id|>user<|end_header_id|>\n\n{message['content']}<|eot_id|>"
+        elif message['role'] == 'assistant':
+            result += f"<|start_header_id|>assistant<|end_header_id|>\n\n{message['content']}<|eot_id|>"
+        else:
+            result += f"<|start_header_id|>{message['role']}<|end_header_id|>\n\n{message['content']}<|eot_id|>"
+    return result
+
+
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
 model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, device_map="auto")
 model.eval()
@@ -27,7 +43,7 @@ streamer = TextIteratorStreamer(tokenizer)
 
 cplusplus = None
 def translate(python):
-    inputs = tokenizer(messages_for(python), return_tensors="pt").to(model.device)
+    inputs = tokenizer(apply_chat_template(messages_for(python)), return_tensors="pt").to(model.device)
     generation_kwargs = dict(
                             inputs,
                             streamer=streamer,