Update README.md

README.md

@@ -42,8 +42,8 @@ copy <your_ryzen_ai-sw_install_path>\RyzenAI-SW\example\transformers\models\llam
 set XLNX_VART_FIRMWARE=<your_firmware_install_path>\voe-4.0-win_amd64\1x4.xclbin
 set NUM_OF_DPU_RUNNERS=1
 
-# save below sample script as utf8 and llama-3.1-test.py
-python llama3.1-test.py
+# save below sample script as utf8 and llama3.1-8b_translate-test.py
+python llama3.1-8b_translate-test.py
 ```
 
 ### Sample Script
@@ -56,38 +56,62 @@ from transformers import AutoTokenizer, set_seed
 import qlinear
 import logging
 
-set_seed(123)
-transformers.logging.set_verbosity_error()
-logging.disable(logging.CRITICAL)
-
-messages = [
-    {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
-]
-
-message_list = [
-	"Who are you? ", 
-	# Japanese
-	"あなたの乗っている船の名前は何ですか？英語ではなく全て日本語だけを使って返事をしてください",
-	# Chainese
-	"你经历过的最危险的冒险是什么？请用中文回答所有问题，不要用英文。",
-	# French
-	"À quelle vitesse va votre bateau ? Veuillez répondre uniquement en français et non en anglais.",
-	# Korean
-	"당신은 그 배의 어디를 좋아합니까? 영어를 사용하지 않고 모두 한국어로 대답하십시오.",
-	# German
-	"Wie würde Ihr Schiffsname auf Deutsch lauten? Bitte antwortet alle auf Deutsch statt auf Englisch.", 
-	# Taiwanese
-	"您發現過的最令人驚奇的寶藏是什麼？請僅使用台語和繁體中文回答，不要使用英文。",
-]
+
+def translation(instruction,  input):
+    system =  """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+You are a highly skilled professional translator. You are a native speaker of English, Japanese, French and Mandarin. Translate the given text accurately, taking into account the context and specific instructions provided. Steps may include hints enclosed in square brackets [] with the key and value separated by a colon:. If no additional instructions or context are provided, use your expertise to consider what the most appropriate context is and provide a natural translation that aligns with that context. When translating, strive to faithfully reflect the meaning and tone of the original text, pay attention to cultural nuances and differences in language usage, and ensure that the translation is grammatically correct and easy to read. For technical terms and proper nouns, either leave them in the original language or use appropriate translations as necessary. Take a deep breath, calm down, and start translating.<|eot_id|><|start_header_id|>user<|end_header_id|>"""
+
+    prompt = f"""{system}
+### Instruction:
+{instruction}
+
+### Input:
+{input}
+
+### Response:
+<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+"""
+
+    tokenized_input = tokenizer(prompt, return_tensors="pt",
+        padding=True, max_length=1600, truncation=True)
+
+    terminators = [
+        tokenizer.eos_token_id,
+    ]
+
+    outputs = model.generate(tokenized_input['input_ids'],
+            max_new_tokens=600,
+            eos_token_id=terminators,
+            attention_mask=tokenized_input['attention_mask'],
+            do_sample=True,
+            temperature=0.3,
+            top_p=0.5)
+    response = outputs[0][tokenized_input['input_ids'].shape[-1]:]
+    response_message = tokenizer.decode(response, skip_special_tokens=True)
+    return response_message
 
 
 if __name__ == "__main__":
-    p = psutil.Process()
-    p.cpu_affinity([0, 1, 2, 3])
-    torch.set_num_threads(4)
 
-    tokenizer = AutoTokenizer.from_pretrained("llama3.1-8b-Instruct-amd-npu")
-    ckpt = r"llama3.1-8b-Instruct-amd-npu\llama3.1_8b_w_bit_4_awq_amd.pt"
+  set_seed(123)
+  p = psutil.Process()
+  p.cpu_affinity([0, 1, 2, 3])
+  torch.set_num_threads(4)
+
+  tokenizer = AutoTokenizer.from_pretrained("llama3.1-8b_translate-amd-npu")
+  tokenizer.pad_token_id = tokenizer.add_special_tokens({'pad_token': '<|finetune_right_pad_id|>'})
+  ckpt = "llama3.1_8b_translate_w_bit_4_awq_amd.pt"
+
+  model = torch.load(ckpt)
+  model.eval()
+  model = model.to(torch.bfloat16)
+
+  for n, m in model.named_modules():
+      if isinstance(m, qlinear.QLinearPerGrp):
+          print(f"Preparing weights of layer : {n}")
+          m.device = "aie"
+          m.quantize_weights()
+
     terminators = [
         tokenizer.eos_token_id,
         tokenizer.convert_tokens_to_ids("<|eot_id|>")
@@ -102,31 +126,10 @@ if __name__ == "__main__":
             m.device = "aie"
             m.quantize_weights()
 
-    print("system: " + messages[0]['content'])
-
-    for i in range(len(message_list)):
-        messages.append({"role": "user",  "content": message_list[i]})
-        print("user: " + message_list[i])
-
-        input = tokenizer.apply_chat_template(
-            messages,
-            add_generation_prompt=True,
-            return_tensors="pt",
-            return_dict=True
-        )
-
-        outputs = model.generate(input['input_ids'],
-		max_new_tokens=600,
-    		eos_token_id=terminators,
-		attention_mask=input['attention_mask'],
-    		do_sample=True,
-    		temperature=0.6,
-    		top_p=0.9)
-
-        response = outputs[0][input['input_ids'].shape[-1]:]
-        response_message = tokenizer.decode(response, skip_special_tokens=True)
-        print("assistant: " + response_message)
-        messages.append({"role": "system", "content": response_message})
+
+  print("Translate Japanese to English.", "1月1日は日本の祝日です。その日は日曜日で、5日ぶりに雨が降りました")
+  print("Translate English to Japanese.", "It’s raining cats and dogs.")
+
 
 ```