Merge branch 'main' of https://huggingface.co/npvinHnivqn/stablelm_zephyr_3b_with_cross_attn

README.md

@@ -39,27 +39,26 @@ List 3 synonyms for the word "tiny"<|endoftext|>
 This format is also available through the tokenizer's `apply_chat_template` method:
 
 ```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModelForSeq2SeqLM
+import torch
 
-tokenizer = AutoTokenizer.from_pretrained('stabilityai/stablelm-zephyr-3b')
+tokenizer = AutoTokenizer.from_pretrained('npvinHnivqn/stablelm_zephyr_3b_with_cross_attn')
 model = AutoModelForCausalLM.from_pretrained(
-    'stabilityai/stablelm-zephyr-3b',
+    'npvinHnivqn/stablelm_zephyr_3b_with_cross_attn',
     trust_remote_code=True,
     device_map="auto"
 )
 
-prompt = [{'role': 'user', 'content': 'List 3 synonyms for the word "tiny"'}]
+prompt = [{'role': 'user', 'content': 'What make a dog bark?'}]
 inputs = tokenizer.apply_chat_template(
     prompt,
     add_generation_prompt=True,
     return_tensors='pt'
 )
-
-tokens = model.generate(
+    
+tokens = model(
     inputs.to(model.device),
-    max_new_tokens=1024,
-    temperature=0.8,
-    do_sample=True
+    torch.randn((1,19,2560)).to(model.device)
 )
 
 print(tokenizer.decode(tokens[0], skip_special_tokens=False))