google

/

gemma-7b

like 2.97k

tokenizer = AutoTokenizer.from_pretrained(
            "google/{}".format(model),
            use_fast=False,
        )
generator = AutoModelForCausalLM.from_pretrained(
            "google/{}".format(model),
            torch_dtype=torch.float16,
            device_map="auto",
        )

user_prompt = "can you provide future suggestions for my a/b test and provide a summary for this a/b test?"
prompt = f"<bos><start_of_turn>user\nPlease respond to user questions based on the analysis of this a/b test experiment.\n\n{system_prompt}\n\n{user_prompt}<end_of_turn>\n<start_of_turn>model\n"
    
token_ids = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt")
print("starting this")
with torch.no_grad():
        output_ids = generator.generate(
            token_ids.to(generator.device),
            max_new_tokens=50,
            do_sample=True,
            temperature=0.7,
            top_p=1.0,
            pad_token_id=tokenizer.pad_token_id,
            bos_token_id=tokenizer.bos_token_id,
            eos_token_id=tokenizer.eos_token_id,
        )
print("fnished this")
response = tokenizer.decode(output_ids.tolist()[0][token_ids.size(1):])
print("My response from gemma")
print(response)

This was running fine on my macbook but as I am trying to run this on dell, I am having some issues. Any suggestions?