harithapliyal
commited on
Update README.md
Browse files
README.md
CHANGED
@@ -42,14 +42,17 @@ from unsloth import is_bfloat16_supported
|
|
42 |
from transformers import AutoModelForCausalLM, BitsAndBytesConfig
|
43 |
|
44 |
# Configure the quantization
|
|
|
45 |
bnb_config = BitsAndBytesConfig(
|
46 |
load_in_4bit=True,
|
47 |
bnb_4bit_use_double_quant=True,
|
48 |
bnb_4bit_quant_type="nf4",
|
49 |
bnb_4bit_compute_dtype="float16"
|
50 |
)
|
|
|
51 |
|
52 |
# Load the model with quantization
|
|
|
53 |
model1 = AutoModelForCausalLM.from_pretrained(
|
54 |
"harithapliyal/llama-3-8b-bnb-4bit-finetuned-SentAnalysis",
|
55 |
quantization_config=bnb_config
|
@@ -70,5 +73,5 @@ inputs = tokenizer(
|
|
70 |
outputs = model.generate(**inputs, max_new_tokens = 64, use_cache = True)
|
71 |
outputs = tokenizer.decode(outputs[0])
|
72 |
print(outputs)
|
73 |
-
|
74 |
|
|
|
42 |
from transformers import AutoModelForCausalLM, BitsAndBytesConfig
|
43 |
|
44 |
# Configure the quantization
|
45 |
+
```
|
46 |
bnb_config = BitsAndBytesConfig(
|
47 |
load_in_4bit=True,
|
48 |
bnb_4bit_use_double_quant=True,
|
49 |
bnb_4bit_quant_type="nf4",
|
50 |
bnb_4bit_compute_dtype="float16"
|
51 |
)
|
52 |
+
```
|
53 |
|
54 |
# Load the model with quantization
|
55 |
+
```
|
56 |
model1 = AutoModelForCausalLM.from_pretrained(
|
57 |
"harithapliyal/llama-3-8b-bnb-4bit-finetuned-SentAnalysis",
|
58 |
quantization_config=bnb_config
|
|
|
73 |
outputs = model.generate(**inputs, max_new_tokens = 64, use_cache = True)
|
74 |
outputs = tokenizer.decode(outputs[0])
|
75 |
print(outputs)
|
76 |
+
```
|
77 |
|