Added draft

app.py

@@ -0,0 +1,37 @@
+import gradio as gr
+import torch
+from peft import PeftConfig, AutoPeftModelForCausalLM
+from transformers import AutoTokenizer, BitsAndBytesConfig
+
+checkpoint = 'ernestoBocini/Phi3-mini-DPO-Tuned'
+base_model_id = 'microsoft/Phi-3-mini-4k-instruct'
+
+nf4_config = BitsAndBytesConfig(
+   load_in_4bit=True,
+   bnb_4bit_quant_type="nf4",
+   bnb_4bit_use_double_quant=True,
+   bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+config = PeftConfig.from_pretrained(checkpoint)
+tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=True)
+tokenizer.model_max_length = 512
+tokenizer.pad_token = tokenizer.unk_token  # use unk rather than eos token to prevent endless generation
+tokenizer.pad_token_id = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)
+tokenizer.padding_side = 'left'
+
+model = AutoPeftModelForCausalLM.from_pretrained(
+    checkpoint,
+    quantization_config=nf4_config,
+    is_trainable=False,
+    trust_remote_code=True
+).eval()
+
+def chat(user_input):
+    inputs = tokenizer(user_input, return_tensors="pt", truncation=True, padding="max_length")
+    outputs = model(**inputs, max_new_tokens=512, num_return_sequences=1, pad_token_id=tokenizer.eos_token_id)
+    reply = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
+    return reply
+
+iface = gr.Interface(fn=chat, inputs="text", outputs="text", title="Chatbot")
+iface.launch()