Update README.md

README.md

@@ -11,9 +11,9 @@ tags:
 - text2sql
 ---
 # Model Description
-
 Our Model is fine tuned on Llama-2 7B model on text-2-sql Dataset on alpaca format described by Meta. The dataset is provided by "b-mc2/sql-create-context" present on Huggingface . We have used QLora, Bits&Bytes, Accelerate and Transformers Library to implement PEFT concept. We have fine-tuned this model based on pre-trained llama-2 7B model provided by 'NousResearch/Llama-2-7b-chat-hf'.
 
+
 # Inference
 ```python
 !pip install transformers accelerate xformers bitsandbytes
@@ -37,4 +37,47 @@ prompt = f"""Below is an context that describes a sql query, paired with an ques
 pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=200)
 result = pipe(prompt)
 print(result[0]['generated_text'])
-```
+```
+
+
+# Model Information
+model_name = "NousResearch/Llama-2-7b-chat-hf"
+dataset_name = "b-mc2/sql-create-context"
+
+
+# QLoRA parameters
+lora_r = 64
+lora_alpha = 16
+lora_dropout = 0.1
+
+
+# bitsandbytes parameters
+use_4bit = True
+bnb_4bit_compute_dtype = "float16"
+bnb_4bit_quant_type = "nf4"
+use_nested_quant = False
+
+
+# TrainingArguments parameters
+num_train_epochs = 1
+fp16 = False
+bf16 = False
+per_device_train_batch_size = 8
+per_device_eval_batch_size = 4
+gradient_accumulation_steps = 1
+gradient_checkpointing = True
+max_grad_norm = 0.3
+learning_rate = 2e-4
+weight_decay = 0.001
+optim = "paged_adamw_32bit"
+lr_scheduler_type = "cosine"
+max_steps = -1
+warmup_ratio = 0.03
+group_by_length = True
+save_steps = 0
+logging_steps = 25
+
+
+# SFT parameters
+max_seq_length = None
+packing = False