Upload 3 files

README.md

@@ -1,3 +1,34 @@
 ---
 license: apache-2.0
+datasets:
+- c-s-ale/alpaca-gpt4-data
 ---
+
+This repo provides the training checkpoint of LLaMA on the alpaca_data_gpt4 dataset via LoRA [MLP]. 
+
+He et al. 2022 gave an insight that FFN can better utilize modification at larger capacities.
+
+The codes is provided by [tloen/alpaca-lora: Instruct-tune LLaMA on consumer hardware (github.com)](https://github.com/tloen/alpaca-lora).
+
+We modify the running scripts to 
+```bash
+torchrun --nproc_per_node=8 finetune.py \
+    --base_model '/cache1/chtan/large_models/llama-hf/llama-65b' \
+    --data_path './alpaca_data_gpt4.json' \
+    --output_dir './gpt4-alpaca-lora_mlp-65b' \
+    --batch_size 128 \
+    --micro_batch_size 2 \
+    --num_epochs 3 \
+    --learning_rate 1e-4 \
+    --cutoff_len 512 \
+    --val_set_size 2000 \
+    --lora_r 8 \
+    --lora_alpha 16 \
+    --lora_dropout 0.05 \
+    --lora_target_modules '[gate_proj,down_proj,up_proj]' \
+    --train_on_inputs \
+    --group_by_length
+```
+
+> [1] Junxian He, Chunting Zhou, Xuezhe Ma, Taylor Berg-Kirkpatrick, Graham Neubig: Towards a Unified View of Parameter-Efficient Transfer Learning. ICLR 2022
+

adapter_config.json

@@ -0,0 +1,19 @@
+{
+  "base_model_name_or_path": "/cache1/chtan/large_models/llama-hf/llama-65b",
+  "bias": "none",
+  "enable_lora": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "merge_weights": false,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a48054d036354e5fbe0c2420ee80a765b1ceb66187b7fde33d1e743220a5fdb
+size 232169613