Add model weight

README.md

@@ -1,10 +1,37 @@
 ---
 license: mit
 datasets:
-- izumi-lab/llm-japanese-dataset
+  - izumi-lab/llm-japanese-dataset
 language:
-- ja
+  - ja
 tags:
-- llama
-- causal-lm
----
+  - llama
+  - causal-lm
+---
+
+This repo contains a low-rank adapter for LLaMA-13b
+fit on the [llm-japanese-dataset](https://github.com/masanorihirano/llm-japanese-dataset) dataset.
+
+This version of the weights was trained with the following hyperparameters:
+
+- Epochs: 1
+- Batch size: 130
+- Cutoff length: 256
+- Learning rate: 3e-4
+- Lora _r_: 4
+- Lora target modules: q_proj, v_proj
+
+```python
+import torch
+from transformers import LlamaForCausalLM, LlamaTokenizer
+from peft import PeftModel
+
+base_model = "decapoda-research/llama-13b-hf"
+model = LlamaForCausalLM.from_pretrained(base_model, torch_dtype=torch.float16)
+tokenizer = LlamaTokenizer.from_pretrained(base_model)
+model = PeftModel.from_pretrained(
+    model,
+    "izumi-lab/llama-13b-japanese-lora-v0",
+    torch_dtype=torch.float16,
+)
+```

adapter_config.json

@@ -0,0 +1,17 @@
+{
+  "base_model_name_or_path": "decapoda-research/llama-13b-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 4,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84ebf795b240561cea674046dd1f0861072755b8fdecd9a7896706ac0986cbbc
+size 13164557