Init

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00001-of-00002.bin filter=lfs diff=lfs merge=lfs -text
+pytorch_model-00002-of-00002.bin filter=lfs diff=lfs merge=lfs -text
+tokenizer.model filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,106 @@
 ---
+language:
+- en
+library_name: transformers
+pipeline_tag: text-generation
+datasets:
+- jondurbin/airoboros-2.2
+- Open-Orca/OpenOrca
+- garage-bAInd/Open-Platypus
+- WizardLM/WizardLM_evol_instruct_V2_196k
+- TokenBender/python_eval_instruct_51k
+tags:
+- llama-2
+- code
 license: llama2
+model-index:
+- name: SpeechlessCoder
+  results:
+  - task:
+      type: text-generation
+    dataset:
+      type: openai_humaneval
+      name: HumanEval
+    metrics:
+    - name: pass@1
+      type: pass@1
+      value: 0.0
+      verified: false
 ---
+
+<p><h1> speechless-tora-code-7b-v1.0  </h1></p>
+
+Use the following dataset to fine-tune llm_agents/tora-code-7b-v0.1 in order to improve the model's reasoning and planning abilities.
+
+Total 201,981 samples.
+- jondurbin/airoboros-2.2: Filter categories related to coding, reasoning and planning. 23,462 samples.
+- Open-Orca/OpenOrca: Filter the 'cot' category in 1M GPT4 dataset. 74,440 samples.
+- garage-bAInd/Open-Platypus: 100%, 24,926 samples.
+- WizardLM/WizardLM_evol_instruct_V2_196k: Coding coversation part. 30,185 samples
+- TokenBender/python_eval_instruct_51k: “python” in output .40,309 samples
+- Spider: 8,659 samples
+
+
+| | |
+|------ | ------ |
+| lr | 2e-4 |
+| lr_scheduler_type | cosine |
+| weight_decay | 0.0 |
+| optim | paged_adamw_8bit |
+| flash_attention | True |
+| rerope | False |
+| max_new_tokens | 4096 |
+| num_train_epochs | 2 |
+| bits | 4 |
+| lora_r | 64 |
+| lora_alpha | 16 |
+| lora_dropout | 0.05 |
+| double_quant | True |
+| quant_type | nf4 |
+| dataset_format | airoboros |
+| mini_batch_size | 2 |
+| grandient_accumulation_steps | 32 |
+| bf16 | True |
+
+A100-40G x 4
+
+| | |
+|------ | ------ |
+| epoch                    |                2.0 |
+| etrain_loss               |             0.4708 |
+| etrain_runtime            | 12:12:53.64 |
+| etrain_samples_per_second |              9.002 |
+| etrain_steps_per_second   |              0.07 |
+| eeval_loss               |     0.4851 |
+| eeval_runtime            | 0:00:10.31 |
+| eeval_samples_per_second |      19.385 |
+| eeval_steps_per_second   |      4.846 |
+
+| Metric | Value |
+| --- | --- |
+| humaneval-python | |
+
+[Big Code Models Leaderboard](https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard)
+
+CodeLlama-34B-Python: 53.29
+
+CodeLlama-34B-Instruct: 50.79
+
+CodeLlama-13B-Instruct: 50.6
+
+CodeLlama-34B: 45.11
+
+CodeLlama-13B-Python: 42.89
+
+CodeLlama-13B: 35.07
+
+[Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+| Metric | Value |
+| --- | --- |
+| ARC | |
+| HellaSwag | |
+| MMLU | |
+| TruthfulQA |  |
+| Average |  |
+
+

added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "</s>": 2,
+  "<pad>": 32000,
+  "<s>": 1,
+  "<unk>": 0
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 16384,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.34.0",
+  "use_cache": true,
+  "vocab_size": 32001
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.34.0"
+}

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34de0eeb73499dff4312178870a24cc62d15caab73c13dcade8b0472e54dbe68
+size 9976628314

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0426b2042db9f55a1bcac302cc6e7b491eaeb1c9250e16749a1ec5529a48cd3b
+size 3500318979

pytorch_model.bin.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 13476847616
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00002-of-00002.bin",
+    "model.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.norm.weight": "pytorch_model-00002-of-00002.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tasks/speechless-tora-code-7b-v1.0/merge_peft_adapters.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# From speechless-code-mistral-7b-v1.0
+SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
+PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
+
+source ${SCRIPT_PATH}/task.env
+
+CHECKPOINT_DIR=${TASK_CHECKPOINT_DIR}/checkpoint-3092/adapter_model
+
+python ${SCRIPT_PATH}/../../scripts/merge_peft_adapters.py \
+    --base_model_name_or_path ${BASE_MODEL_PATH} \
+    --peft_model_path ${CHECKPOINT_DIR} \
+    --merged_model_name_or_path ${TEST_MODEL_PATH} \

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/all_results.json

@@ -0,0 +1,11 @@
+{
+    "epoch": 2.0,
+    "eval_loss": 0.5871890783309937,
+    "eval_runtime": 15.5979,
+    "eval_samples_per_second": 12.822,
+    "eval_steps_per_second": 6.411,
+    "train_loss": 0.5890884618112218,
+    "train_runtime": 69889.4305,
+    "train_samples_per_second": 5.664,
+    "train_steps_per_second": 0.044
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16.0,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "gate_proj",
+    "v_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe53a63399829bb762760aac51f487a74255439a8ba937191ccecf3a72040093
+size 639792909

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+- PEFT 0.5.0
+
+- PEFT 0.5.0

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16.0,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj",
+    "gate_proj",
+    "v_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe53a63399829bb762760aac51f487a74255439a8ba937191ccecf3a72040093
+size 639792909

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "</s>": 2,
+  "<pad>": 32000,
+  "<s>": 1,
+  "<unk>": 0
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ded5b21ac4123d95f741097b6ba2f10e14abf2fea6dc83011548234db36631c8
+size 320970527

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b465ff783be6837c8cdf3aa9aa36366d7695d1fdb6b4888ec0c8a7238379af77
+size 15735

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f79c2a3250c7c5ce53d25ba72962b46362a9f39b3282d1958c6bee70bd711b2e
+size 15735

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:403f7d4ddb29af77fdfd9e883d0c8935063872d5e73dea699394bc703f947ea8
+size 627

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/tokenizer_config.json

@@ -0,0 +1,52 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "tokenizer_file": null,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88c746833b5a7f3b81a2dbfe4fd0d18678e9740460c1a97584c75375f13a6c11
+size 4667

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3092/adapter_model/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+- PEFT 0.5.0
+
+- PEFT 0.5.0

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3092/adapter_model/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16.0,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "gate_proj",
+    "down_proj",
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3092/adapter_model/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:064bb620173b89e327154ada8f72dbd1eeea1d16d1ce351c2240385268dc3dac
+size 639792909

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/eval_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "eval_loss": 0.5871890783309937,
+    "eval_runtime": 15.5979,
+    "eval_samples_per_second": 12.822,
+    "eval_steps_per_second": 6.411
+}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/logs/finetune_20231009_061922.log

@@ -0,0 +1,24 @@
+2023-10-09 06:19:22.839 | INFO     | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-061916', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=70000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU  Backend: nccl
+Num processes: 2
+Process index: 0
+Local process index: 0
+Device: cuda:0
+, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=0), deepspeed_plugin=None)
+2023-10-09 06:19:22.840 | INFO     | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
+2023-10-09 06:19:22.848 | INFO     | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=1, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-061916', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=70000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU  Backend: nccl
+Num processes: 2
+Process index: 1
+Local process index: 1
+Device: cuda:1
+, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=1), deepspeed_plugin=None)
+2023-10-09 06:19:22.849 | INFO     | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
+2023-10-09 06:19:51.853 | INFO     | __main__:get_accelerate_model:350 - adding LoRA modules...
+2023-10-09 06:19:53.297 | INFO     | __main__:get_accelerate_model:350 - adding LoRA modules...
+2023-10-09 06:21:03.048 | INFO     | __main__:train:942 - loaded model
+2023-10-09 06:21:03.150 | INFO     | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
+2023-10-09 06:21:03.160 | INFO     | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
+2023-10-09 06:21:03.205 | INFO     | __main__:train:985 - *** Train ***
+2023-10-09 06:21:06.112 | INFO     | __main__:train:942 - loaded model
+2023-10-09 06:21:06.212 | INFO     | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
+2023-10-09 06:21:06.222 | INFO     | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
+2023-10-09 06:21:06.265 | INFO     | __main__:train:985 - *** Train ***

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/logs/finetune_20231009_122312.log

@@ -0,0 +1,88 @@
+2023-10-09 12:23:12.104 | INFO     | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=16, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=1, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-122305', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=32000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU  Backend: nccl
+Num processes: 2
+Process index: 1
+Local process index: 1
+Device: cuda:1
+, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=1), deepspeed_plugin=None)
+2023-10-09 12:23:12.105 | INFO     | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
+2023-10-09 12:23:12.105 | INFO     | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=16, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-122305', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=32000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU  Backend: nccl
+Num processes: 2
+Process index: 0
+Local process index: 0
+Device: cuda:0
+, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=0), deepspeed_plugin=None)
+2023-10-09 12:23:12.105 | INFO     | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
+2023-10-09 12:23:40.070 | INFO     | __main__:get_accelerate_model:350 - adding LoRA modules...
+2023-10-09 12:23:40.822 | INFO     | __main__:get_accelerate_model:350 - adding LoRA modules...
+2023-10-09 12:24:52.367 | INFO     | __main__:train:942 - loaded model
+2023-10-09 12:24:52.467 | INFO     | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
+2023-10-09 12:24:52.478 | INFO     | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
+2023-10-09 12:24:52.523 | INFO     | __main__:train:985 - *** Train ***
+2023-10-09 12:24:53.833 | INFO     | __main__:train:942 - loaded model
+2023-10-09 12:24:53.932 | INFO     | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
+2023-10-09 12:24:53.942 | INFO     | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
+2023-10-09 12:24:53.987 | INFO     | __main__:train:985 - *** Train ***
+2023-10-09 13:02:59.195 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 13:03:00.511 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 13:40:57.489 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 13:40:59.007 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 14:18:56.508 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 14:18:57.522 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 14:56:40.535 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 14:56:41.789 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 15:34:06.992 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 15:34:08.286 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 16:12:02.503 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 16:12:03.732 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 16:49:21.174 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 16:49:22.504 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 17:26:54.676 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 17:26:55.668 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 18:04:46.740 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 18:04:47.949 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 18:42:04.644 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 18:42:05.914 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 19:19:37.187 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 19:19:38.502 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 19:56:57.213 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 19:56:58.406 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 20:34:41.712 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 20:34:43.045 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 21:12:11.382 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 21:12:12.669 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 21:49:45.432 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 21:49:46.721 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 22:27:13.362 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 22:27:14.775 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 23:04:40.350 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 23:04:41.734 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 23:43:00.849 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-09 23:43:01.897 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 00:20:56.083 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 00:20:57.315 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 00:58:45.672 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 00:58:46.917 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 01:36:16.111 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 01:36:17.107 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 02:14:04.076 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 02:14:05.116 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 02:51:35.732 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 02:51:37.116 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 03:29:10.549 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 03:29:11.807 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 04:07:10.597 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 04:07:12.121 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 04:45:13.080 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 04:45:14.388 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 05:22:29.236 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 05:22:30.313 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 06:00:01.444 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 06:00:02.697 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 06:37:56.658 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 06:37:57.703 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 07:15:19.303 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 07:15:20.460 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 07:49:44.958 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 07:49:44.959 | INFO     | __main__:save_model:262 - Saving PEFT checkpoint...
+2023-10-10 07:49:45.497 | INFO     | __main__:train:996 - *** Evaluate ***
+2023-10-10 07:49:45.564 | INFO     | __main__:train:996 - *** Evaluate ***

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/metrics.json

@@ -0,0 +1 @@
+{"run_name": "20231009-122305", "train_runtime": 69889.4305, "train_samples_per_second": 5.664, "train_steps_per_second": 0.044, "train_loss": 0.5890884618112218, "epoch": 2.0, "eval_loss": 0.5871890783309937, "eval_runtime": 15.5979, "eval_samples_per_second": 12.822, "eval_steps_per_second": 6.411}

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e/events.out.tfevents.1696832467.I156f1f3f410070163e.7536.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2462cbdd451f1620c45fdc00446bef4939780801d8fc5937b775dd515dee8c8
+size 7869

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e/events.out.tfevents.1696854295.I156f1f3f410070163e.1701.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed40796c7e6086dc6fed30704e34ee1f93696be1510cda6f81cd0b162e7b03c
+size 498794

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e/events.out.tfevents.1696924201.I156f1f3f410070163e.1701.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1c5672e1dd6fd08655f555c268b13123ff736c6281a9bbef2fc2dd916bb156
+size 359

tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.5890884618112218,
+    "train_runtime": 69889.4305,
+    "train_samples_per_second": 5.664,
+    "train_steps_per_second": 0.044
+}

tasks/speechless-tora-code-7b-v1.0/run_api_server.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# From speechless-code-mistral-7b-v1.0
+SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
+PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
+
+source ${SCRIPT_PATH}/task.env
+
+PYTHONPATH=${SCRIPT_PATH}/../../.. \
+python ../../api/server.py \
+        --model_name_or_path=${TEST_MODEL_PATH} \
+        --model_family vllm

tasks/speechless-tora-code-7b-v1.0/run_finetune.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# From speechless-code-mistral-7b-v1.0
+SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
+PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
+
+source ${SCRIPT_PATH}/task.env
+
+PYTHONPATH=${PWD}/../.. \
+torchrun --nnodes=1 --nproc_per_node=${NUM_GPUS} \
+    ../../finetune.py \
+    ${DEEPSPEED_STAGE2} \
+    --task_name ${TASK_NAME} \
+    --run_name $(date +%Y%m%d-%H%M%S) \
+    --model_name_or_path ${BASE_MODEL_PATH} \
+    --output_dir ${OUTPUT_DIR} \
+    --num_train_epochs ${NUM_TRAIN_EPOCHS} \
+    --data_seed 10042 \
+    --save_strategy steps \
+    --save_total_limit 1 \
+    --evaluation_strategy steps \
+    --eval_dataset_size ${EVAL_DATASET_SIZE} \
+    --save_steps 100 \
+    --eval_steps 100 \
+    --warmup_steps 20 \
+    --max_eval_samples 200 \
+    --dataloader_num_workers 3 \
+    --logging_strategy steps \
+    --logging_steps 1 \
+    --report_to tensorboard \
+    --remove_unused_columns False \
+    --do_train \
+    --max_memory_MB ${MAX_MEMORY_MB} \
+    --bits 4 \
+    --lora_r ${LORA_R} \
+    --lora_alpha 16 \
+    --lora_dropout 0.05 \
+    --lora_modules all \
+    --double_quant \
+    --quant_type nf4 \
+    --bf16 \
+    --dataset ${DATASET} \
+    --dataset_format ${DATASET_FORMAT} \
+    --max_new_tokens 4096 \
+    --model_max_len 4096 \
+    --per_device_train_batch_size ${PER_DEVICE_TRAIN_BATCH_SIZE} \
+    --gradient_accumulation_steps ${GRADIENT_ACCUMULATION_STEPS} \
+    --per_device_eval_batch_size 1 \
+    --learning_rate ${LEARNING_RATE} \
+    --lr_scheduler_type cosine \
+    --weight_decay 0.0 \
+    --seed 10042 \
+    --optim paged_adamw_8bit \
+    --gradient_checkpointing True \
+    --group_by_length ${GROUP_BY_LENGTH} \
+    --ddp_find_unused_parameters False \
+    --force_remove_overlength_samples False \
+    --flash_attention True \
+    --rerope False \
+    --repeat_steps 0

tasks/speechless-tora-code-7b-v1.0/task.env

@@ -0,0 +1,47 @@
+# From speechless-code-mistral-7b-v1.0
+# -------------------- Model --------------------
+export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
+# FIXME
+export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/llm_agents/tora-code-7b-v1.0
+# FIXME
+export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-tora-code-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
+
+# -------------------- Dataset --------------------
+# FIXME
+# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
+# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
+# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
+export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
+
+# -------------------- Environment --------------------
+export OUTPUT_DIR=./outputs
+# export TORCH_DISTRIBUTED_DEBUG=DETAIL
+export RAY_memory_monitor_refresh_ms=0
+
+# -------------------- Task --------------------
+# FIXME
+export TASK_NAME=$(basename ${TEST_MODEL_PATH})
+export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
+export WANDB_PROJECT=${TASK_NAME}
+
+# -------------------- Task --------------------
+export DATASET_FORMAT=airoboros
+export NUM_TRAIN_EPOCHS=2
+export EVAL_DATASET_SIZE=0.005
+export LEARNING_RATE=2e-4
+export LORA_R=64
+export GROUP_BY_LENGTH=False
+
+#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
+
+export NUM_GPUS=2
+# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
+# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8 
+export PER_DEVICE_TRAIN_BATCH_SIZE=4
+export GRADIENT_ACCUMULATION_STEPS=16
+
+# No more than 85% VRAM.
+# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
+export MAX_MEMORY_MB=32000

tasks/speechless-tora-code-7b-v1.0/task_a100_40gx2.env

@@ -0,0 +1,49 @@
+# From speechless-code-mistral-orca-7b-v1.0
+# -------------------- Model --------------------
+export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
+# FIXME
+export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/Open-Orca/Mistral-7B-OpenOrca
+# export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
+# FIXME
+export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-orca-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
+
+# -------------------- Dataset --------------------
+# FIXME
+# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
+# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
+# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
+export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
+
+# -------------------- Environment --------------------
+export OUTPUT_DIR=./outputs
+export TORCH_DISTRIBUTED_DEBUG=DETAIL
+export RAY_memory_monitor_refresh_ms=0
+
+# -------------------- Task --------------------
+# FIXME
+export TASK_NAME=$(basename ${TEST_MODEL_PATH})
+export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
+export WANDB_PROJECT=${TASK_NAME}
+
+# -------------------- Task --------------------
+export DATASET_FORMAT=airoboros
+export NUM_TRAIN_EPOCHS=2
+export EVAL_DATASET_SIZE=0.005
+export LEARNING_RATE=2e-4
+export LORA_R=64
+export GROUP_BY_LENGTH=False
+
+#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
+
+export NUM_GPUS=2
+# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
+# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8 
+export PER_DEVICE_TRAIN_BATCH_SIZE=2
+export GRADIENT_ACCUMULATION_STEPS=32
+
+# No more than 85% VRAM.
+# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
+export MAX_MEMORY_MB=34000

tasks/speechless-tora-code-7b-v1.0/task_a100_40gx4.env

@@ -0,0 +1,49 @@
+# From speechless-code-mistral-orca-7b-v1.0
+# -------------------- Model --------------------
+export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
+# FIXME
+export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/Open-Orca/Mistral-7B-OpenOrca
+# export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
+# FIXME
+export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-orca-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
+
+# -------------------- Dataset --------------------
+# FIXME
+# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
+# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
+# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
+export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
+
+# -------------------- Environment --------------------
+export OUTPUT_DIR=./outputs
+export TORCH_DISTRIBUTED_DEBUG=DETAIL
+export RAY_memory_monitor_refresh_ms=0
+
+# -------------------- Task --------------------
+# FIXME
+export TASK_NAME=$(basename ${TEST_MODEL_PATH})
+export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
+export WANDB_PROJECT=${TASK_NAME}
+
+# -------------------- Task --------------------
+export DATASET_FORMAT=airoboros
+export NUM_TRAIN_EPOCHS=2
+export EVAL_DATASET_SIZE=0.005
+export LEARNING_RATE=2e-4
+export LORA_R=64
+export GROUP_BY_LENGTH=False
+
+#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
+
+export NUM_GPUS=4
+# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
+# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8 
+export PER_DEVICE_TRAIN_BATCH_SIZE=2
+export GRADIENT_ACCUMULATION_STEPS=16
+
+# No more than 85% VRAM.
+# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
+export MAX_MEMORY_MB=34000

tasks/speechless-tora-code-7b-v1.0/task_a100_80gx2.env

@@ -0,0 +1,47 @@
+# From speechless-code-mistral-7b-v1.0
+# -------------------- Model --------------------
+export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
+# FIXME
+export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
+# FIXME
+export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
+
+# -------------------- Dataset --------------------
+# FIXME
+# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
+# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
+# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
+export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
+
+# -------------------- Environment --------------------
+export OUTPUT_DIR=./outputs
+export TORCH_DISTRIBUTED_DEBUG=DETAIL
+export RAY_memory_monitor_refresh_ms=0
+
+# -------------------- Task --------------------
+# FIXME
+export TASK_NAME=$(basename ${TEST_MODEL_PATH})
+export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
+export WANDB_PROJECT=${TASK_NAME}
+
+# -------------------- Task --------------------
+export DATASET_FORMAT=airoboros
+export NUM_TRAIN_EPOCHS=2
+export EVAL_DATASET_SIZE=0.005
+export LEARNING_RATE=2e-4
+export LORA_R=64
+export GROUP_BY_LENGTH=False
+
+#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
+
+export NUM_GPUS=2
+# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
+# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8 
+export PER_DEVICE_TRAIN_BATCH_SIZE=2
+export GRADIENT_ACCUMULATION_STEPS=32
+
+# No more than 85% VRAM.
+# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
+export MAX_MEMORY_MB=70000

tasks/speechless-tora-code-7b-v1.0/task_a40_48gx2.env

@@ -0,0 +1,47 @@
+# From speechless-code-mistral-7b-v1.0
+# -------------------- Model --------------------
+export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
+# FIXME
+export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
+# FIXME
+export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
+
+# -------------------- Dataset --------------------
+# FIXME
+# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
+# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
+# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
+export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
+
+# -------------------- Environment --------------------
+export OUTPUT_DIR=./outputs
+export TORCH_DISTRIBUTED_DEBUG=DETAIL
+export RAY_memory_monitor_refresh_ms=0
+
+# -------------------- Task --------------------
+# FIXME
+export TASK_NAME=$(basename ${TEST_MODEL_PATH})
+export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
+export WANDB_PROJECT=${TASK_NAME}
+
+# -------------------- Task --------------------
+export DATASET_FORMAT=airoboros
+export NUM_TRAIN_EPOCHS=2
+export EVAL_DATASET_SIZE=0.005
+export LEARNING_RATE=2e-4
+export LORA_R=64
+export GROUP_BY_LENGTH=False
+
+#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
+
+export NUM_GPUS=2
+# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
+# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8 
+export PER_DEVICE_TRAIN_BATCH_SIZE=2
+export GRADIENT_ACCUMULATION_STEPS=32
+
+# No more than 85% VRAM.
+# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
+export MAX_MEMORY_MB=40000

tasks/speechless-tora-code-7b-v1.0/task_a40_48gx4.env

@@ -0,0 +1,47 @@
+# From speechless-code-mistral-7b-v1.0
+# -------------------- Model --------------------
+export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
+# FIXME
+export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
+# FIXME
+export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
+# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
+
+# -------------------- Dataset --------------------
+# FIXME
+# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
+# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
+# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
+export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
+
+# -------------------- Environment --------------------
+export OUTPUT_DIR=./outputs
+export TORCH_DISTRIBUTED_DEBUG=DETAIL
+export RAY_memory_monitor_refresh_ms=0
+
+# -------------------- Task --------------------
+# FIXME
+export TASK_NAME=$(basename ${TEST_MODEL_PATH})
+export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
+export WANDB_PROJECT=${TASK_NAME}
+
+# -------------------- Task --------------------
+export DATASET_FORMAT=airoboros
+export NUM_TRAIN_EPOCHS=2
+export EVAL_DATASET_SIZE=0.005
+export LEARNING_RATE=2e-4
+export LORA_R=64
+export GROUP_BY_LENGTH=False
+
+#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
+
+export NUM_GPUS=4
+# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
+# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8 
+export PER_DEVICE_TRAIN_BATCH_SIZE=2
+export GRADIENT_ACCUMULATION_STEPS=16
+
+# No more than 85% VRAM.
+# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
+export MAX_MEMORY_MB=40000

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723