Training in progress, step 5000

README.md

@@ -78,12 +78,12 @@ LlamaForCausalLM(
 
 # Resource Usage
 
-- Max Train VRAM Use: 20.9417 GB
+- Max Train VRAM Use: 12.7946 GB
 - Available VRAM: 23.4329 GB
 - GPUs: 
   - 1x NVIDIA GeForce RTX 4090
 - CPUs: 64
-- CPU Memory: 251.7190 GB
+- CPU Memory: 251.7299 GB
 - CPU Memory Bandwidth: 1600 GB/s
 
 # Distillation (Teacher -> Student) Architecture Difference:
@@ -115,7 +115,7 @@ LlamaForCausalLM(
 <br/>
 
 # Train Dataset
-Trained on 84,851,671 tokens from the [wikimedia/wikipedia](https://huggingface.co/datasets/wikimedia/wikipedia) dataset.
+Trained on 84,871,894 tokens from the [wikimedia/wikipedia](https://huggingface.co/datasets/wikimedia/wikipedia) dataset.
 
 - Num Samples: `99,800`
 - Subset: `20231101.en`
@@ -134,11 +134,7 @@ DistillationObjective(
         weight=0
     ),
     attn_loss_component=LossComponent(
-        weight=5.0,
-        loss_fn='raw_mse',
-        layer_mapper='layer-2',
-        norm='layernorm_teacher_only_affine',
-        projector='mlp'
+        weight=0
     )
 )
 ```
@@ -150,7 +146,7 @@ The following hyperparameters were used during training:
 <summary>Expand</summary>
 
 - learning_rate: `0.0002`
-- train_batch_size: `2`
+- train_batch_size: `4`
 - eval_batch_size: `2`
 - seed: `42`
 - optimizer: `Adam with betas=(0.9,0.999) and epsilon=1e-08`
@@ -165,14 +161,10 @@ The following hyperparameters were used during training:
         weight=0
     ),
     attn_loss_component=LossComponent(
-        weight=5.0,
-        loss_fn='raw_mse',
-        layer_mapper='layer-2',
-        norm='layernorm_teacher_only_affine',
-        projector='mlp'
+        weight=0
     )
 )`
-- lr_scheduler: `<torch.optim.lr_scheduler.LambdaLR object at 0x7f6ad1b142e0>`
+- lr_scheduler: `<torch.optim.lr_scheduler.LambdaLR object at 0x7eb253ff9660>`
 - student_model_name_or_path: `None`
 - student_config_name_or_path: `None`
 - student_model_config: `{'num_hidden_layers': 15}`
@@ -205,6 +197,6 @@ The following hyperparameters were used during training:
 
 # Framework Versions
 - Distily 0.5.0
-- Transformers 4.45.0.dev0
+- Transformers 4.44.2
 - Pytorch 2.5.0.dev20240911+cu121
-- Datasets 3.0.0
+- Datasets 2.21.0

config.json

@@ -7,7 +7,6 @@
   "attention_dropout": 0.0,
   "bos_token_id": 0,
   "eos_token_id": 0,
-  "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 576,
   "initializer_range": 0.02,
@@ -24,7 +23,7 @@
   "rope_theta": 10000.0,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.45.0.dev0",
+  "transformers_version": "4.44.2",
   "use_cache": false,
   "vocab_size": 49152
 }

generation_config.json

@@ -2,6 +2,6 @@
   "_from_model_config": true,
   "bos_token_id": 0,
   "eos_token_id": 0,
-  "transformers_version": "4.45.0.dev0",
+  "transformers_version": "4.44.2",
   "use_cache": false
 }

logs/attn_weight=0.0, per_device_train_batch_size=4, run_name=bs4_NO_liger_baseline, student_model_use_liger=False/events.out.tfevents.1726148253.1c1a426a2fee

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf4731dea58e40db4e1b8a523b91f2b8e9b403d55da8ebebf39d902946255bab
-size 253
+oid sha256:aae803ff6dd328699919493928d13e384e3295111e1bbc7544b79d04a10f8b4c
+size 529

logs/attn_weight=0.0, per_device_train_batch_size=4, run_name=logits_bs4_liger_torch_compile, student_model_use_liger=True, torch_compile=True/events.out.tfevents.1726155371.1c1a426a2fee

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78a4c0b3458c4104b6611e0e2f78d4bd512b7d55a41ef1d8d2b882d2159db624
+size 5616

logs/attn_weight=0.0, per_device_train_batch_size=4, run_name=logits_bs4_liger_torch_compile, student_model_use_liger=True, torch_compile=True/events.out.tfevents.1726155767.1c1a426a2fee

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3dfda7795f8dca11e9c2853b47db79f8d5e4c59a5bc1a0bc9c83322f37addd8
+size 5616

logs/attn_weight=0.0, per_device_train_batch_size=4, run_name=logits_bs4_torch_compile, student_model_use_liger=False, torch_compile=True/events.out.tfevents.1726156182.1c1a426a2fee

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:644939ffd31e8ce5fa6b45b761c3446ed209e14a56b40929a54cb540fd206e39
+size 140550

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4d35e5221c1a9340eef663c0674f7b544ed09e4f95457f0c98eb1c255a2acce
+oid sha256:6c1db03f19d5282f8261330094f26b6cf7d4178ec35cb3b13b390167637a7fdc
 size 162842416

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:738e9326fbb30717baf30f64578731beac353bb8abd9108ce8783db4565fa56e
-size 5368
+oid sha256:3127321d43c3138dea3921e8b46cf74e63fb22ba80f0338e4425f98d3dd416cd
+size 5432