Training in progress, step 5000

config.json

@@ -7,7 +7,6 @@
   "attention_dropout": 0.0,
   "bos_token_id": 0,
   "eos_token_id": 0,
-  "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 576,
   "initializer_range": 0.02,
@@ -23,8 +22,8 @@
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.45.0.dev0",
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
   "use_cache": false,
   "vocab_size": 49152
 }

logs/gradient_accumulation_steps=2, per_device_train_batch_size=4, run_name=baseline/events.out.tfevents.1726221102.7eb020c3c65f

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f824c8e04e9a0930136209162a6b6901243f6abb10794b310d7487a11d465f26
+size 36512

logs/gradient_accumulation_steps=2, per_device_train_batch_size=4, run_name=baseline/events.out.tfevents.1726221897.7eb020c3c65f

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15a3fc1670071019cb682232cdaaa592a9a4a0b4b65906935ab65731a62f1b90
+size 140161

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:703c7a2fa368f06d7d81fd94c3d0ceb80d6095fd1a9a1062e60d4157b0caff75
-size 162842416
+oid sha256:7ad35b103674bf8e623362925f07ee7d10e3b0acef34f59545453700bd139283
+size 325669528

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8546763828ef41cacffda85467dac3cf6ddd9e36331a1a84cb902a717a10c5b6
+oid sha256:ddeef2630dae46e02a3d29cc1eceb330cdf1e4b3cfc7ea66cca5c985068dbe02
 size 5368