Training in progress, step 30000

config.json

@@ -7,6 +7,7 @@
   "attention_dropout": 0.0,
   "bos_token_id": 0,
   "eos_token_id": 0,
+  "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 576,
   "initializer_range": 0.02,
@@ -23,7 +24,7 @@
   "rope_theta": 10000.0,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.44.2",
+  "transformers_version": "4.45.0.dev0",
   "use_cache": false,
   "vocab_size": 49152
 }

logs/attn_projector=mlp, per_device_train_batch_size=2, run_name=bs2/events.out.tfevents.1726128095.46d00238c241

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a82c601feae41ce6b5ecfa8cdb3a118ffffa0009f7e3ff94aa9c58048dce8dd0
-size 688397
+oid sha256:46b273bb4465c5b9b9c7feaab1c78cb902981b70b52c67c2131d08c2fd08a500
+size 827434

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e58598147f8d537501c78635e17f84da28cd73c0af6de7953b43defd46f7756
+oid sha256:b4daca763ebd90dc016f63434e4ebcc90c001011bb0721131b341d053ace96c8
 size 162842416

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdb30d215278960a24b28efe91c6ca2db7380865e7318789dc26ff26097066c0
-size 5432
+oid sha256:738e9326fbb30717baf30f64578731beac353bb8abd9108ce8783db4565fa56e
+size 5368