Training in progress, step 45000

config.json

@@ -7,6 +7,7 @@
   "attention_dropout": 0.0,
   "bos_token_id": 0,
   "eos_token_id": 0,
+  "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 576,
   "initializer_range": 0.02,
@@ -23,7 +24,7 @@
   "rope_theta": 10000.0,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.44.2",
+  "transformers_version": "4.45.0.dev0",
   "use_cache": false,
   "vocab_size": 49152
 }

logs/attn_projector=mlp, per_device_train_batch_size=2, run_name=bs2/events.out.tfevents.1726128095.46d00238c241

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c1ec767c0a95b726bd9c27cb5226d8aec2587a919a594730352a09c2ffe34ab
-size 1104254
+oid sha256:05049f5d87e4a7ddb650af801d9766520fbc61a7d71713dd3954ec73157c8e85
+size 1242037

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2392770ba433e3966f57d02de2a7f10c7a5737785fb782a4b4d92cbac9887b83
+oid sha256:49c91e9fad2cc1c74abd81a3f6c974af6e03e2d9f0df89fddf9b7e96382cd6bc
 size 162842416

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:555ceb45b35190b72c7d1a0f76611e172bacea5f42f7a7ad17b8bf6a95802281
-size 5432
+oid sha256:738e9326fbb30717baf30f64578731beac353bb8abd9108ce8783db4565fa56e
+size 5368