End of training

README.md

@@ -15,14 +15,14 @@ This student model is distilled from the teacher model [roneneldan/TinyStories-3
 The [Distily](https://github.com/lapp0/distily) library was used for this distillation.
 
 It achieves the following results on the evaluation set:
-- eval_enwikippl: 1847.6166
-- eval_frwikippl: 30257.3203
-- eval_zhwikippl: 54131.9805
-- eval_tinystoriesppl: 468.7259
-- eval_loss: 4.4170
-- eval_runtime: 13.2864
-- eval_samples_per_second: 75.265
-- eval_steps_per_second: 9.408
+- eval_enwikippl: 5868.0605
+- eval_frwikippl: 32990.6758
+- eval_zhwikippl: 54785.7930
+- eval_tinystoriesppl: 2293.1941
+- eval_loss: 4.9180
+- eval_runtime: 13.0935
+- eval_samples_per_second: 76.374
+- eval_steps_per_second: 9.547
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment.
@@ -62,23 +62,23 @@ Peak GPU Memory: 8.1729 GB
 | step | epoch | enwikippl | frwikippl | loss | runtime | samples_per_second | steps_per_second | tinystoriesppl | zhwikippl |
 | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
 | **teacher eval** |  | 169.9865 | 47377.9414 |  |  |  |  | 3.9789 | 4998.1294 |
-| 0 | 0 | 10622.625 | 61209.6641 | 5.8730 | 13.3736 | 74.774 | 9.347 | 4331.7744 | 63461.6641 |
-| 1000 | 0.0808 | 1842.4707 | 30291.4473 | 4.4180 | 13.2514 | 75.464 | 9.433 | 466.2911 | 53987.7266 |
-| 2000 | 0.1616 | 1843.3275 | 30257.3203 | 4.4170 | 13.3491 | 74.912 | 9.364 | 466.7538 | 54016.5664 |
-| 3000 | 0.2424 | 1843.3275 | 30257.3203 | 4.4170 | 13.2545 | 75.446 | 9.431 | 466.7538 | 54045.3711 |
-| 4000 | 0.3232 | 1848.9050 | 30274.3789 | 4.4170 | 13.3151 | 75.103 | 9.388 | 469.0360 | 54131.9805 |
-| 5000 | 0.4040 | 1847.6166 | 30257.3203 | 4.4170 | 13.3271 | 75.035 | 9.379 | 468.7259 | 54131.9805 |
-| 6000 | 0.4848 | 1847.6166 | 30257.3203 | 4.4170 | 13.2448 | 75.501 | 9.438 | 468.6870 | 54131.9805 |
-| 7000 | 0.5657 | 1846.4716 | 30257.3203 | 4.4170 | 13.2796 | 75.303 | 9.413 | 468.1838 | 54131.9805 |
-| 8000 | 0.6465 | 1847.6166 | 30257.3203 | 4.4170 | 13.276 | 75.324 | 9.415 | 468.7259 | 54131.9805 |
-| 9000 | 0.7273 | 1847.6166 | 30257.3203 | 4.4170 | 13.2864 | 75.265 | 9.408 | 468.7259 | 54131.9805 |
-| 10000 | 0.8081 | 1847.7593 | 30257.3203 | 4.4170 | 13.2591 | 75.42 | 9.428 | 469.0360 | 54131.9805 |
-| 11000 | 0.8889 | 1848.9050 | 30257.3203 | 4.4170 | 13.2286 | 75.594 | 9.449 | 469.0360 | 54189.7773 |
-| 12000 | 0.9697 | 1847.7593 | 30257.3203 | 4.4170 | 13.3711 | 74.788 | 9.349 | 469.0360 | 54160.8438 |
-| 12375 | 1.0 | 1847.7593 | 30257.3203 | 4.4170 | 13.3215 | 75.067 | 9.383 | 469.0360 | 54160.8438 |
+| 0 | 0 | 34961.5352 | 67685.8906 | 6.4082 | 13.0484 | 76.638 | 9.58 | 22307.2852 | 64899.9219 |
+| 1000 | 0.0808 | 5926.0762 | 32934.9414 | 4.9183 | 13.0602 | 76.568 | 9.571 | 2332.1941 | 55049.4961 |
+| 2000 | 0.1616 | 5852.6240 | 32990.6758 | 4.9180 | 13.0475 | 76.643 | 9.58 | 2287.5139 | 54785.7930 |
+| 3000 | 0.2424 | 5843.5669 | 32990.6758 | 4.9177 | 13.045 | 76.658 | 9.582 | 2281.4705 | 54785.7930 |
+| 4000 | 0.3232 | 5878.9780 | 32990.6758 | 4.9180 | 13.0627 | 76.554 | 9.569 | 2303.0730 | 54815.0078 |
+| 5000 | 0.4040 | 5868.0605 | 32990.6758 | 4.9180 | 13.0226 | 76.789 | 9.599 | 2295.0898 | 54815.0078 |
+| 6000 | 0.4848 | 5867.1484 | 32990.6758 | 4.9180 | 13.0139 | 76.841 | 9.605 | 2291.6780 | 54785.7930 |
+| 7000 | 0.5657 | 5869.8799 | 32990.6758 | 4.9177 | 13.0183 | 76.815 | 9.602 | 2297.7485 | 54815.0078 |
+| 8000 | 0.6465 | 5868.0605 | 32990.6758 | 4.9180 | 13.084 | 76.429 | 9.554 | 2294.3315 | 54815.0078 |
+| 9000 | 0.7273 | 5868.0605 | 32990.6758 | 4.9180 | 13.0935 | 76.374 | 9.547 | 2293.1941 | 54785.7930 |
+| 10000 | 0.8081 | 5845.3784 | 32990.6758 | 4.9177 | 13.0045 | 76.896 | 9.612 | 2282.6021 | 54785.7930 |
+| 11000 | 0.8889 | 5848.9976 | 32990.6758 | 4.9177 | 13.0015 | 76.914 | 9.614 | 2284.8682 | 54785.7930 |
+| 12000 | 0.9697 | 5868.0605 | 32990.6758 | 4.9183 | 13.0386 | 76.696 | 9.587 | 2296.9883 | 54815.0078 |
+| 12375 | 1.0 | 5868.0605 | 32990.6758 | 4.9183 | 13.0038 | 76.9 | 9.613 | 2296.9883 | 54815.0078 |
 
 ### Framework versions
 - Distily 0.2.0
 - Transformers 4.44.0
 - Pytorch 2.3.0
-- Datasets 2.20.0
+- Datasets 2.21.0

logs/attn_loss_fn=None, attn_weight=0, gradient_accumulation_steps=1, hs_loss_fn=mse, hs_weight=2.0, learning_rate=0.0004, lr_scheduler_kwargs=__num_cycles___8_, lr_scheduler_type=cosine_with_restarts, max/events.out.tfevents.1723837024.5f530b1cf724

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0694831ab11e6712063412abfef2c8f7b06c8b570f3b51589857f97f811fe3f8
+size 307

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:960451a35c801281ca6e7a523aa41528c03c25770fecdf15b63277ee5c70517d
+oid sha256:35496e013e763fcac21a36025f56f533b776c899e1511b2504bd42b144cbfd95
 size 137033984

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2b2238360718a308cbe0b72932acfb5b6c7e3ae3d7e0e80b1a95534a44afacb
+oid sha256:e7ecd90b6f9698edbd2fa0694bead9e8f2b58c3d396bf6d8c467b1fc17b4c7c5
 size 1017948232