Spaces:
Running
Running
| 5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base,Comments | |
| time: 76 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192,1. Takes 25min to load checkpoints. | |
| 5k,0.1944,0.1764,,0.1025,0.1232,0.1086,0.1260,0.1647,,0.0841,0.0762,0.0824,0.1066,2. GPU utility is only 20% | |
| 10k,0.3372,0.3292,,0.2073,,0.2636,0.1150,0.2659,0.2604,0.1348,0.1343,0.1585,0.1850, | |
| 15k,0.4050,0.3909,,0.3005,,0.3250,0.1872,0.3445,0.3244,0.1821,0.1930,0.1968,0.2443, | |
| 20k,0.4451,0.4497,,0.3506,0.2795,,0.2719,0.3802,0.3637,0.2086,0.2196,0.2231,0.2772, | |
| 25k,0.4899,0.4601,,0.3070,,0.3975,0.4093,0.4105,0.4120,0.2261,0.2375,0.2574,0.3146, | |
| 30k,0.5125,0.4824,,0.2461,0.2974,0.0303,0.4195,0.4330,0.4294,0.2352,0.2484,0.2675,0.3328, | |
| 35k,0.5249,0.5091,,0.3639,0.3572,0.1983,0.3587,0.4434,0.4428,0.2433,,0.2863,0.3507, | |
| 40k,0.5555,0.5166,,0.3537,0.0346,0.4571,0.4434,0.4618,0.4623,0.2708,0.2828,0.3020,0.3606, | |
| 45k,0.5664,0.5403,,0.3602,0.2674,0.2654,0.4366,0.4746,0.4792,0.2668,0.3018,0.3065,0.3726, | |
| 50k,0.5690,0.5217,,0.2407,0.3689,0.4355,0.4051,0.4885,0.4795,0.2906,0.2952,0.3187,0.3807, | |
| 55k,0.5843,0.5680,,0.2081,0.4101,0.4341,0.3230,0.4931,0.4940,0.2940,0.3117,0.3242,0.3984, | |
| 60k,0.5916,0.5814,,0.4068,0.4107,0.4861,0.4469,0.4955,0.5130,0.3137,0.3090,0.3422,0.4081, | |
| 65k,0.6032,0.5774,,0.3145,0.4477,0.4858,0.4907,0.5039,0.5087,0.3097,0.3184,0.3397,0.4156, | |
| 70k,0.6030,0.5920,,0.4102,0.4736,0.5080,0.4920,0.5164,0.5129,0.3236,0.3360,0.3375,0.4242, | |
| 75k,0.6216,0.6187,,0.2820,0.4226,0.4777,0.2245,0.5190,0.5042,0.3265,0.3341,0.3483,0.4220, | |
| 80k,0.6397,0.6127,,0.0975,0.4217,0.3698,,0.5185,0.5301,0.3352,0.3412,0.3532,0.4306, | |
| 85k,0.6416,0.6254,,0.0722,0.4763,0.3700,0.5029,0.5249,0.5350,0.3448,0.3423,0.3530,0.4340, | |
| 90k,0.6510,0.6317,,0.3388,0.1472,0.4793,0.0317,0.5337,0.5220,0.3440,0.3559,0.3644,0.4418, | |
| 95k,0.6655,0.6479,,0.5283,0.4938,0.5144,0.5180,0.5432,0.5446,0.3331,0.3393,0.3683,0.4454, | |
| 100k,0.6723,0.6486,,0.4317,0.1100,0.5121,0.5358,0.5383,0.5514,0.3520,0.3544,0.3698,0.4378, | |
| 105k,0.6755,0.6582,,0.1886,,0.5280,0.5153,0.5499,0.5562,0.3626,0.3642,0.3683,0.4525, | |
| 110k,0.6798,0.6668,,0.3510,,0.5468,0.5182,0.5541,0.5654,0.3694,,0.3903,0.4566, | |
| 115k,0.6796,0.6668,,0.3692,0.4759,0.5347,0.5132,0.5508,0.5577,0.3741,,0.3908,0.4482, | |
| 120k,0.6822,0.6688,,0.3690,0.4352,0.5376,0.5483,0.5567,0.5658,0.3881,,0.3950,0.4524, | |
| 125k,0.6894,0.6743,,0.3365,0.5206,0.4855,0.5211,0.5617,0.5658,0.3725,,0.3880,0.4592, | |
| 130k,0.6914,0.6709,,0.3550,0.0088,0.5238,0.5245,0.5597,0.5609,0.3698,,,0.4594, | |
| 135k,0.6915,0.6721,,0.3892,,0.5467,0.3977,0.5541,0.5774,0.3782,,,0.4636, | |
| 140k,,0.6773,,0.3930,,0.3110,0.4991,0.5572,0.5675,0.3906,,,0.4741, | |
| 145k,,,,0.4538,,0.5720,0.4872,0.5642,0.5639,,,,0.4720, | |
| 150k,,,,0.2883,,0.5612,,0.5701,0.5844,0.3899,,,0.4651, | |
| 155k,,,,0.4185,,0.5030,0.1586,0.5790,0.5755,0.4044,,,0.4784, | |
| 160k,,,,0.2720,,0.5701,0.5630,0.5819,0.5864,0.4049,,,0.4665, | |
| 165k,,,,0.4252,,0.5388,0.5642,,0.5853,0.4007,,,0.4793, | |
| 170k,,,,0.1507,,0.5951,0.5739,,,0.4150,,,0.4846, | |
| 175k,,,,0.3242,,0.5437,0.5640,,0.5979,0.4092,,,0.4908, | |
| 180k,,,,0.2653,,0.5580,0.5912,,0.6054,0.4189,,,, | |
| 185k,,,,0.2651,,0.5709,0.5852,,0.6064,,,,0.5030, | |
| 190k,,,,0.2380,,0.5142,,,0.5996,0.4193,,,0.5115, | |
| 195k,,,,0.4048,,0.5964,,,0.6243,0.4265,,,, | |
| 200k,,,,0.5058,,0.5684,,,0.6248,0.4256,,,, | |
| 205k,,,,0.0945,,0.5878,,,0.6224,0.4190,,,0.5105, | |
| 210k,,,,0.1557,,0.6020,,,0.6311,0.4415,,,0.5164, | |
| 215k,,,,0.2483,,0.5995,,,0.6293,0.4353,,,0.5163, | |
| 220k,,,,0.1725,,0.5924,,,0.6375,,,,, | |
| 225k,,,,0.2467,,0.4832,,,0.6340,0.4556,,,, | |
| 230k,,,,0.1653,,,,,0.6436,0.4622,,,, | |
| 235k,,,,0.1884,,,,,0.6411,0.4608,,,, | |
| 240k,,,,0.0719,,,,,,0.4536,,,, | |
| 245k,,,,0.3757,,,,,,0.4641,,,, | |
| 250k,,,,0.5859,,,,,,,,,, | |
| 255k,,,,0.4987,,,,,,0.4741,,,, | |
| 260k,,,,0.3940,,,,,,0.4712,,,, | |
| 265k,,,,0.3607,,,,,,0.4767,,,, | |
| 270k,,,,0.3898,,,,,,0.4795,,,, | |
| 275k,,,,0.4123,,,,,,,,,, | |
| 280k,,,,0.2413,,,,,,0.4787,,,, | |
| 285k,,,,0.3665,,,,,,0.4843,,,, | |
| 290k,,,,,,,,,,0.4818,,,, | |
| 300k,,,,,,,,,,0.4969,,,, | |
| 305k,,,,,,,,,,0.4941,,,, | |
| 310k,,,,,,,,,,0.4963,,,, | |
| 315k,,,,,,,,,,,,,, | |
| 320k,,,,,,,,,,,,,, | |
| 325k,,,,,,,,,,,,,, | |
| 330k,,,,,,,,,,,,,, | |
| 335k,,,,,,,,,,,,,, |