fix data columns

data/txt360_eval/CKPT Eval - BoolQ.csv

@@ -1,68 +0,0 @@
-0-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
-hf-time: 4 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
-5k,0.5761,0.5624,,0.6116,0.5514,0.5945,0.5446,0.5336,0.5902,0.5908,0.5394,0.5865,0.5284
-10k,0.6242,0.5853,,0.6131,,0.5358,0.6122,0.6080,0.5471,0.5511,0.6138,0.5902,0.5780
-15k,0.6480,0.6291,,0.6061,0.6217,0.5468,0.6205,0.6242,0.6248,0.5917,0.6211,0.5933,0.5713
-20k,0.6541,0.6474,,0.5865,0.6187,0.6122,0.6199,0.6116,0.6119,0.5636,0.6239,0.5988,0.5850
-25k,0.6670,0.6012,,0.6398,0.6251,0.6162,0.6349,0.6239,0.6291,0.5630,0.6336,0.6232,0.6312
-30k,0.6777,0.6523,,0.6379,0.6083,0.6260,0.6437,0.6263,0.6107,0.5835,0.5865,0.6391,0.6425
-35k,0.6495,0.6584,,0.6388,,0.6333,0.6346,0.6343,0.6144,0.4933,0.6043,0.6278,0.6480
-40k,0.6771,0.6930,,0.6489,0.6410,0.6596,0.6330,0.6214,0.6520,0.5685,0.5768,0.6343,0.6505
-45k,0.6624,0.6887,,0.6590,0.6422,0.6223,0.6401,0.6131,0.6153,0.5578,0.6058,0.6336,0.6529
-50k,0.6761,0.6951,,0.6575,0.6566,0.6593,0.6557,0.6058,0.6541,0.5972,0.6018,0.6177,0.6563
-55k,0.6847,0.6725,,0.6752,0.6321,0.6688,0.6523,0.6520,0.6679,0.5908,0.5343,0.6214,0.6618
-60k,0.6920,0.6697,,0.6566,0.6226,0.6642,0.6401,0.6162,0.6361,0.5908,0.5972,0.6226,0.6645
-65k,0.6979,0.6905,,0.6865,0.6352,0.6758,0.6688,0.6691,0.6942,0.6315,0.5682,0.6196,0.6352
-70k,0.7104,0.6966,,0.6795,0.6456,0.6746,0.6651,0.6624,0.6575,0.5997,0.5324,0.6358,0.6526
-75k,0.7269,0.6850,,0.6862,0.6514,,0.6621,0.6774,0.6817,0.6217,0.6009,0.6453,0.6535
-80k,0.6997,0.6817,,0.6945,0.6327,0.6664,0.6667,0.6709,0.6703,0.6275,0.5896,0.6502,0.6612
-85k,0.7346,0.6939,,0.6853,0.6746,0.6902,0.6602,0.6330,0.6737,0.6272,0.5239,0.6489,0.6703
-90k,0.7254,0.6908,,0.6936,0.6612,0.6713,0.6755,0.6835,0.6315,0.6275,0.5428,0.6128,0.6807
-95k,0.7165,0.7229,,0.7003,0.6587,,0.6823,0.6404,0.6670,0.6089,0.6138,0.6456,0.6612
-100k,0.7153,0.7073,,0.6869,,0.6676,0.6746,0.6618,0.6587,0.6006,0.5584,0.6566,0.6810
-105k,0.7333,0.7147,,0.6682,,0.6899,0.6609,0.6853,0.6853,0.6544,0.5740,0.6520,0.6755
-110k,0.7376,0.7095,,0.6954,0.6664,0.6703,0.6810,0.6612,0.6798,0.6618,,0.6346,0.6434
-115k,0.7168,0.7095,,0.7156,0.6645,0.6746,0.6997,0.6829,0.6813,0.6523,,0.6596,0.6920
-120k,0.7370,0.7226,,0.7177,0.6648,0.6752,0.7015,,0.6841,0.6633,,0.6587,0.6890
-125k,0.7361,0.7144,,0.7034,0.6636,0.6826,0.6869,0.6657,,0.6593,,0.6593,0.6795
-130k,0.7284,0.7269,,0.6939,0.6786,0.6554,0.6988,0.6719,0.6777,0.6260,,,0.7018
-135k,0.7483,0.7141,,0.7128,,0.6847,0.7028,0.6838,0.6933,0.6602,,,0.6966
-140k,,0.7312,,0.7080,,0.6777,0.6997,0.6957,0.7040,0.6624,,,0.6884
-145k,,,,0.7281,,0.6844,0.6908,0.6743,0.6914,0.6657,,,0.7061
-150k,,,,0.7297,,0.6795,,0.6807,0.6991,0.6526,,,0.7024
-155k,,,,0.7162,,0.7021,0.6976,0.6792,0.6927,0.6587,,,0.7028
-160k,,,,0.6902,,0.6810,0.6985,0.6930,0.6893,0.6434,,,0.7098
-165k,,,,0.7239,,0.6896,0.7037,,0.7021,0.6581,,,0.7080
-170k,,,,0.7471,,0.6780,0.7141,,0.6911,0.6761,,,0.7058
-175k,,,,0.7486,,0.6817,0.6942,,0.7095,0.6557,,,0.7021
-180k,,,,0.6985,,0.6979,0.7162,,0.7067,0.6468,,,0.6523
-185k,,,,0.7187,,0.6887,0.7031,,0.6917,0.6642,,,0.6914
-190k,,,,0.7333,,0.6963,,,0.7113,0.6563,,,0.718
-195k,,,,0.7269,,0.7021,,,0.7199,0.6817,,,0.7165
-200k,,,,0.7135,,0.7080,,,0.707,0.6709,,,0.7015
-205k,,,,0.7388,,0.7015,,,0.7168,0.6722,,,0.722
-210k,,,,0.7489,,0.7089,,,,0.6765,,,0.6948
-215k,,,,0.7538,,0.7183,,,0.7309,0.6869,,,0.6835
-220k,,,,0.7474,,0.7171,,,0.7398,0.6893,,,
-225k,,,,0.7251,,0.7131,,,0.7061,0.6801,,,
-230k,,,,0.7083,,,,,0.7232,0.6765,,,
-235k,,,,0.6930,,,,,0.6884,0.6434,,,
-240k,,,,0.7541,,,,,,0.6875,,,
-245k,,,,0.7541,,,,,,0.6713,,,
-250k,,,,0.7498,,,,,,0.6798,,,
-255k,,,,0.7749,,,,,,0.6578,,,
-260k,,,,0.7615,,,,,,0.6954,,,
-265k,,,,0.7486,,,,,,0.6807,,,
-270k,,,,0.7226,,,,,,0.6869,,,
-275k,,,,0.7269,,,,,,0.6841,,,
-280k,,,,0.7517,,,,,,0.6804,,,
-285k,,,,0.7150,,,,,,0.7006,,,
-290k,,,,,,,,,,0.6826,,,
-300k,,,,,,,,,,0.6706,,,
-305k,,,,,,,,,,0.7006,,,
-310k,,,,,,,,,,0.6777,,,
-315k,,,,,,,,,,0.6859,,,
-320k,,,,,,,,,,0.6939,,,
-325k,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,

data/txt360_eval/CKPT Eval - GSM8K.csv

@@ -1,68 +0,0 @@
-5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
-hf-time: 115 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
-5k,0.0152,0.0099,,,0.0076,0.0015,0.0045,0.0030,,0.0152,0.0106,0.0197,0.0197
-10k,0.0152,0.0190,,0.0015,,0.0091,0.0000,0.0212,0.0144,0.0159,0.0136,0.0174,0.0243
-15k,0.0182,0.0167,,0.0053,0.0068,0.0045,0.0083,0.0212,0.0068,0.0174,0.0190,0.0174,0.0136
-20k,0.0250,0.0212,,,,,0.0030,0.0159,0.0220,0.0167,0.0190,0.0220,0.0174
-25k,0.0288,0.0114,,,,0.0129,0.0053,0.0258,0.0144,0.0152,0.0144,0.0144,0.0144
-30k,0.0220,0.0265,,0.0197,0.0038,0.0152,0.0167,0.0227,0.0220,0.0205,0.0129,0.0167,0.0038
-35k,0.0296,0.0212,,0.0136,0.0045,0.0190,0.0045,0.0227,0.0220,0.0174,0.0174,0.0243,0.0182
-40k,0.0235,0.0288,,0.0068,0.0121,0.0220,0.0015,0.0243,0.0265,0.0152,0.0212,0.0190,0.0182
-45k,0.0387,0.0250,,0.0258,0.0038,0.0273,0.0106,0.0296,0.0273,0.0182,0.0152,0.0174,0.0129
-50k,0.0318,0.0303,,0.0015,0.0243,0.0227,0.0121,0.0190,0.0220,0.0197,0.0205,0.0182,0.0068
-55k,0.0296,0.0311,,0.0023,0.0235,0.0235,0.0250,0.0326,0.0197,0.0182,0.0174,0.0250,0.0091
-60k,0.0432,0.0326,,0.0167,0.0212,0.0212,0.0182,0.0349,0.0220,0.0182,0.0099,0.0190,0.0197
-65k,0.0470,0.0379,,0.0015,0.0159,0.0281,0.0136,0.0296,0.0212,0.0212,0.0129,0.0205,0.0114
-70k,0.0432,0.0417,,0.0136,0.0197,0.0174,0.0114,0.0341,0.0243,0.0205,0.0136,0.0250,0.0091
-75k,0.0508,0.0470,,0.0174,0.0121,0.0250,0.0182,0.0356,0.0288,0.0281,0.0174,0.0190,0.0106
-80k,0.0561,0.0417,,0.0068,0.0000,0.0190,0.0083,0.0318,0.0356,0.0273,0.0167,0.0265,0.0182
-85k,0.0728,0.0341,,0.0341,0.0190,0.0296,0.0205,0.0265,0.0250,0.0220,0.0129,0.0235,0.0083
-90k,0.0690,0.0425,,0.0197,0.0190,0.0281,0.0061,0.0417,0.0265,0.0273,0.0167,0.0190,0.0182
-95k,0.0735,0.0447,,0.0167,0.0250,0.0281,0.0136,0.0349,0.0281,0.0174,0.0106,0.0288,0.0159
-100k,0.0637,0.0470,,0.0159,,0.0227,0.0045,0.0409,0.0311,0.0265,0.0205,0.0190,0.0190
-105k,0.0637,0.0447,,0.0341,,0.0303,0.0129,0.0371,0.0311,0.0273,0.0205,0.0311,0.0129
-110k,0.0872,0.0576,,0.0038,0.0273,0.0129,0.0205,0.0478,0.0296,0.0212,,0.0281,0.0182
-115k,0.0788,0.0576,,0.0091,0.0167,0.0311,0.0167,0.0508,0.0349,0.0220,,0.0220,0.0174
-120k,0.0834,0.0455,,0.0227,0.0265,0.0167,0.0212,0.0371,0.0318,0.0167,,0.0220,0.0152
-125k,0.1001,0.0493,,0.0288,0.0250,0.0205,0.0387,0.0402,0.0318,0.0182,,0.0235,0.0144
-130k,0.0766,0.0470,,0.0068,0.0258,0.0288,0.0174,,0.0341,0.0243,,,0.0205
-135k,0.0879,0.0607,,0.0190,,0.0349,0.0258,0.0409,0.0288,0.0212,,,0.0281
-140k,,0.0569,,0.0379,,0.0356,0.0227,0.0440,0.0341,0.0144,,,0.0144
-145k,,,,0.0341,,0.0379,0.0015,0.0387,,0.0174,,,0.0273
-150k,,,,,,0.0281,,0.0470,0.0265,0.0220,,,0.0258
-155k,,,,0.0318,,0.0303,0.0121,0.0561,0.0523,0.0227,,,0.0243
-160k,,,,0.0356,,0.0243,0.0061,0.0425,0.0432,0.0220,,,0.0303
-165k,,,,0.0167,,0.0409,0.0015,,0.0470,0.0281,,,
-170k,,,,0.0334,,0.0281,0.0129,,0.0455,0.0273,,,0.0235
-175k,,,,0.0371,,0.0326,0.0190,,0.0409,0.0190,,,0.0273
-180k,,,,0.0425,,0.0364,0.0227,,0.0356,0.0243,,,0.0288
-185k,,,,0.0341,,0.0318,0.0341,,0.0546,0.0235,,,0.0364
-190k,,,,0.0296,,0.0364,,,0.0425,0.0220,,,0.0349
-195k,,,,0.0250,,0.0303,,,0.0493,0.0258,,,
-200k,,,,0.0250,,0.0371,,,0.0493,0.0273,,,0.0205
-205k,,,,0.0455,,0.0409,,,0.0553,0.0220,,,0.0258
-210k,,,,0.0462,,0.0371,,,0.0523,0.0281,,,
-215k,,,,0.0349,,0.0265,,,0.0500,0.0235,,,0.0281
-220k,,,,0.0432,,0.0167,,,0.0462,0.0326,,,
-225k,,,,0.0447,,0.0212,,,,0.0265,,,
-230k,,,,0.0440,,,,,0.0493,0.0273,,,
-235k,,,,0.0402,,,,,0.0508,0.0220,,,
-240k,,,,0.0341,,,,,,0.0281,,,
-245k,,,,0.0462,,,,,,0.0356,,,
-250k,,,,0.0500,,,,,,,,,
-255k,,,,0.0569,,,,,,0.0303,,,
-260k,,,,0.0500,,,,,,0.0334,,,
-265k,,,,0.0455,,,,,,0.0318,,,
-270k,,,,0.0538,,,,,,0.0273,,,
-275k,,,,0.0470,,,,,,,,,
-280k,,,,0.0553,,,,,,0.0364,,,
-285k,,,,0.0531,,,,,,0.0349,,,
-290k,,,,,,,,,,0.0311,,,
-300k,,,,,,,,,,,,,
-305k,,,,,,,,,,0.0311,,,
-310k,,,,,,,,,,0.0273,,,
-315k,,,,,,,,,,,,,
-320k,,,,,,,,,,,,,
-325k,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,

data/txt360_eval/CKPT Eval - HellaSwag.csv

@@ -1,69 +1,68 @@
-ga,Slim-Pajama 600B (bsz=4K x 1024),,,,,,FineWeb-1.5T,,Ours-Base,,Ours-Upsampling1,,Ours-Upsampling2,,Ours-Code-Upsampling2,,All-Upsampling1,,All-Upsampling1,,All-Upsampling1,,All-Upsampling1,,DCLM-Base,
-0-shot: 5 min,Llama-8x8B-baseline,,Llama-8x8B-seq8192,,Llama-8x8B-mup,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-1x8B-seq8192,,Llama_extend-1x8B-seq8192,,Jais-1x8B-seq8192,,Llama-1x8B-seq8192,
-10-shot: 36 min,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot,0-shot,10-shot
-5k,0.5315,0.5301,0.5165,0.5693,,,0.5622,0.5376,0.5254,0.5119,0.5356,0.5291,0.5324,0.5210,0.5377,0.5339,0.5366,0.5278,0.4507,0.4300,0.4413,0.4262,0.4497,0.4397,0.4624,0.4469
-10k,0.6076,0.6008,0.5949,0.5693,,,0.6433,0.6202,0.5836,0.5827,,0.5975,0.6046,0.5886,0.6036,0.5987,0.6139,0.5901,0.5279,0.4889,0.5141,0.4872,0.5219,0.5028,0.5454,0.5150
-15k,0.6422,0.6278,0.6314,0.5998,,,0.6716,0.6367,0.6114,0.6002,0.6281,0.6079,0.6336,0.6118,0.6399,0.6266,0.6388,0.6172,0.5495,0.5211,0.5444,0.5142,0.5469,0.5096,0.5785,0.5484
-20k,0.6616,0.6424,0.6496,0.6244,,,0.6855,,0.6271,0.6223,0.6461,0.6230,0.6492,0.6329,0.6511,0.6475,0.6548,0.6382,0.5685,0.5310,0.5579,0.5270,0.5813,0.5377,0.5946,0.5649
-25k,0.6738,0.6577,0.6683,0.6390,,,0.6945,0.6662,0.6413,,0.6612,0.6404,0.6665,0.6417,0.6652,0.6629,0.6683,0.6499,0.5759,0.5369,0.5787,0.5486,0.5864,0.5598,0.6105,0.5796
-30k,0.6863,0.6656,0.6758,0.6368,,,0.7059,0.6639,,0.6387,0.6692,0.6425,0.6746,0.6485,0.6708,0.6584,0.6741,0.6587,0.5891,0.5490,0.5915,0.5437,0.5990,0.5625,0.6197,0.5897
-35k,0.6956,0.6762,0.6850,0.6430,,,0.7158,0.6602,0.6547,0.6420,,0.6348,0.6832,0.6572,0.6816,0.6705,0.6864,0.6682,0.5985,0.5590,0.5954,0.5553,0.6090,0.5667,0.6343,0.6046
-40k,0.7022,0.6812,0.6966,0.6524,,,0.7184,0.6814,0.6642,0.6452,0.6751,0.6347,0.6821,0.6533,0.6865,0.6717,0.6917,0.6646,0.6015,0.5595,0.6033,0.5592,0.6112,0.5704,0.6429,0.6100
-45k,0.7048,0.6954,0.6991,0.6583,,,0.7220,0.6921,0.6698,0.6479,0.6802,,0.6905,0.6616,0.6919,0.6812,0.6933,0.6704,0.6103,0.5663,0.6040,0.5623,0.6175,0.5773,0.6473,0.6212
-50k,0.7171,0.6998,0.7041,0.6574,,,0.7250,0.6785,0.6689,0.6611,0.6931,0.6726,0.6964,0.6720,0.6905,0.6715,0.7018,0.6902,0.6106,0.5510,0.6138,0.5676,0.6230,0.5934,0.6477,0.6109
-55k,0.7187,0.7012,0.7080,0.6768,,,0.7305,0.6967,0.6697,0.6571,0.6899,0.6614,0.6959,0.6764,0.7047,0.6816,0.7052,0.6799,0.6182,0.5759,0.6200,0.5753,0.6260,0.5857,0.6518,0.6145
-60k,0.7240,0.7037,0.7129,0.6748,,,0.7236,0.6955,0.6748,0.6573,0.6941,0.6584,0.6904,0.6850,0.6982,0.6731,0.7040,0.6767,0.6207,0.5849,0.6217,0.5711,0.6318,0.5744,0.6566,0.6204
-65k,0.7297,0.7130,0.7142,0.6700,,,0.7355,0.6994,0.6752,0.6590,0.6907,0.6598,0.7061,0.6772,0.6963,0.6824,0.7074,0.6857,0.6235,0.5766,0.6299,0.5750,0.6381,0.5973,0.6544,0.6264
-70k,0.7298,0.7148,0.7224,0.6796,,,0.7399,0.7034,0.6773,0.6631,0.6968,0.6735,0.7054,0.6789,0.7043,0.6936,0.7074,0.6883,0.6294,0.5982,0.6341,0.5872,0.6403,0.5928,0.6617,0.6220
-75k,0.7329,0.7144,0.7261,0.6972,,,0.7374,0.6934,0.6854,0.6661,0.7014,0.6622,0.7065,0.6843,0.7029,0.6837,0.7027,0.6853,0.6285,0.5932,0.6336,0.5830,0.6376,0.5907,0.6706,0.6237
-80k,0.7414,0.7271,0.7316,0.6937,,,0.7422,0.6989,0.6862,0.6717,0.7051,0.6762,0.7118,0.6954,0.7178,0.6909,0.7139,0.6908,0.6315,0.5898,0.6363,0.5877,0.6491,0.5968,0.6710,0.6032
-85k,0.7449,0.7278,0.7334,0.7011,,,0.7444,0.7101,0.6887,0.6635,0.7086,0.6739,0.7126,0.6872,0.7052,0.6927,0.7178,0.7048,0.6359,0.5970,0.6375,0.5941,0.6380,0.5897,0.6789,0.6203
-90k,0.7483,0.7379,0.7379,0.6949,,,0.7443,0.7064,0.6917,0.6818,0.7079,0.6804,0.7148,0.6926,0.7106,0.6976,0.7146,0.6818,0.6400,0.6052,0.6327,0.5846,0.6521,0.6120,0.6781,0.6271
-95k,0.7510,0.7411,0.7427,0.6987,,,0.7376,0.6943,0.6901,0.6719,0.7097,0.6616,0.7115,0.6946,0.7221,0.6979,0.7240,0.6953,0.6388,0.5870,0.6373,0.5899,0.6460,0.5970,0.6798,0.6320
-100k,0.7550,0.7419,0.7437,0.7070,,,0.7457,0.7153,,,0.7060,0.6902,0.7117,0.6955,0.7167,0.7002,0.7241,0.7013,0.6447,0.6079,0.6431,0.5916,0.6490,0.6095,0.6854,0.6467
-105k,0.7547,0.7424,0.7445,0.7042,,,0.7476,0.7158,,,0.7141,0.6804,0.7132,0.6953,0.7222,0.6980,0.7263,0.6912,0.6470,0.6060,0.6473,0.5908,0.6588,0.6023,0.6809,0.6144
-110k,0.7605,0.7491,0.7540,0.7070,,,0.7486,0.7210,0.6942,0.6850,0.7107,0.6696,0.7166,0.6883,0.7221,0.7020,0.7284,0.7013,0.6482,0.6196,,,0.6620,0.6166,0.6888,0.6269
-115k,0.7626,0.7491,0.7540,0.7070,,,0.7522,0.7213,0.6957,0.6832,0.7120,0.6698,0.7179,0.6955,0.7284,0.7101,0.7274,0.7045,0.6511,0.6004,,,0.6636,0.5998,0.6882,0.6250
-120k,0.7641,0.7545,0.7532,0.7110,,,0.7520,0.7217,0.7022,0.6911,0.7139,0.6855,0.7224,0.7017,0.7132,0.6866,0.7329,0.7089,0.6532,0.6145,,,0.6611,0.6085,0.6874,0.6250
-125k,0.7636,0.7552,0.7538,0.7126,,,0.7533,0.7195,0.7029,0.6946,0.7211,0.6944,0.7221,0.6937,0.7250,0.7155,0.7285,0.7194,0.6571,0.6184,,,0.6624,0.6071,0.6896,0.6294
-130k,0.7619,0.7547,0.7539,0.7168,,,0.7573,0.7178,0.7032,0.6929,0.7195,0.6969,0.7261,0.7103,0.7320,0.7221,0.7337,0.7096,0.6593,0.6174,,,,,0.6929,0.6273
-135k,0.7641,0.7570,0.7543,0.7162,,,0.7580,0.7324,,,0.7177,0.6978,0.7198,0.6969,0.7249,0.7162,0.7324,0.7107,0.6584,0.6316,,,,,0.6941,0.6392
-140k,,,0.7615,0.7250,,,0.7596,0.7329,,,0.7236,0.7106,0.7245,0.7140,0.7306,0.7228,0.7338,0.7099,0.6577,0.6142,,,,,0.6925,0.6334
-145k,,,,,,,0.7573,0.7207,,,0.7194,0.7040,0.7247,0.7077,0.7347,0.7231,0.7431,0.7195,0.6628,0.6295,,,,,0.6984,0.6543
-150k,,,,,,,0.7614,0.7352,,,0.7170,0.7029,,,0.7304,0.7116,0.7386,0.7233,0.6592,0.6212,,,,,0.6978,0.6291
-155k,,,,,,,0.7579,0.7360,,,0.7245,0.7127,0.7294,0.7058,0.7378,0.7162,0.7448,0.7139,0.6662,0.6246,,,,,0.6929,0.6396
-160k,,,,,,,0.7606,0.7356,,,0.7199,0.6983,0.7279,0.7109,0.7343,0.7230,0.7385,0.7172,0.6666,0.6169,,,,,0.7009,0.6266
-165k,,,,,,,,0.7403,,,0.7249,0.7058,0.7297,0.7119,,,0.7493,0.7234,0.6680,0.6268,,,,,0.7003,0.6104
-170k,,,,,,,0.7696,0.7422,,,0.7262,0.7070,0.7323,0.7031,,,0.7499,0.7260,0.6710,0.6319,,,,,0.7010,0.6514
-175k,,,,,,,0.7745,0.7450,,,0.7303,0.7180,0.7338,0.7206,,,0.7502,0.7257,0.6707,0.6205,,,,,0.7047,0.6401
-180k,,,,,,,0.7676,0.7384,,,0.7299,0.7249,0.7316,0.7250,,,0.7457,0.7270,0.6721,0.6327,,,,,0.7079,0.6421
-185k,,,,,,,0.7678,0.7441,,,0.7319,0.7232,0.7354,0.7340,,,0.7519,0.7309,0.6732,0.6275,,,,,0.7050,0.6130
-190k,,,,,,,0.7701,0.7505,,,0.7336,0.7193,,,,,0.7493,0.7305,0.6729,0.6343,,,,,0.7097,0.6568
-195k,,,,,,,0.7730,0.7504,,,0.7293,0.7137,,,,,0.7579,0.7376,0.6774,0.6251,,,,,0.7074,0.6363
-200k,,,,,,,0.7753,0.7521,,,0.7366,0.7138,,,,,0.7567,0.7372,0.6795,0.6279,,,,,0.7122,0.6430
-205k,,,,,,,0.7744,0.7537,,,0.7360,0.7312,,,,,0.7560,0.7453,,0.6293,,,,,0.7175,0.6647
-210k,,,,,,,0.7729,0.7539,,,0.7368,0.7284,,,,,0.7658,0.7465,,0.6431,,,,,0.7179,0.6109
-215k,,,,,,,0.7804,0.7596,,,0.7359,0.7295,,,,,0.7621,0.7357,0.6819,0.6370,,,,,0.7136,0.6287
-220k,,,,,,,0.7752,0.7633,,,0.7384,0.7436,,,,,0.7678,0.7457,0.6860,0.6384,,,,,,
-225k,,,,,,,0.7808,0.7607,,,0.7340,0.7366,,,,,0.7649,0.7427,0.6805,0.6354,,,,,,
-230k,,,,,,,0.7786,0.7614,,,,,,,,,0.7662,0.7561,0.6855,0.6483,,,,,,
-235k,,,,,,,0.7844,0.7619,,,,,,,,,0.7676,0.7532,0.6880,0.6471,,,,,,
-240k,,,,,,,0.7866,0.7677,,,,,,,,,,,0.6841,0.6509,,,,,,
-245k,,,,,,,0.7857,0.7684,,,,,,,,,,,0.6850,0.6487,,,,,,
-250k,,,,,,,0.7851,0.7738,,,,,,,,,,,0.6892,0.6541,,,,,,
-255k,,,,,,,0.7845,0.7716,,,,,,,,,,,0.6875,0.6448,,,,,,
-260k,,,,,,,0.7893,0.7705,,,,,,,,,,,0.6945,0.6480,,,,,,
-265k,,,,,,,0.7918,0.7727,,,,,,,,,,,0.6933,0.6552,,,,,,
-270k,,,,,,,0.7917,0.7725,,,,,,,,,,,0.6980,0.6548,,,,,,
-275k,,,,,,,0.7925,0.7741,,,,,,,,,,,0.6950,0.6604,,,,,,
-280k,,,,,,,0.7943,0.7769,,,,,,,,,,,,0.6574,,,,,,
-285k,,,,,,,0.7946,0.7781,,,,,,,,,,,0.6970,0.6644,,,,,,
-290k,,,,,,,,,,,,,,,,,,,0.6970,0.6674,,,,,,
-300k,,,,,,,,,,,,,,,,,,,0.6969,0.6592,,,,,,
-305k,,,,,,,,,,,,,,,,,,,0.6997,0.6655,,,,,,
-310k,,,,,,,,,,,,,,,,,,,0.6988,0.6639,,,,,,
-315k,,,,,,,,,,,,,,,,,,,0.7023,0.6749,,,,,,
-320k,,,,,,,,,,,,,,,,,,,0.7012,0.6706,,,,,,
-325k,,,,,,,,,,,,,,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,,,,,,,,,,,,,,
+ga,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
+0-shot: 5 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
+5k,0.5622,0.5254,0.5324,0.5366
+10k,0.6433,0.5836,0.6046,0.6139
+15k,0.6716,0.6114,0.6336,0.6388
+20k,0.6855,0.6271,0.6492,0.6548
+25k,0.6945,0.6413,0.6665,0.6683
+30k,0.7059,,0.6746,0.6741
+35k,0.7158,0.6547,0.6832,0.6864
+40k,0.7184,0.6642,0.6821,0.6917
+45k,0.722,0.6698,0.6905,0.6933
+50k,0.725,0.6689,0.6964,0.7018
+55k,0.7305,0.6697,0.6959,0.7052
+60k,0.7236,0.6748,0.6904,0.704
+65k,0.7355,0.6752,0.7061,0.7074
+70k,0.7399,0.6773,0.7054,0.7074
+75k,0.7374,0.6854,0.7065,0.7027
+80k,0.7422,0.6862,0.7118,0.7139
+85k,0.7444,0.6887,0.7126,0.7178
+90k,0.7443,0.6917,0.7148,0.7146
+95k,0.7376,0.6901,0.7115,0.724
+100k,0.7457,,0.7117,0.7241
+105k,0.7476,,0.7132,0.7263
+110k,0.7486,0.6942,0.7166,0.7284
+115k,0.7522,0.6957,0.7179,0.7274
+120k,0.752,0.7022,0.7224,0.7329
+125k,0.7533,0.7029,0.7221,0.7285
+130k,0.7573,0.7032,0.7261,0.7337
+135k,0.758,,0.7198,0.7324
+140k,0.7596,,0.7245,0.7338
+145k,0.7573,,0.7247,0.7431
+150k,0.7614,,,0.7386
+155k,0.7579,,0.7294,0.7448
+160k,0.7606,,0.7279,0.7385
+165k,,,0.7297,0.7493
+170k,0.7696,,0.7323,0.7499
+175k,0.7745,,0.7338,0.7502
+180k,0.7676,,0.7316,0.7457
+185k,0.7678,,0.7354,0.7519
+190k,0.7701,,,0.7493
+195k,0.773,,,0.7579
+200k,0.7753,,,0.7567
+205k,0.7744,,,0.756
+210k,0.7729,,,0.7658
+215k,0.7804,,,0.7621
+220k,0.7752,,,0.7678
+225k,0.7808,,,0.7649
+230k,0.7786,,,0.7662
+235k,0.7844,,,0.7676
+240k,0.7866,,,
+245k,0.7857,,,
+250k,0.7851,,,
+255k,0.7845,,,
+260k,0.7893,,,
+265k,0.7918,,,
+270k,0.7917,,,
+275k,0.7925,,,
+280k,0.7943,,,
+285k,0.7946,,,
+290k,,,,
+300k,,,,
+305k,,,,
+310k,,,,
+315k,,,,
+320k,,,,
+325k,,,,
+330k,,,,
+335k,,,,

data/txt360_eval/CKPT Eval - MATH.csv

@@ -1,68 +0,0 @@
-5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
-time: 5 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
-5k,0.2335,0.2308,,0.2251,,0.2157,0.2221,0.2231,0.2211,0.2251,0.2191,0.2271,0.2238
-10k,0.2489,0.2519,,0.2379,0.2211,0.2332,0.2415,0.2342,0.2399,0.2285,0.2342,0.2402,0.2224
-15k,0.2626,0.2469,,0.2526,,0.2389,0.2322,0.2479,0.2580,0.2375,0.2271,0.2355,0.2375
-20k,0.2737,0.2606,,0.2469,0.2399,,0.2419,0.2526,0.2663,0.2469,0.2499,0.2439,0.2322
-25k,0.2700,0.2653,,0.2523,0.2395,0.2600,0.2526,0.2616,0.2559,0.2369,0.2476,0.2462,0.2355
-30k,0.2687,0.2556,,0.2402,,0.2452,0.2533,0.2606,0.2503,0.2456,0.2452,0.2446,0.2372
-35k,0.2765,0.2533,,0.2683,0.2596,0.2590,0.2509,0.2630,0.2737,0.2392,0.2405,0.2536,0.2402
-40k,0.2667,0.2683,,0.2496,0.2496,0.2593,0.2529,0.2697,0.2663,0.2379,0.2486,0.2526,0.2422
-45k,0.2750,0.2620,,0.2616,0.2586,0.2563,0.2503,0.2683,0.2673,0.2479,0.2496,0.2513,0.2472
-50k,0.2861,0.2697,,0.2693,0.2553,0.2596,0.2553,0.2700,0.2771,0.2442,0.2425,0.2546,0.2395
-55k,0.2848,0.2693,,0.2640,0.2630,0.2566,0.2479,0.2630,0.2757,0.2526,0.2506,0.2586,0.2509
-60k,0.2945,0.2784,,0.2727,0.2596,0.2633,0.2590,0.2690,0.2714,0.2519,0.2563,0.2553,0.2479
-65k,0.3008,0.2767,,0.2680,0.2623,0.2704,0.2610,0.2492,0.2727,0.2529,0.2559,0.2647,0.2462
-70k,0.2891,0.2824,,0.2730,0.2596,0.2710,0.2700,0.2677,0.2807,0.2469,0.2459,0.2626,0.2576
-75k,0.2982,0.2938,,0.2784,0.2647,0.2630,0.2697,0.2777,0.2620,0.2626,0.2499,0.2583,0.2549
-80k,0.2948,0.2801,,0.2737,0.2727,0.2643,0.2553,0.2657,0.2704,0.2509,0.2590,0.2549,0.2563
-85k,0.2992,0.2938,,0.2754,0.2620,0.2704,0.2677,0.2600,0.2771,0.2496,0.2385,0.2620,0.2529
-90k,0.3002,0.2888,,0.2764,0.2714,0.2737,0.2573,0.2693,0.2918,0.2616,0.2492,0.2566,0.2516
-95k,0.3025,0.2817,,0.2616,0.2690,0.2737,0.2523,0.2690,0.2791,0.2492,0.2576,0.2576,0.2549
-100k,0.2951,0.2894,,0.2616,,0.2817,0.2660,0.2757,0.2861,0.2546,0.2479,0.2667,0.2559
-105k,0.3052,0.2928,,0.2653,,0.2710,0.2707,0.2771,0.2868,0.2529,0.2482,0.2640,0.2633
-110k,0.3052,0.2985,,0.2600,0.2764,0.2781,0.2600,0.2764,0.2824,0.2536,,0.2727,0.2606
-115k,0.3025,0.2985,,0.2690,0.2791,0.2720,0.2704,0.2744,0.2918,0.2623,,0.2807,0.2496
-120k,0.3042,0.2985,,0.2750,0.2647,0.2650,0.2814,0.2754,0.2955,0.2677,,0.2626,0.2586
-125k,0.3149,0.3018,,0.2683,0.2707,0.2647,0.2757,0.2760,0.2804,0.2509,,0.2704,0.2496
-130k,0.3179,0.2978,,0.2781,0.2747,0.2653,0.2760,0.2774,0.2767,0.2593,,,0.2513
-135k,0.3226,0.2945,,0.2747,,0.2717,0.2673,0.2784,0.2884,0.2606,,,0.2533
-140k,,0.3018,,0.2771,,0.2757,0.2794,0.2787,0.2821,0.2459,,,0.2596
-145k,,,,0.2724,,0.2650,0.2720,0.2888,0.2801,0.2543,,,0.2633
-150k,,,,0.2720,,0.2814,,0.2864,0.2901,0.2590,,,0.2543
-155k,,,,,,0.2784,0.2720,0.2874,0.2938,0.2580,,,0.2566
-160k,,,,0.2817,,0.2834,0.2653,0.2807,0.2814,0.2563,,,0.2549
-165k,,,,0.2834,,0.2821,0.2804,,0.2955,0.2559,,,0.2536
-170k,,,,0.2854,,0.2824,0.2804,,0.3119,0.2536,,,0.2626
-175k,,,,0.2804,,0.2915,0.2750,,0.2988,0.2489,,,0.2657
-180k,,,,0.2767,,0.2901,0.2958,,0.3099,0.2623,,,0.2643
-185k,,,,0.2767,,0.2948,0.2804,,0.3055,0.2570,,,0.2643
-190k,,,,0.2787,,0.2925,,,0.3065,0.2573,,,0.2760
-195k,,,,0.2858,,0.2898,,,0.3119,0.2640,,,0.2657
-200k,,,,0.2771,,0.3028,,,0.3112,0.2610,,,0.2687
-205k,,,,0.2851,,0.2921,,,0.3002,0.2680,,,0.2667
-210k,,,,0.2838,,0.2817,,,0.3022,0.2650,,,0.2714
-215k,,,,0.2838,,0.2851,,,0.3069,0.2653,,,0.2600
-220k,,,,0.2938,,0.2814,,,0.3002,0.2549,,,
-225k,,,,0.2935,,0.2898,,,0.3049,0.2633,,,
-230k,,,,0.2888,,,,,0.3132,0.2653,,,
-235k,,,,0.3055,,,,,0.2951,0.2717,,,
-240k,,,,0.2995,,,,,,0.2667,,,
-245k,,,,0.2928,,,,,,0.2610,,,
-250k,,,,0.3092,,,,,,0.2650,,,
-255k,,,,0.3152,,,,,,0.2643,,,
-260k,,,,0.2951,,,,,,0.2616,,,
-265k,,,,0.3045,,,,,,0.2610,,,
-270k,,,,0.3018,,,,,,,,,
-275k,,,,0.3065,,,,,,,,,
-280k,,,,0.3015,,,,,,,,,
-285k,,,,0.2965,,,,,,0.2586,,,
-290k,,,,,,,,,,0.2623,,,
-300k,,,,,,,,,,0.2603,,,
-305k,,,,,,,,,,0.2630,,,
-310k,,,,,,,,,,0.2710,,,
-315k,,,,,,,,,,0.2677,,,
-320k,,,,,,,,,,0.2650,,,
-325k,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,

data/txt360_eval/CKPT Eval - MMLU.csv

@@ -1,68 +1,68 @@
-5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base,Comments
-time: 20 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192,"1. Comparing with upsample2-155k, the social science acc of dclm-195k is much higher"
-5k,0.2398,0.2671,,,0.2579,0.2418,0.2482,0.2690,0.2456,0.2512,0.2532,0.2428,0.2530,
-10k,0.2535,0.2520,,0.2594,0.2612,,0.2628,0.2319,0.2525,0.2462,0.2582,0.2713,0.2529,
-15k,0.2527,0.2347,,,,0.2489,0.2334,0.2483,0.2503,0.2549,0.2380,0.2653,0.2507,
-20k,0.2530,0.2478,,0.2495,0.2467,0.2677,0.2449,0.2507,0.2540,0.2416,0.2612,0.2482,0.2553,
-25k,0.2503,0.2488,,,0.2431,0.2597,0.2571,0.2506,0.2534,0.2505,0.2464,0.2577,0.2552,
-30k,0.2297,0.2539,,,,0.2592,0.2678,0.2389,0.2557,0.2468,0.2517,0.2485,0.2556,
-35k,0.2356,0.2374,,0.2426,0.2591,0.2550,0.2562,0.2594,0.2494,0.2403,0.2451,0.2547,0.2443,
-40k,0.2406,0.2462,,0.2467,0.2485,0.2344,0.2408,0.2555,0.2686,0.2552,0.2500,0.2553,0.2775,
-45k,0.2470,0.2428,,0.2418,0.2296,0.2512,0.2712,0.2630,0.2503,0.2368,0.2536,0.2557,0.2393,
-50k,0.2421,0.2368,,0.2382,0.2441,0.2727,0.2558,0.2558,0.2322,0.2499,0.2563,0.2305,0.2485,
-55k,0.2460,0.2551,,0.2408,0.2536,0.2389,0.2440,0.2444,0.2747,0.2552,0.2516,0.2339,0.2595,
-60k,0.2415,0.2397,,0.2718,0.2539,0.2518,0.2339,0.2551,0.2432,0.2517,0.2589,0.2379,0.2589,
-65k,0.2490,0.2641,,0.2637,0.2423,0.2589,0.2342,0.2303,0.2478,0.2485,0.2643,0.2485,0.2798,
-70k,0.2578,0.2641,,0.2534,0.2359,0.2716,0.2673,0.2307,0.2478,0.2483,0.2426,0.2499,0.2583,
-75k,0.2587,0.2599,,0.2529,0.2372,0.2514,0.2579,0.2519,0.2478,0.2742,0.2594,0.2371,0.2653,
-80k,0.2493,0.2519,,0.2504,0.2344,0.2582,0.2535,0.2433,0.2718,0.2596,0.2536,0.2553,0.2573,
-85k,0.2527,0.2789,,0.2547,0.2496,0.2564,0.2418,0.2572,0.2465,0.2663,0.2552,0.2485,0.2584,
-90k,0.2679,0.2668,,0.2595,0.2464,0.2608,0.2359,0.2777,0.2475,0.2543,0.2514,0.2411,0.2499,
-95k,0.2551,0.2763,,0.2621,0.2469,0.2505,0.2534,0.2584,0.2424,0.2607,0.2742,0.2385,0.2521,
-100k,0.2594,0.2564,,0.2550,,0.2614,0.2461,0.2611,0.2497,0.2675,0.2545,0.2540,0.2574,
-105k,0.2787,0.2473,,0.2659,,0.2542,0.2729,0.2666,0.2468,0.2610,0.2726,0.2465,0.2798,
-110k,0.3079,0.2458,,0.2551,0.2629,0.2512,0.2604,0.3027,0.2522,0.2673,,0.2410,0.2540,
-115k,0.3185,0.2458,,0.2624,0.2324,0.2569,0.2590,0.2863,0.2584,0.2624,,0.2396,0.2771,
-120k,0.3139,0.2832,,0.2626,0.2663,0.2718,0.2629,0.3190,0.2748,0.2419,,0.2544,0.2772,
-125k,0.2960,0.2928,,0.2712,0.2733,0.2663,0.2768,0.2788,0.2570,0.2616,,0.2466,0.2856,
-130k,0.3033,0.2844,,0.2404,0.2635,0.2767,0.2676,0.3191,0.2812,0.2538,,,0.2973,
-135k,0.2934,0.2895,,0.2641,,0.2713,0.2735,0.3119,0.2882,0.2661,,,0.3203,
-140k,,0.3045,,0.2553,,0.2811,0.2765,0.2866,0.3019,0.2730,,,0.2772,
-145k,,,,0.2492,,0.2850,0.2708,0.3107,0.3090,0.2582,,,0.3435,
-150k,,,,0.2595,,0.2780,,0.3225,0.3199,0.2541,,,0.3112,
-155k,,,,0.2681,,0.2664,0.2463,0.3618,0.3116,0.2594,,,0.3361,
-160k,,,,0.2605,,0.2793,0.2821,0.3047,0.3240,0.2688,,,0.3392,
-165k,,,,0.2725,,0.2933,0.2816,,0.3478,0.2653,,,0.3485,
-170k,,,,0.2514,,0.2656,0.2893,,0.3423,0.2537,,,0.3355,
-175k,,,,0.2535,,0.3007,0.3317,,0.3156,0.2621,,,0.3162,
-180k,,,,0.2561,,0.2785,0.2624,,0.2893,0.2555,,,0.3398,
-185k,,,,0.2523,,0.3131,0.3026,,0.3876,0.2461,,,0.3631,
-190k,,,,0.2653,,0.3226,,,0.3131,0.2540,,,0.3930,
-195k,,,,0.2681,,0.3136,,,0.3473,0.2550,,,0.3972,
-200k,,,,0.2515,,0.2811,,,0.3257,0.2481,,,0.3660,
-205k,,,,0.2619,,0.3004,,,0.3836,,,,0.3748,
-210k,,,,0.2687,,0.2996,,,0.3063,0.2646,,,0.3668,
-215k,,,,0.2653,,0.3329,,,0.3947,0.2626,,,0.3641,
-220k,,,,0.2631,,0.3590,,,0.3621,0.2600,,,,
-225k,,,,0.2737,,0.3453,,,0.4151,0.2589,,,,
-230k,,,,0.2833,,,,,0.3825,0.2587,,,,
-235k,,,,0.2703,,,,,0.3897,,,,,
-240k,,,,0.2572,,,,,,0.2610,,,,
-245k,,,,0.2700,,,,,,0.2612,,,,
-250k,,,,0.2639,,,,,,0.2583,,,,
-255k,,,,0.2680,,,,,,0.2564,,,,
-260k,,,,0.2897,,,,,,0.2631,,,,
-265k,,,,0.2815,,,,,,0.2635,,,,
-270k,,,,0.2693,,,,,,,,,,
-275k,,,,0.2789,,,,,,0.2643,,,,
-280k,,,,0.3052,,,,,,0.2687,,,,
-285k,,,,0.2850,,,,,,0.2605,,,,
-290k,,,,,,,,,,0.2779,,,,
-300k,,,,,,,,,,0.2755,,,,
-305k,,,,,,,,,,,,,,
-310k,,,,,,,,,,0.2614,,,,
-315k,,,,,,,,,,0.2646,,,,
-320k,,,,,,,,,,0.2745,,,,
-325k,,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,,
+5-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
+time: 20 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
+5k,,0.2579,0.2482,0.2456
+10k,0.2594,0.2612,0.2628,0.2525
+15k,,,0.2334,0.2503
+20k,0.2495,0.2467,0.2449,0.254
+25k,,0.2431,0.2571,0.2534
+30k,,,0.2678,0.2557
+35k,0.2426,0.2591,0.2562,0.2494
+40k,0.2467,0.2485,0.2408,0.2686
+45k,0.2418,0.2296,0.2712,0.2503
+50k,0.2382,0.2441,0.2558,0.2322
+55k,0.2408,0.2536,0.244,0.2747
+60k,0.2718,0.2539,0.2339,0.2432
+65k,0.2637,0.2423,0.2342,0.2478
+70k,0.2534,0.2359,0.2673,0.2478
+75k,0.2529,0.2372,0.2579,0.2478
+80k,0.2504,0.2344,0.2535,0.2718
+85k,0.2547,0.2496,0.2418,0.2465
+90k,0.2595,0.2464,0.2359,0.2475
+95k,0.2621,0.2469,0.2534,0.2424
+100k,0.255,,0.2461,0.2497
+105k,0.2659,,0.2729,0.2468
+110k,0.2551,0.2629,0.2604,0.2522
+115k,0.2624,0.2324,0.259,0.2584
+120k,0.2626,0.2663,0.2629,0.2748
+125k,0.2712,0.2733,0.2768,0.257
+130k,0.2404,0.2635,0.2676,0.2812
+135k,0.2641,,0.2735,0.2882
+140k,0.2553,,0.2765,0.3019
+145k,0.2492,,0.2708,0.309
+150k,0.2595,,,0.3199
+155k,0.2681,,0.2463,0.3116
+160k,0.2605,,0.2821,0.324
+165k,0.2725,,0.2816,0.3478
+170k,0.2514,,0.2893,0.3423
+175k,0.2535,,0.3317,0.3156
+180k,0.2561,,0.2624,0.2893
+185k,0.2523,,0.3026,0.3876
+190k,0.2653,,,0.3131
+195k,0.2681,,,0.3473
+200k,0.2515,,,0.3257
+205k,0.2619,,,0.3836
+210k,0.2687,,,0.3063
+215k,0.2653,,,0.3947
+220k,0.2631,,,0.3621
+225k,0.2737,,,0.4151
+230k,0.2833,,,0.3825
+235k,0.2703,,,0.3897
+240k,0.2572,,,
+245k,0.27,,,
+250k,0.2639,,,
+255k,0.268,,,
+260k,0.2897,,,
+265k,0.2815,,,
+270k,0.2693,,,
+275k,0.2789,,,
+280k,0.3052,,,
+285k,0.285,,,
+290k,,,,
+300k,,,,
+305k,,,,
+310k,,,,
+315k,,,,
+320k,,,,
+325k,,,,
+330k,,,,
+335k,,,,

data/txt360_eval/CKPT Eval - MedQA.csv

@@ -1,68 +1,68 @@
-0-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
-time: 3 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
-5k,0.2679,0.2946,,0.2152,,0.2781,0.2482,0.2160,0.2687,0.2639,0.2215,0.2584,0.2396
-10k,0.2749,0.2506,,0.2380,0.2372,0.2624,0.2616,0.2734,0.2718,0.2647,0.2435,0.2789,0.2435
-15k,0.2388,0.2773,,0.2270,,0.2749,0.2797,0.2545,0.2639,0.2726,0.2773,0.2647,0.2412
-20k,0.2412,0.2773,,0.2419,,,0.2317,0.2789,0.2757,0.2372,0.2506,0.2742,0.2577
-25k,0.2302,0.2624,,0.2184,,0.2561,0.2569,0.2804,0.2474,0.2616,0.2498,0.2624,0.2419
-30k,0.2545,0.2553,,0.2679,0.2522,0.2239,0.2097,0.2742,0.2608,0.2592,0.2694,0.2655,0.2765
-35k,0.2757,0.2577,,0.2647,0.2655,0.2671,0.2467,0.2907,0.2694,0.2797,0.2498,0.2749,0.2514
-40k,0.2710,0.2608,,0.2671,0.2396,0.2490,0.2569,0.2734,0.2482,0.2647,0.2742,0.2608,0.2671
-45k,0.2765,0.2506,,0.2742,0.2734,0.2600,0.2255,0.2859,0.2333,0.2671,0.2435,0.2474,0.2749
-50k,0.2742,0.2624,,0.2749,0.2537,0.2482,0.2372,0.2412,0.2655,0.2789,0.2412,0.2702,0.2577
-55k,0.2679,0.2545,,0.2797,0.2561,0.2632,0.2294,0.2718,0.2537,0.2647,0.2647,0.2757,0.2632
-60k,0.2773,0.2427,,0.2294,0.2325,0.2789,,0.2419,0.2639,0.2679,0.2506,0.2687,0.2419
-65k,0.2404,0.2687,,0.2663,0.2757,0.2310,0.2749,0.2836,0.2726,0.2734,0.2537,0.2608,0.2459
-70k,0.2600,0.2592,,0.2592,0.2757,0.2797,0.2632,0.2569,0.2435,0.2773,0.2765,0.2702,0.2584
-75k,0.2679,0.2584,,0.2490,0.2679,0.2789,0.2616,0.2710,0.2765,0.2742,0.2710,0.2687,0.2789
-80k,0.2632,0.2702,,0.2797,0.2419,0.2757,0.2522,0.2616,0.2789,0.2655,0.2694,0.2435,0.2757
-85k,0.2734,0.2451,,0.2655,0.2844,0.2608,0.2687,0.2742,0.2553,0.2663,0.2749,0.2639,0.2773
-90k,0.2797,0.2506,,0.2310,0.2364,0.2679,0.2624,,0.2679,0.2608,0.2561,0.2765,0.2820
-95k,0.2529,0.2545,,0.2742,0.2820,0.2797,0.2647,0.2757,0.2749,0.2663,0.2105,0.2655,0.2749
-100k,0.2694,0.2459,,0.2679,,0.2168,0.2702,0.2459,0.2663,0.2655,0.2537,0.2655,0.2781
-105k,0.2537,0.2529,,0.2655,,0.2773,0.2632,0.2592,0.2726,0.2687,0.2671,0.2749,0.2812
-110k,0.2663,0.2419,,0.2718,0.2474,0.2584,0.2537,0.2569,0.2537,0.2349,,0.2537,0.2765
-115k,0.2459,0.2419,,0.2655,0.2718,0.2773,0.2247,0.2852,0.2867,0.2490,,0.2561,0.2364
-120k,0.2624,0.2561,,0.2930,0.2537,0.2671,,0.2718,0.2844,0.2545,,0.2608,0.2443
-125k,0.2451,0.2742,,0.2624,0.2364,0.2451,0.2145,0.2985,0.2883,0.2726,,0.2498,0.2867
-130k,0.2655,0.2797,,0.2828,0.2412,0.2836,0.2891,0.2930,0.2922,0.2522,,,0.2765
-135k,0.2749,0.2655,,,,0.2443,0.2765,0.2883,0.2702,0.2679,,,0.2679
-140k,,0.2781,,0.2529,,0.2427,0.2545,0.2962,0.2930,0.2569,,,0.2820
-145k,,,,0.2490,,0.2427,0.2718,0.3048,0.3024,0.2639,,,0.2632
-150k,,,,,,0.2694,,0.2482,0.3244,0.2655,,,0.3150
-155k,,,,0.2608,,0.2789,0.2624,0.3134,,0.2490,,,0.3009
-160k,,,,0.2529,,0.2765,0.2726,0.3079,0.2852,0.2577,,,0.2757
-165k,,,,0.2388,,0.2592,0.2742,,0.2561,0.2380,,,0.3009
-170k,,,,0.2435,,0.2852,0.2506,,0.3056,0.2380,,,0.2836
-175k,,,,0.2632,,0.2757,0.2647,,0.3126,0.2671,,,0.2993
-180k,,,,0.2608,,0.2592,0.2899,,0.3166,0.2396,,,0.3071
-185k,,,,0.2710,,0.2859,0.2561,,0.3268,0.2537,,,0.2490
-190k,,,,0.2812,,0.2914,,,0.3040,0.2577,,,0.2828
-195k,,,,0.2482,,0.2797,,,0.3472,0.2694,,,0.2883
-200k,,,,0.2639,,0.2584,,,0.3339,0.2639,,,0.3126
-205k,,,,0.2514,,0.3158,,,0.3409,,,,0.2710
-210k,,,,0.2742,,0.3016,,,0.3378,0.2624,,,0.2962
-215k,,,,0.2592,,0.2859,,,0.3362,,,,0.2859
-220k,,,,0.2262,,0.3001,,,0.3559,0.2781,,,
-225k,,,,0.2490,,0.3134,,,0.3213,0.2608,,,
-230k,,,,0.2357,,,,,0.3472,0.2828,,,
-235k,,,,0.2514,,,,,0.3614,0.2639,,,
-240k,,,,0.2624,,,,,,0.2867,,,
-245k,,,,0.2482,,,,,,0.2718,,,
-250k,,,,0.2592,,,,,,0.2624,,,
-255k,,,,0.2537,,,,,,0.2781,,,
-260k,,,,0.2639,,,,,,0.2679,,,
-265k,,,,0.2844,,,,,,0.2616,,,
-270k,,,,0.2624,,,,,,,,,
-275k,,,,0.2757,,,,,,,,,
-280k,,,,0.2852,,,,,,0.2592,,,
-285k,,,,0.2726,,,,,,0.2781,,,
-290k,,,,,,,,,,0.2671,,,
-300k,,,,,,,,,,0.2742,,,
-305k,,,,,,,,,,0.2624,,,
-310k,,,,,,,,,,0.2718,,,
-315k,,,,,,,,,,0.2694,,,
-320k,,,,,,,,,,0.2749,,,
-325k,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,
+0-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
+time: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
+5k,0.2152,,0.2482,0.2687
+10k,0.238,0.2372,0.2616,0.2718
+15k,0.227,,0.2797,0.2639
+20k,0.2419,,0.2317,0.2757
+25k,0.2184,,0.2569,0.2474
+30k,0.2679,0.2522,0.2097,0.2608
+35k,0.2647,0.2655,0.2467,0.2694
+40k,0.2671,0.2396,0.2569,0.2482
+45k,0.2742,0.2734,0.2255,0.2333
+50k,0.2749,0.2537,0.2372,0.2655
+55k,0.2797,0.2561,0.2294,0.2537
+60k,0.2294,0.2325,,0.2639
+65k,0.2663,0.2757,0.2749,0.2726
+70k,0.2592,0.2757,0.2632,0.2435
+75k,0.249,0.2679,0.2616,0.2765
+80k,0.2797,0.2419,0.2522,0.2789
+85k,0.2655,0.2844,0.2687,0.2553
+90k,0.231,0.2364,0.2624,0.2679
+95k,0.2742,0.282,0.2647,0.2749
+100k,0.2679,,0.2702,0.2663
+105k,0.2655,,0.2632,0.2726
+110k,0.2718,0.2474,0.2537,0.2537
+115k,0.2655,0.2718,0.2247,0.2867
+120k,0.293,0.2537,,0.2844
+125k,0.2624,0.2364,0.2145,0.2883
+130k,0.2828,0.2412,0.2891,0.2922
+135k,,,0.2765,0.2702
+140k,0.2529,,0.2545,0.293
+145k,0.249,,0.2718,0.3024
+150k,,,,0.3244
+155k,0.2608,,0.2624,
+160k,0.2529,,0.2726,0.2852
+165k,0.2388,,0.2742,0.2561
+170k,0.2435,,0.2506,0.3056
+175k,0.2632,,0.2647,0.3126
+180k,0.2608,,0.2899,0.3166
+185k,0.271,,0.2561,0.3268
+190k,0.2812,,,0.304
+195k,0.2482,,,0.3472
+200k,0.2639,,,0.3339
+205k,0.2514,,,0.3409
+210k,0.2742,,,0.3378
+215k,0.2592,,,0.3362
+220k,0.2262,,,0.3559
+225k,0.249,,,0.3213
+230k,0.2357,,,0.3472
+235k,0.2514,,,0.3614
+240k,0.2624,,,
+245k,0.2482,,,
+250k,0.2592,,,
+255k,0.2537,,,
+260k,0.2639,,,
+265k,0.2844,,,
+270k,0.2624,,,
+275k,0.2757,,,
+280k,0.2852,,,
+285k,0.2726,,,
+290k,,,,
+300k,,,,
+305k,,,,
+310k,,,,
+315k,,,,
+320k,,,,
+325k,,,,
+330k,,,,
+335k,,,,

data/txt360_eval/CKPT Eval - NQ.csv

@@ -1,68 +1,68 @@
-5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base
-time: 22 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192
-5k,0.0615,0.0537,,0.0341,0.0416,0.0634,0.0565,0.0579,0.0526,0.0219,0.0213,0.0205,0.0274
-10k,0.1075,0.1053,,0.0715,,0.0906,0.0931,0.0828,0.0767,0.0391,0.0418,0.0529,0.0554
-15k,0.1382,0.1136,,0.0765,,0.1147,0.1061,0.1152,0.1127,0.0607,0.0560,0.0629,0.0587
-20k,0.1490,0.1393,,0.0787,,0.1161,0.1183,0.1285,0.1247,0.0529,0.0623,0.0668,0.0709
-25k,0.1687,0.1416,,0.0892,0.1150,0.1402,0.1352,0.1380,0.1343,0.0584,0.0687,0.0762,0.0828
-30k,0.1767,0.1557,,0.0911,0.1366,0.1454,0.1271,0.1501,0.1421,0.0723,0.0687,0.0723,0.0839
-35k,0.1706,0.1756,,0.0970,0.1488,0.1573,0.1485,0.1565,0.1524,0.0803,0.0798,0.0803,0.0778
-40k,0.1942,0.1759,,0.1028,0.1355,0.1560,0.1488,0.1554,0.1562,0.0759,0.0848,0.0845,0.0886
-45k,0.1798,0.1820,,0.1078,0.1488,0.1715,0.1620,0.1684,0.1598,0.0881,0.0911,0.0867,0.0848
-50k,0.1972,0.1809,,0.1050,0.1540,,0.1590,0.1657,0.1698,0.0864,0.0909,0.0909,0.0884
-55k,0.2158,0.1956,,0.1097,0.1607,0.1659,0.1662,0.1751,0.1704,0.0892,0.0898,0.0745,0.0931
-60k,0.2039,0.2036,,0.1211,0.1654,0.1734,0.1612,0.1745,0.1801,0.0817,0.0850,0.0922,0.0986
-65k,0.2244,0.2044,,0.1089,0.1573,0.1765,0.1693,0.1776,0.1823,0.0920,0.0967,0.1025,0.1066
-70k,0.2233,0.2233,,0.1222,0.1634,0.1845,0.1679,0.1859,0.1767,0.1022,0.0925,0.1039,0.1177
-75k,0.2305,0.2277,,0.1097,0.1709,0.1825,0.1881,0.1737,0.1762,0.1069,0.0936,0.1116,0.1199
-80k,0.2457,0.2252,,0.1277,0.1573,0.1900,0.1776,0.1787,0.1964,0.1047,0.0981,0.1033,0.1097
-85k,0.2501,0.2285,,0.1280,0.1776,0.1914,0.1889,0.1870,0.1889,0.0942,0.0964,0.1144,0.1213
-90k,0.2504,0.2521,,0.1158,0.1598,0.1911,0.1806,0.1898,0.1773,0.1058,0.0964,0.1186,0.1163
-95k,0.2579,0.2443,,0.1235,0.1762,0.1911,0.1781,0.1989,0.1917,0.1097,0.0928,0.1213,0.1169
-100k,0.2526,0.2446,,0.1258,,0.2097,0.1928,0.1903,0.1947,0.1125,0.1025,0.1127,0.1188
-105k,0.2679,0.2482,,0.1366,,0.2028,0.1814,0.1922,0.2094,0.1199,0.1069,0.1186,0.1269
-110k,0.2717,0.2562,,0.1377,0.1756,0.2019,0.1859,0.1975,,0.1152,,0.1252,0.1252
-115k,0.2745,0.2562,,0.1346,0.1831,0.1956,0.1947,0.1903,0.2119,0.1127,,0.1285,0.1111
-120k,0.2801,0.2612,,0.1402,0.2014,0.2000,,0.2044,0.2119,0.1188,,0.1166,0.1219
-125k,0.2751,0.2657,,0.1307,0.2030,0.2014,0.1992,0.2053,0.1787,0.1230,,0.1274,0.1418
-130k,0.2884,0.2673,,0.1368,0.1997,0.2125,0.1994,0.2011,0.2086,0.1127,,,0.1335
-135k,0.2842,0.2673,,0.1363,,0.2069,0.2014,0.2036,0.2069,0.1255,,,0.1299
-140k,,0.2679,,0.1435,,0.2039,0.1986,0.2042,0.2058,0.1263,,,0.1299
-145k,,,,0.1532,,0.2172,0.1953,0.2078,0.2102,0.1274,,,0.1443
-150k,,,,0.1404,,0.2125,,0.2127,0.2075,0.1263,,,0.1410
-155k,,,,0.1418,,0.2235,0.1931,0.2066,0.2205,0.1418,,,0.1460
-160k,,,,0.1346,,0.2183,0.2116,0.2069,0.2208,0.1319,,,0.1413
-165k,,,,0.1524,,0.2219,0.2139,,0.2213,0.1296,,,0.1424
-170k,,,,0.1388,,0.2175,,,0.2169,0.1366,,,0.1454
-175k,,,,0.1438,,0.2235,0.2222,,0.2321,0.1349,,,0.1399
-180k,,,,0.1471,,0.2260,0.2249,,0.236,0.1465,,,0.1421
-185k,,,,0.1499,,0.2341,0.2222,,0.2366,0.1449,,,0.1421
-190k,,,,0.1504,,0.2233,,,0.2274,0.1413,,,0.1471
-195k,,,,0.1554,,0.2330,,,0.2454,0.1440,,,0.1407
-200k,,,,0.1565,,0.2238,,,0.2346,0.1407,,,0.1449
-205k,,,,0.1726,,0.2271,,,0.2316,0.1382,,,0.1501
-210k,,,,0.1623,,0.2305,,,0.2493,0.1526,,,0.1424
-215k,,,,0.1576,,0.2299,,,0.2355,0.1518,,,0.1535
-220k,,,,0.1693,,0.2330,,,0.2427,0.1529,,,
-225k,,,,0.1596,,0.2366,,,0.2440,0.1479,,,
-230k,,,,0.1693,,,,,0.2554,0.1560,,,
-235k,,,,0.1720,,,,,0.2535,0.1540,,,
-240k,,,,0.1712,,,,,,0.1554,,,
-245k,,,,0.1704,,,,,,0.1532,,,
-250k,,,,0.1784,,,,,,0.1551,,,
-255k,,,,0.1740,,,,,,0.1623,,,
-260k,,,,0.1756,,,,,,0.1618,,,
-265k,,,,0.1886,,,,,,0.1604,,,
-270k,,,,0.1820,,,,,,0.1612,,,
-275k,,,,0.1870,,,,,,0.1629,,,
-280k,,,,0.1704,,,,,,0.1645,,,
-285k,,,,0.1903,,,,,,0.1665,,,
-290k,,,,,,,,,,0.1648,,,
-300k,,,,,,,,,,0.1712,,,
-305k,,,,,,,,,,0.1690,,,
-310k,,,,,,,,,,0.1712,,,
-315k,,,,,,,,,,,,,
-320k,,,,,,,,,,,,,
-325k,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,
+5-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
+time: 22 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
+5k,0.0341,0.0416,0.0565,0.0526
+10k,0.0715,,0.0931,0.0767
+15k,0.0765,,0.1061,0.1127
+20k,0.0787,,0.1183,0.1247
+25k,0.0892,0.115,0.1352,0.1343
+30k,0.0911,0.1366,0.1271,0.1421
+35k,0.097,0.1488,0.1485,0.1524
+40k,0.1028,0.1355,0.1488,0.1562
+45k,0.1078,0.1488,0.162,0.1598
+50k,0.105,0.154,0.159,0.1698
+55k,0.1097,0.1607,0.1662,0.1704
+60k,0.1211,0.1654,0.1612,0.1801
+65k,0.1089,0.1573,0.1693,0.1823
+70k,0.1222,0.1634,0.1679,0.1767
+75k,0.1097,0.1709,0.1881,0.1762
+80k,0.1277,0.1573,0.1776,0.1964
+85k,0.128,0.1776,0.1889,0.1889
+90k,0.1158,0.1598,0.1806,0.1773
+95k,0.1235,0.1762,0.1781,0.1917
+100k,0.1258,,0.1928,0.1947
+105k,0.1366,,0.1814,0.2094
+110k,0.1377,0.1756,0.1859,
+115k,0.1346,0.1831,0.1947,0.2119
+120k,0.1402,0.2014,,0.2119
+125k,0.1307,0.203,0.1992,0.1787
+130k,0.1368,0.1997,0.1994,0.2086
+135k,0.1363,,0.2014,0.2069
+140k,0.1435,,0.1986,0.2058
+145k,0.1532,,0.1953,0.2102
+150k,0.1404,,,0.2075
+155k,0.1418,,0.1931,0.2205
+160k,0.1346,,0.2116,0.2208
+165k,0.1524,,0.2139,0.2213
+170k,0.1388,,,0.2169
+175k,0.1438,,0.2222,0.2321
+180k,0.1471,,0.2249,0.236
+185k,0.1499,,0.2222,0.2366
+190k,0.1504,,,0.2274
+195k,0.1554,,,0.2454
+200k,0.1565,,,0.2346
+205k,0.1726,,,0.2316
+210k,0.1623,,,0.2493
+215k,0.1576,,,0.2355
+220k,0.1693,,,0.2427
+225k,0.1596,,,0.244
+230k,0.1693,,,0.2554
+235k,0.172,,,0.2535
+240k,0.1712,,,
+245k,0.1704,,,
+250k,0.1784,,,
+255k,0.174,,,
+260k,0.1756,,,
+265k,0.1886,,,
+270k,0.182,,,
+275k,0.187,,,
+280k,0.1704,,,
+285k,0.1903,,,
+290k,,,,
+300k,,,,
+305k,,,,
+310k,,,,
+315k,,,,
+320k,,,,
+325k,,,,
+330k,,,,
+335k,,,,

data/txt360_eval/CKPT Eval - PIQA.csv

@@ -1,69 +1,68 @@
-,Slim-Pajama 600B (bsz=4K x 1024),,,,,,FineWeb-1.5T,,Ours-Base,,Ours-Upsampling1,,Ours-Upsampling2,,Ours-Code-Upsampling2,,All-Upsampling1,,All-Upsampling1,,All-Upsampling1,,All-Upsampling1,,DCLM-Base,
-0-shot: 3 min,Llama-8x8B-baseline,,Llama-8x8B-seq8192,,Llama-8x8B-mup,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-1x8B-seq8192,,Llama_extend-1x8B-seq8192,,Jais-1x8B-seq8192,,Llama-1x8B-seq8192,
-5-shot: 4 min,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot
-5k,0.7236,0.7073,0.7176,0.7133,,,0.7470,0.7263,,0.7106,0.7280,0.7242,0.7378,0.7296,0.7356,0.7323,0.7318,0.7263,0.7078,0.7002,0.7057,0.6844,0.7116,0.6975,0.7046,0.6942
-10k,0.7492,0.7421,0.7427,0.7318,,,0.7650,0.7524,0.7573,0.7454,0.7568,0.7486,0.7557,0.7492,0.7726,0.7568,0.7612,0.7486,0.7198,0.7089,0.7209,0.7127,0.7280,0.7236,0.7367,0.7144
-15k,0.7688,0.7541,0.7639,0.7481,,,0.7775,0.7655,0.7628,,0.7748,0.7622,0.7769,0.7622,0.7786,0.7737,0.7655,0.7661,0.7367,0.7329,0.7378,0.7312,0.7350,0.7318,0.7443,0.7133
-20k,0.7639,0.7655,0.7682,0.7579,,,0.7807,0.7612,0.7671,0.7590,0.7845,,0.7709,0.7693,0.7813,0.7650,0.7840,0.7758,0.7465,0.7378,0.7470,0.7312,0.7486,0.7394,0.7514,0.7323
-25k,0.7639,0.7677,0.7682,0.7671,,,0.7878,0.7748,,0.7590,,0.7693,0.7913,0.7671,0.7818,0.7715,0.7791,0.7715,0.7470,0.7378,0.7503,0.7437,0.7470,0.7492,0.7497,0.7345
-30k,0.7764,0.7677,0.7797,0.7563,,,0.7862,0.7704,0.7780,0.7617,0.7802,0.7737,0.7829,0.7655,0.7813,0.7661,0.7889,0.7731,0.7524,0.7388,0.7497,0.7497,0.7546,0.7437,0.7563,0.7427
-35k,0.7802,0.7677,0.7769,0.7622,,,0.7933,0.7726,0.7769,0.7699,0.7878,0.7682,0.7824,0.7737,0.7813,0.7797,0.7987,0.7780,0.7573,0.7361,0.7508,0.7421,0.7612,0.7579,0.7655,0.7486
-40k,0.7873,0.7802,0.7802,0.7682,,,0.7905,0.7758,,0.7731,0.7889,0.7731,0.7943,0.7704,0.7835,0.7775,0.7878,0.7769,0.7573,0.7383,0.7579,0.7481,0.7606,0.7492,0.7650,0.7519
-45k,0.7813,0.7786,0.7764,0.7699,,,0.7982,0.7824,0.7786,0.7661,0.7911,0.7775,0.7829,0.7737,0.7894,0.7835,0.7949,0.7780,0.7579,0.7465,0.7639,0.7465,0.7612,0.7514,0.7677,0.7497
-50k,0.7818,0.7797,0.7878,0.7753,,,0.7992,0.7780,0.7775,0.7748,0.7856,0.7775,0.7943,0.7699,0.7998,0.7851,0.7933,0.7786,0.7557,0.7437,0.7524,0.7443,0.7677,0.7579,0.7802,0.7601
-55k,0.7900,0.7780,0.7905,0.7829,,,0.8079,0.7737,0.7786,0.7775,0.7878,0.7731,0.7884,0.7780,0.7976,0.7905,0.7943,0.7824,0.7661,0.7546,0.7655,0.7541,0.7704,0.7606,0.7704,0.7606
-60k,0.7916,0.7851,0.7911,0.7797,,,0.7922,0.7797,0.7818,0.7813,0.7900,0.7699,0.7905,0.7661,0.7943,0.7878,0.8003,0.7818,0.7650,0.7530,0.7628,0.7557,0.7661,0.7628,0.7661,0.7579
-65k,0.7938,0.7840,0.7927,0.7840,,,0.7976,0.7769,0.7900,0.7780,0.7933,0.7731,0.7835,0.7671,0.7960,0.7845,0.7943,0.7780,0.7720,0.7492,0.7606,0.7535,0.7748,0.7639,0.7704,0.7584
-70k,0.7922,0.7835,0.7922,0.7845,,,0.8052,0.7916,0.7916,0.7818,0.7949,0.7807,0.7900,0.7726,0.7889,0.7845,0.7976,0.7900,0.7633,0.7524,0.7612,0.7552,0.7644,0.7622,0.7699,0.7568
-75k,0.7938,0.7927,0.7949,0.7840,,,0.8030,0.7873,0.7878,0.7715,0.7938,0.7807,0.8079,0.7922,0.7927,0.7905,0.8020,0.7933,0.7655,0.7541,0.7682,0.7508,0.7737,0.7568,0.7737,0.7573
-80k,0.7911,0.7878,0.7873,0.7894,,,0.7971,0.7742,0.7829,0.7797,0.7987,0.7824,0.7992,0.7894,0.8003,0.7900,0.7933,0.7884,0.7671,0.7497,0.7682,0.7524,0.7748,0.7563,0.7742,0.7628
-85k,0.7949,0.7894,0.7900,0.7889,,,0.8003,0.7840,0.8014,0.7786,0.8025,0.7894,0.7949,0.7818,0.7992,0.7894,0.7965,0.7851,0.7682,0.7530,0.7731,0.7563,0.7829,0.7622,0.7780,0.7704
-90k,0.7982,0.7894,0.7916,0.7943,,,0.7976,0.7797,0.7873,0.7720,0.7971,0.7862,0.7856,0.7845,0.7960,0.7976,0.7998,0.7878,0.7731,0.7535,0.7650,0.7552,0.7737,0.7622,0.7742,0.7617
-95k,0.8058,0.7992,0.8020,0.7873,,,0.8041,0.7742,0.7905,0.7840,0.8014,0.7807,0.7954,0.7829,0.8025,0.7911,0.8003,0.7884,0.7709,0.7535,0.7699,0.7519,0.7731,0.7612,0.7753,0.7704
-100k,0.8069,0.7992,0.8052,0.7873,,,0.8069,0.7856,,,0.8041,0.7851,0.7998,0.7824,0.8014,0.7927,0.8009,0.7905,0.7628,0.7508,0.7715,0.7628,0.7748,0.7584,0.7758,0.7720
-105k,0.8058,0.7965,0.8025,0.7943,,,0.8074,0.7916,,,0.8030,0.7900,0.8063,0.7927,0.8036,0.7949,0.7960,0.7905,0.7688,0.7568,0.7644,0.7601,0.7753,0.7682,0.7797,0.7639
-110k,0.8041,0.7987,0.8069,0.7982,,,0.8085,0.7797,0.7856,0.7856,0.8009,0.7922,0.7938,0.7856,0.8020,0.7911,0.7998,0.7916,0.7682,0.7563,,,0.7791,0.7699,0.7845,0.7633
-115k,0.8090,0.8009,0.8069,0.7982,,,0.8118,0.7867,0.7911,0.7802,0.8020,0.7867,0.8041,0.7922,0.8052,0.7916,0.8052,0.7938,0.7612,0.7541,,,0.7780,0.7633,0.7709,0.7639
-120k,0.8145,0.7949,0.8041,0.7911,,,0.8074,0.7878,0.7982,0.7851,0.7976,0.7922,0.8025,0.7905,0.7938,0.7927,0.7949,0.7905,0.7704,0.7715,,,0.7813,0.7720,0.7867,0.7758
-125k,0.8079,0.8009,0.8058,0.7900,,,0.8107,0.7829,0.8009,0.7900,0.8020,0.7894,0.8047,0.7916,0.8047,0.7976,0.8003,0.7922,0.7677,0.7671,,,0.7824,0.7737,0.7764,0.7699
-130k,0.8069,0.8058,0.8041,0.7982,,,0.8079,0.7845,0.7916,0.7797,0.8036,0.7916,0.8014,0.7949,0.8058,0.8014,0.7922,0.7943,0.7835,0.7622,,,,,0.7748,0.7720
-135k,0.8063,0.8047,0.8090,0.8020,,,0.8074,0.7878,,,0.8009,0.7878,0.8052,0.7835,0.8014,0.8030,0.8014,0.7927,0.7764,0.7682,,,,,0.7867,0.7813
-140k,,,0.8090,0.7992,,,0.8123,0.7911,,,0.8047,0.7916,0.8063,0.7971,0.8079,0.8036,0.7987,0.7976,0.7764,0.7628,,,,,0.7862,0.7720
-145k,,,,,,,0.8069,0.7807,,,0.8047,0.7922,0.8052,0.7845,0.7982,0.8025,0.8030,0.8085,0.7748,0.7688,,,,,0.7791,0.7699
-150k,,,,,,,0.8058,0.7949,,,0.8058,0.7878,,,0.8090,0.7998,0.7987,0.8025,0.7693,0.7579,,,,,0.7916,0.7769
-155k,,,,,,,0.8096,0.8041,,,0.8096,0.7922,0.7954,0.7775,0.8101,0.8041,0.8107,0.7965,0.7769,0.7639,,,,,0.7933,0.7731
-160k,,,,,,,0.8101,0.7900,,,0.8014,0.7976,0.8020,0.7894,0.8128,0.8036,0.8079,0.8009,0.7753,0.7715,,,,,0.7987,0.7709
-165k,,,,,,,0.8112,0.7933,,,0.8030,0.7971,0.8058,0.7878,,,0.8101,0.8009,0.7824,0.7709,,,,,0.7873,0.7682
-170k,,,,,,,,0.7916,,,0.8047,0.7954,0.8041,0.7922,,,0.8036,0.8041,0.7797,0.7720,,,,,0.7884,0.7715
-175k,,,,,,,0.8194,0.7965,,,0.8030,0.7911,0.7982,0.7927,,,0.8118,0.8096,0.7709,0.7666,,,,,0.7911,0.7802
-180k,,,,,,,0.8118,0.7845,,,0.8041,0.7954,0.8025,0.7987,,,0.8172,0.7976,0.7775,0.7677,,,,,0.7884,0.7851
-185k,,,,,,,0.8259,0.7982,,,0.8025,0.7960,0.8036,0.7905,,,0.8096,0.7987,0.7851,0.7737,,,,,0.7927,0.7813
-190k,,,,,,,0.8139,0.8025,,,0.7998,0.7987,,,,,0.8128,0.7998,0.7840,0.7758,,,,,0.7922,0.7867
-195k,,,,,,,0.8188,0.7965,,,0.8090,0.7878,,,,,0.8161,0.8052,0.7748,0.7677,,,,,0.7884,0.7769
-200k,,,,,,,0.8112,0.8025,,,0.8079,0.8009,,,,,0.8128,0.8041,0.7802,0.7726,,,,,0.7916,0.7802
-205k,,,,,,,0.8188,0.8009,,,0.8003,0.7938,,,,,0.8177,0.8145,0.7813,0.7726,,,,,0.7949,0.7748
-210k,,,,,,,0.8188,0.7971,,,0.8047,0.7889,,,,,0.8161,0.8101,0.7818,0.7786,,,,,0.7894,0.7867
-215k,,,,,,,0.8188,0.7992,,,0.8030,0.7922,,,,,0.8085,0.8085,0.7813,0.7748,,,,,0.7845,0.7802
-220k,,,,,,,0.8199,0.8030,,,0.8085,0.7976,,,,,0.8096,0.8074,0.7769,0.7704,,,,,,
-225k,,,,,,,0.8199,0.8041,,,0.8052,0.8014,,,,,0.8134,0.8101,0.7829,0.7731,,,,,,
-230k,,,,,,,0.8172,0.8041,,,,,,,,,0.8134,0.8107,0.7824,0.7802,,,,,,
-235k,,,,,,,0.8199,0.8085,,,,,,,,,0.8205,0.8118,0.7813,0.7764,,,,,,
-240k,,,,,,,0.8166,0.8101,,,,,,,,,,,0.7829,0.7824,,,,,,
-245k,,,,,,,0.8215,0.8090,,,,,,,,,,,0.7873,0.7753,,,,,,
-250k,,,,,,,0.8172,0.8107,,,,,,,,,,,0.7807,0.7797,,,,,,
-255k,,,,,,,0.8254,0.8128,,,,,,,,,,,0.7824,0.7737,,,,,,
-260k,,,,,,,0.8215,0.809,,,,,,,,,,,0.7807,0.7797,,,,,,
-265k,,,,,,,0.8210,0.8139,,,,,,,,,,,0.7775,0.7753,,,,,,
-270k,,,,,,,0.8145,0.8079,,,,,,,,,,,0.7824,,,,,,,
-275k,,,,,,,0.8161,0.8139,,,,,,,,,,,0.7889,0.7769,,,,,,
-280k,,,,,,,0.8248,0.8150,,,,,,,,,,,0.7807,0.7726,,,,,,
-285k,,,,,,,0.8210,0.8101,,,,,,,,,,,0.7916,0.7818,,,,,,
-290k,,,,,,,,,,,,,,,,,,,0.7851,0.7758,,,,,,
-300k,,,,,,,,,,,,,,,,,,,0.7840,0.7780,,,,,,
-305k,,,,,,,,,,,,,,,,,,,0.7873,0.7829,,,,,,
-310k,,,,,,,,,,,,,,,,,,,0.7813,0.7829,,,,,,
-315k,,,,,,,,,,,,,,,,,,,0.7851,0.7791,,,,,,
-320k,,,,,,,,,,,,,,,,,,,0.7873,0.7813,,,,,,
-325k,,,,,,,,,,,,,,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,,,,,,,,,,,,,,
+,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
+0-shot: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
+5k,0.747,,0.7378,0.7318
+10k,0.765,0.7573,0.7557,0.7612
+15k,0.7775,0.7628,0.7769,0.7655
+20k,0.7807,0.7671,0.7709,0.784
+25k,0.7878,,0.7913,0.7791
+30k,0.7862,0.778,0.7829,0.7889
+35k,0.7933,0.7769,0.7824,0.7987
+40k,0.7905,,0.7943,0.7878
+45k,0.7982,0.7786,0.7829,0.7949
+50k,0.7992,0.7775,0.7943,0.7933
+55k,0.8079,0.7786,0.7884,0.7943
+60k,0.7922,0.7818,0.7905,0.8003
+65k,0.7976,0.79,0.7835,0.7943
+70k,0.8052,0.7916,0.79,0.7976
+75k,0.803,0.7878,0.8079,0.802
+80k,0.7971,0.7829,0.7992,0.7933
+85k,0.8003,0.8014,0.7949,0.7965
+90k,0.7976,0.7873,0.7856,0.7998
+95k,0.8041,0.7905,0.7954,0.8003
+100k,0.8069,,0.7998,0.8009
+105k,0.8074,,0.8063,0.796
+110k,0.8085,0.7856,0.7938,0.7998
+115k,0.8118,0.7911,0.8041,0.8052
+120k,0.8074,0.7982,0.8025,0.7949
+125k,0.8107,0.8009,0.8047,0.8003
+130k,0.8079,0.7916,0.8014,0.7922
+135k,0.8074,,0.8052,0.8014
+140k,0.8123,,0.8063,0.7987
+145k,0.8069,,0.8052,0.803
+150k,0.8058,,,0.7987
+155k,0.8096,,0.7954,0.8107
+160k,0.8101,,0.802,0.8079
+165k,0.8112,,0.8058,0.8101
+170k,,,0.8041,0.8036
+175k,0.8194,,0.7982,0.8118
+180k,0.8118,,0.8025,0.8172
+185k,0.8259,,0.8036,0.8096
+190k,0.8139,,,0.8128
+195k,0.8188,,,0.8161
+200k,0.8112,,,0.8128
+205k,0.8188,,,0.8177
+210k,0.8188,,,0.8161
+215k,0.8188,,,0.8085
+220k,0.8199,,,0.8096
+225k,0.8199,,,0.8134
+230k,0.8172,,,0.8134
+235k,0.8199,,,0.8205
+240k,0.8166,,,
+245k,0.8215,,,
+250k,0.8172,,,
+255k,0.8254,,,
+260k,0.8215,,,
+265k,0.821,,,
+270k,0.8145,,,
+275k,0.8161,,,
+280k,0.8248,,,
+285k,0.821,,,
+290k,,,,
+300k,,,,
+305k,,,,
+310k,,,,
+315k,,,,
+320k,,,,
+325k,,,,
+330k,,,,
+335k,,,,

data/txt360_eval/CKPT Eval - TriviaQA.csv

@@ -1,68 +1,68 @@
-5-shot,Slim-Pajama 600B (bsz=4K x 1024),,,FineWeb-1.5T,Ours-Base,Ours-Upsampling1,Ours-Upsampling2,Ours-Code-Upsampling2,All-Upsampling1,All-Upsampling1,All-Upsampling1,All-Upsampling1,DCLM-Base,Comments
-time: 76 min,Llama-8x8B-baseline,Llama-8x8B-seq8192,Llama-8x8B-mup,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-1x8B-seq8192,Llama_extend-1x8B-seq8192,Jais-1x8B-seq8192,Llama-1x8B-seq8192,1. Takes 25min to load checkpoints.
-5k,0.1944,0.1764,,0.1025,0.1232,0.1086,0.1260,0.1647,,0.0841,0.0762,0.0824,0.1066,2. GPU utility is only 20%
-10k,0.3372,0.3292,,0.2073,,0.2636,0.1150,0.2659,0.2604,0.1348,0.1343,0.1585,0.1850,
-15k,0.4050,0.3909,,0.3005,,0.3250,0.1872,0.3445,0.3244,0.1821,0.1930,0.1968,0.2443,
-20k,0.4451,0.4497,,0.3506,0.2795,,0.2719,0.3802,0.3637,0.2086,0.2196,0.2231,0.2772,
-25k,0.4899,0.4601,,0.3070,,0.3975,0.4093,0.4105,0.4120,0.2261,0.2375,0.2574,0.3146,
-30k,0.5125,0.4824,,0.2461,0.2974,0.0303,0.4195,0.4330,0.4294,0.2352,0.2484,0.2675,0.3328,
-35k,0.5249,0.5091,,0.3639,0.3572,0.1983,0.3587,0.4434,0.4428,0.2433,,0.2863,0.3507,
-40k,0.5555,0.5166,,0.3537,0.0346,0.4571,0.4434,0.4618,0.4623,0.2708,0.2828,0.3020,0.3606,
-45k,0.5664,0.5403,,0.3602,0.2674,0.2654,0.4366,0.4746,0.4792,0.2668,0.3018,0.3065,0.3726,
-50k,0.5690,0.5217,,0.2407,0.3689,0.4355,0.4051,0.4885,0.4795,0.2906,0.2952,0.3187,0.3807,
-55k,0.5843,0.5680,,0.2081,0.4101,0.4341,0.3230,0.4931,0.4940,0.2940,0.3117,0.3242,0.3984,
-60k,0.5916,0.5814,,0.4068,0.4107,0.4861,0.4469,0.4955,0.5130,0.3137,0.3090,0.3422,0.4081,
-65k,0.6032,0.5774,,0.3145,0.4477,0.4858,0.4907,0.5039,0.5087,0.3097,0.3184,0.3397,0.4156,
-70k,0.6030,0.5920,,0.4102,0.4736,0.5080,0.4920,0.5164,0.5129,0.3236,0.3360,0.3375,0.4242,
-75k,0.6216,0.6187,,0.2820,0.4226,0.4777,0.2245,0.5190,0.5042,0.3265,0.3341,0.3483,0.4220,
-80k,0.6397,0.6127,,0.0975,0.4217,0.3698,,0.5185,0.5301,0.3352,0.3412,0.3532,0.4306,
-85k,0.6416,0.6254,,0.0722,0.4763,0.3700,0.5029,0.5249,0.5350,0.3448,0.3423,0.3530,0.4340,
-90k,0.6510,0.6317,,0.3388,0.1472,0.4793,0.0317,0.5337,0.5220,0.3440,0.3559,0.3644,0.4418,
-95k,0.6655,0.6479,,0.5283,0.4938,0.5144,0.5180,0.5432,0.5446,0.3331,0.3393,0.3683,0.4454,
-100k,0.6723,0.6486,,0.4317,0.1100,0.5121,0.5358,0.5383,0.5514,0.3520,0.3544,0.3698,0.4378,
-105k,0.6755,0.6582,,0.1886,,0.5280,0.5153,0.5499,0.5562,0.3626,0.3642,0.3683,0.4525,
-110k,0.6798,0.6668,,0.3510,,0.5468,0.5182,0.5541,0.5654,0.3694,,0.3903,0.4566,
-115k,0.6796,0.6668,,0.3692,0.4759,0.5347,0.5132,0.5508,0.5577,0.3741,,0.3908,0.4482,
-120k,0.6822,0.6688,,0.3690,0.4352,0.5376,0.5483,0.5567,0.5658,0.3881,,0.3950,0.4524,
-125k,0.6894,0.6743,,0.3365,0.5206,0.4855,0.5211,0.5617,0.5658,0.3725,,0.3880,0.4592,
-130k,0.6914,0.6709,,0.3550,0.0088,0.5238,0.5245,0.5597,0.5609,0.3698,,,0.4594,
-135k,0.6915,0.6721,,0.3892,,0.5467,0.3977,0.5541,0.5774,0.3782,,,0.4636,
-140k,,0.6773,,0.3930,,0.3110,0.4991,0.5572,0.5675,0.3906,,,0.4741,
-145k,,,,0.4538,,0.5720,0.4872,0.5642,0.5639,,,,0.4720,
-150k,,,,0.2883,,0.5612,,0.5701,0.5844,0.3899,,,0.4651,
-155k,,,,0.4185,,0.5030,0.1586,0.5790,0.5755,0.4044,,,0.4784,
-160k,,,,0.2720,,0.5701,0.5630,0.5819,0.5864,0.4049,,,0.4665,
-165k,,,,0.4252,,0.5388,0.5642,,0.5853,0.4007,,,0.4793,
-170k,,,,0.1507,,0.5951,0.5739,,,0.4150,,,0.4846,
-175k,,,,0.3242,,0.5437,0.5640,,0.5979,0.4092,,,0.4908,
-180k,,,,0.2653,,0.5580,0.5912,,0.6054,0.4189,,,,
-185k,,,,0.2651,,0.5709,0.5852,,0.6064,,,,0.5030,
-190k,,,,0.2380,,0.5142,,,0.5996,0.4193,,,0.5115,
-195k,,,,0.4048,,0.5964,,,0.6243,0.4265,,,,
-200k,,,,0.5058,,0.5684,,,0.6248,0.4256,,,,
-205k,,,,0.0945,,0.5878,,,0.6224,0.4190,,,0.5105,
-210k,,,,0.1557,,0.6020,,,0.6311,0.4415,,,0.5164,
-215k,,,,0.2483,,0.5995,,,0.6293,0.4353,,,0.5163,
-220k,,,,0.1725,,0.5924,,,0.6375,,,,,
-225k,,,,0.2467,,0.4832,,,0.6340,0.4556,,,,
-230k,,,,0.1653,,,,,0.6436,0.4622,,,,
-235k,,,,0.1884,,,,,0.6411,0.4608,,,,
-240k,,,,0.0719,,,,,,0.4536,,,,
-245k,,,,0.3757,,,,,,0.4641,,,,
-250k,,,,0.5859,,,,,,,,,,
-255k,,,,0.4987,,,,,,0.4741,,,,
-260k,,,,0.3940,,,,,,0.4712,,,,
-265k,,,,0.3607,,,,,,0.4767,,,,
-270k,,,,0.3898,,,,,,0.4795,,,,
-275k,,,,0.4123,,,,,,,,,,
-280k,,,,0.2413,,,,,,0.4787,,,,
-285k,,,,0.3665,,,,,,0.4843,,,,
-290k,,,,,,,,,,0.4818,,,,
-300k,,,,,,,,,,0.4969,,,,
-305k,,,,,,,,,,0.4941,,,,
-310k,,,,,,,,,,0.4963,,,,
-315k,,,,,,,,,,,,,,
-320k,,,,,,,,,,,,,,
-325k,,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,,
+5-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
+time: 76 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
+5k,0.1025,0.1232,0.126,
+10k,0.2073,,0.115,0.2604
+15k,0.3005,,0.1872,0.3244
+20k,0.3506,0.2795,0.2719,0.3637
+25k,0.307,,0.4093,0.412
+30k,0.2461,0.2974,0.4195,0.4294
+35k,0.3639,0.3572,0.3587,0.4428
+40k,0.3537,0.0346,0.4434,0.4623
+45k,0.3602,0.2674,0.4366,0.4792
+50k,0.2407,0.3689,0.4051,0.4795
+55k,0.2081,0.4101,0.323,0.494
+60k,0.4068,0.4107,0.4469,0.513
+65k,0.3145,0.4477,0.4907,0.5087
+70k,0.4102,0.4736,0.492,0.5129
+75k,0.282,0.4226,0.2245,0.5042
+80k,0.0975,0.4217,,0.5301
+85k,0.0722,0.4763,0.5029,0.535
+90k,0.3388,0.1472,0.0317,0.522
+95k,0.5283,0.4938,0.518,0.5446
+100k,0.4317,0.11,0.5358,0.5514
+105k,0.1886,,0.5153,0.5562
+110k,0.351,,0.5182,0.5654
+115k,0.3692,0.4759,0.5132,0.5577
+120k,0.369,0.4352,0.5483,0.5658
+125k,0.3365,0.5206,0.5211,0.5658
+130k,0.355,0.0088,0.5245,0.5609
+135k,0.3892,,0.3977,0.5774
+140k,0.393,,0.4991,0.5675
+145k,0.4538,,0.4872,0.5639
+150k,0.2883,,,0.5844
+155k,0.4185,,0.1586,0.5755
+160k,0.272,,0.563,0.5864
+165k,0.4252,,0.5642,0.5853
+170k,0.1507,,0.5739,
+175k,0.3242,,0.564,0.5979
+180k,0.2653,,0.5912,0.6054
+185k,0.2651,,0.5852,0.6064
+190k,0.238,,,0.5996
+195k,0.4048,,,0.6243
+200k,0.5058,,,0.6248
+205k,0.0945,,,0.6224
+210k,0.1557,,,0.6311
+215k,0.2483,,,0.6293
+220k,0.1725,,,0.6375
+225k,0.2467,,,0.634
+230k,0.1653,,,0.6436
+235k,0.1884,,,0.6411
+240k,0.0719,,,
+245k,0.3757,,,
+250k,0.5859,,,
+255k,0.4987,,,
+260k,0.394,,,
+265k,0.3607,,,
+270k,0.3898,,,
+275k,0.4123,,,
+280k,0.2413,,,
+285k,0.3665,,,
+290k,,,,
+300k,,,,
+305k,,,,
+310k,,,,
+315k,,,,
+320k,,,,
+325k,,,,
+330k,,,,
+335k,,,,

data/txt360_eval/CKPT Eval - WinoGrande.csv

@@ -1,69 +1,68 @@
-,Slim-Pajama 600B (bsz=4K x 1024),,,,,,FineWeb-1.5T,,Ours-Base,,Ours-Upsampling1,,Ours-Upsampling2,,Ours-Code-Upsampling2,,All-Upsampling1,,All-Upsampling1,,All-Upsampling1,,All-Upsampling1,,DCLM-Base,
-0-shot: 3 min,Llama-8x8B-baseline,,Llama-8x8B-seq8192,,Llama-8x8B-mup,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-8x8B-seq8192,,Llama-1x8B-seq8192,,Llama_extend-1x8B-seq8192,,Jais-1x8B-seq8192,,Llama-1x8B-seq8192,
-5-shot: 3 min,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot,0-shot,5-shot
-5k,0.5454,0.5367,0.5572,0.5367,,,0.5691,0.5335,0.5351,0.5233,0.5241,,0.5367,0.5391,0.5470,0.5414,0.5383,0.5320,0.5067,0.5201,0.5257,0.5122,0.5020,0.5217,0.5217,0.5312
-10k,0.5801,0.6054,0.5730,0.5525,,,0.5904,0.5620,0.5604,0.5738,0.5825,0.5691,0.5817,0.5809,0.5777,0.5706,0.5667,0.5620,0.5383,0.5375,0.5241,0.5430,0.5343,0.5241,0.5722,0.5517
-15k,0.6172,0.6038,0.5848,0.6046,,,0.5927,0.5951,0.5919,0.5699,,0.6014,0.5880,0.5856,0.5991,0.5943,0.5896,0.5841,0.5596,0.5470,0.5391,0.5414,0.5509,0.5454,0.5620,0.5612
-20k,0.6109,0.6267,0.5935,0.6085,,,0.6448,0.6022,0.6006,,0.6204,,0.6180,0.6235,0.6014,0.6062,0.5935,0.6014,0.5580,0.5691,0.5533,0.5328,0.5659,0.5580,0.5943,0.5714
-25k,0.6417,0.6369,0.5998,0.6140,,,0.6196,0.6164,0.6125,0.5998,0.6093,0.6117,0.6062,0.6117,0.6212,0.6014,0.6101,0.6188,0.5785,0.5770,0.5556,0.5572,0.5762,0.5691,0.5904,0.5864
-30k,0.6377,0.6456,0.6251,0.6361,,,0.6488,0.6251,,,0.6330,0.5983,0.6140,0.6022,0.6338,0.6109,0.6322,0.6188,0.5667,0.5770,0.5612,0.5493,0.5919,0.5604,0.6054,0.5935
-35k,0.6472,0.6456,0.6196,0.5935,,,0.6440,0.6227,0.6030,0.6172,0.6393,0.6243,0.6259,0.6093,0.6172,0.6164,0.6212,0.6188,0.5817,0.5572,0.5848,0.5612,0.5777,0.5714,0.6172,0.5975
-40k,0.6606,0.6630,0.6369,0.6259,,,0.6496,0.6077,0.6338,0.5951,,,0.6267,0.6101,0.6417,0.6235,0.6433,0.6361,0.5856,0.5770,0.5817,0.5777,0.5927,0.5635,0.6117,0.5959
-45k,0.6717,0.6496,0.6196,0.6425,,,0.6456,0.6417,0.6172,0.6109,0.6448,0.6235,0.6393,0.6417,0.6425,0.6369,0.6393,0.6346,0.5777,0.5699,0.5872,0.5809,0.5951,0.5833,0.6156,0.6085
-50k,0.6756,0.6725,0.6369,0.6377,,,0.6464,0.6275,0.6401,0.5975,0.6322,0.6322,0.6164,0.614,0.6425,0.6425,0.6472,0.6393,0.5833,0.5683,0.5872,0.5675,0.5967,0.5580,0.6338,0.6022
-55k,0.6661,0.6590,0.6496,0.6614,,,0.6567,0.6314,0.6235,0.6062,0.6519,0.6133,0.6314,0.6235,0.6377,0.648,0.6464,0.6322,0.5738,0.5612,0.5935,0.5722,0.6148,0.5533,0.6140,0.614
-60k,0.6622,0.6511,0.6377,0.6582,,,0.6480,0.6464,0.6251,0.6117,0.6535,0.6251,0.6219,0.6235,0.6480,0.6196,0.6369,0.6338,0.5864,0.5730,0.5746,0.5683,0.5896,0.5785,0.6062,0.6283
-65k,0.6669,0.6772,0.6590,0.6685,,,0.6654,0.6354,0.6283,0.6180,0.6519,0.6196,0.6401,0.6393,0.6559,0.633,0.6504,0.6275,0.5919,0.5754,0.5825,0.5793,0.6101,0.5959,0.6290,0.6125
-70k,0.6827,0.6811,0.6567,0.6701,,,0.6709,0.6401,0.6322,0.6235,0.6622,,0.6417,0.6409,0.6433,0.6338,0.6559,0.6361,0.5975,0.5927,0.5738,0.5588,0.5983,0.5738,0.6330,0.6101
-75k,0.6788,0.6819,0.6543,0.6685,,,0.6709,0.6283,0.6480,0.6172,0.6654,0.6409,0.6527,0.6267,0.6488,0.6551,0.6527,0.6638,0.5959,0.5817,0.5880,0.5517,0.6164,0.5793,0.6196,0.6164
-80k,0.6835,0.6882,0.6748,0.6638,,,0.6843,0.6464,0.6504,0.6275,0.6677,0.6401,0.6369,0.6298,0.6606,0.6488,0.6519,0.6440,0.5872,0.5675,0.5856,0.5564,0.6148,0.6038,0.6188,0.5983
-85k,0.6867,0.6882,0.6638,0.6590,,,0.6875,0.6504,0.6409,0.6188,,0.6535,0.6575,0.6283,0.6606,0.6393,0.6393,0.6543,0.6085,0.5919,0.5872,0.5738,0.6156,0.5817,0.6361,0.6196
-90k,0.6827,0.6803,0.6740,0.6598,,,0.6740,0.6393,0.6369,0.6306,0.6496,0.6543,0.6488,0.6409,0.6606,0.6314,0.6527,0.6409,0.5927,0.5825,0.5951,0.5888,0.6148,0.6077,0.6259,0.6164
-95k,0.6859,0.6859,0.6764,0.6575,,,0.6835,0.6401,0.6369,0.6361,0.6551,0.6322,0.6654,0.6338,0.6630,0.6322,0.6409,0.6582,0.6156,0.5927,0.5864,0.5919,0.6164,0.5809,0.6283,0.6014
-100k,0.6898,,0.6661,0.6851,,,0.6756,0.6567,,,0.6567,0.6472,0.6590,0.6488,0.6748,0.6204,0.6511,0.6519,0.6109,0.5817,0.5919,0.5746,0.6133,0.6030,0.6488,0.6338
-105k,0.6811,0.6772,0.6654,0.6646,,,0.6772,0.6519,,,0.6661,0.6472,0.6732,0.6369,0.6638,0.633,0.6740,0.6638,0.6140,0.5991,0.5912,0.5833,0.6046,0.5943,0.6496,0.618
-110k,0.7017,0.6867,0.6701,0.6654,,,0.6669,0.6480,0.6559,0.6456,0.6756,0.6551,0.6567,0.6401,0.6661,0.6456,0.6551,0.6535,0.6196,0.6006,,,0.6219,0.6069,0.6417,0.6338
-115k,0.6890,0.7040,0.6701,0.6654,,,0.6732,0.6511,0.6456,0.6227,0.6559,0.6456,0.6661,0.6488,0.6748,0.6527,0.6622,0.6448,0.6156,0.6014,,,0.6338,0.6069,0.6575,0.6196
-120k,0.6930,0.6953,0.6717,0.6701,,,0.6764,0.6464,0.6519,0.6275,0.6622,0.6480,0.6590,0.6322,0.6732,0.6377,0.6519,0.6622,0.5872,0.5612,,,0.6227,0.6077,0.6504,0.6275
-125k,0.6961,0.6977,0.6811,0.6819,,,0.6985,0.6433,0.6393,0.6417,0.6685,0.6433,0.6646,0.6338,0.6740,0.6559,0.6803,0.6693,0.6014,0.5888,,,0.6298,0.6243,0.6488,0.6117
-130k,0.6922,0.7056,0.6859,0.6717,,,0.6811,0.6330,0.6614,0.6393,0.6780,0.6322,0.6590,0.6361,0.6748,0.6456,0.6559,0.6472,0.6085,0.5880,,,,,0.6614,0.6235
-135k,0.6961,0.6953,0.6788,0.6756,,,0.6827,0.6614,,,0.6606,0.6575,0.6551,0.6464,0.6748,0.629,0.6677,0.6535,0.5991,0.5959,,,,,0.6433,0.6417
-140k,,,0.6819,0.6827,,,0.6867,0.6630,,,0.6598,0.6551,0.6567,0.6369,0.6709,0.6551,0.6638,0.6519,0.6038,0.5809,,,,,0.6472,0.6314
-145k,,,,,,,0.6819,0.6504,,,0.6717,0.6480,0.6669,0.6551,0.6661,0.6433,0.6725,0.6630,0.6180,0.5801,,,,,0.6606,0.644
-150k,,,,,,,0.6835,0.6646,,,0.6693,0.6654,,,0.6732,0.6148,0.6788,0.6409,0.6062,0.5991,,,,,0.6567,0.6361
-155k,,,,,,,0.6748,0.6590,,,0.6772,0.6677,0.6630,0.648,0.6851,0.6409,0.6922,0.6764,0.6204,0.6006,,,,,0.6677,0.6401
-160k,,,,,,,0.6875,0.6614,,,0.6709,0.6669,0.6748,0.648,0.6622,0.6638,0.6811,0.6803,0.6133,0.5864,,,,,0.6567,0.6322
-165k,,,,,,,0.6788,0.6661,,,0.6709,0.6717,0.6725,0.6433,,,,,0.6006,0.5856,,,,,0.6669,0.6472
-170k,,,,,,,0.6938,0.6709,,,0.6701,0.6598,0.6725,0.6354,,,0.6717,0.6867,0.6085,0.5833,,,,,0.6685,0.6575
-175k,,,,,,,0.6938,0.6693,,,0.6590,0.6622,0.6693,0.6614,,,0.6890,0.6867,0.6133,0.5754,,,,,0.6622,0.6598
-180k,,,,,,,0.6977,0.6646,,,0.6661,0.6646,0.6740,0.6661,,,0.6685,0.6504,0.6235,0.5967,,,,,0.6732,0.6535
-185k,,,,,,,0.6875,0.6519,,,0.6930,0.6535,0.6811,0.663,,,0.6851,0.6819,0.6156,0.5833,,,,,0.6661,0.648
-190k,,,,,,,0.6914,0.6859,,,0.6819,0.6606,,,,,0.6693,0.6638,0.6361,0.6006,,,,,0.6701,0.6488
-195k,,,,,,,0.6859,0.6614,,,0.6946,0.6732,,,,,0.6756,0.6638,0.6259,0.5841,,,,,0.6606,0.6543
-200k,,,,,,,0.6875,0.6669,,,0.6898,0.6780,,,,,0.7017,0.6701,0.6227,0.5872,,,,,0.6590,0.648
-205k,,,,,,,0.7072,0.6906,,,0.6969,0.6780,,,,,0.6827,0.6748,0.6306,0.5888,,,,,0.6725,0.659
-210k,,,,,,,0.6859,0.6661,,,0.6827,0.6748,,,,,0.6882,0.6717,0.6322,0.5919,,,,,0.6669,0.6488
-215k,,,,,,,0.7017,0.6780,,,0.6748,0.6772,,,,,0.6922,0.6709,0.6346,0.6006,,,,,0.6709,0.6661
-220k,,,,,,,0.7040,0.6788,,,0.6859,0.6732,,,,,0.6969,0.6638,0.6346,0.5983,,,,,,
-225k,,,,,,,0.7111,0.6717,,,0.6843,0.6685,,,,,0.6756,0.6606,0.6188,0.5935,,,,,,
-230k,,,,,,,0.7103,0.6811,,,,,,,,,0.7096,0.6701,0.6235,0.5935,,,,,,
-235k,,,,,,,0.7040,0.6772,,,,,,,,,0.7096,0.6764,0.6306,0.6062,,,,,,
-240k,,,,,,,0.7080,0.6851,,,,,,,,,,,0.6219,,,,,,,
-245k,,,,,,,0.6985,0.6938,,,,,,,,,,,0.6267,0.5888,,,,,,
-250k,,,,,,,0.7127,0.6938,,,,,,,,,,,0.6361,0.6006,,,,,,
-255k,,,,,,,0.7119,0.6827,,,,,,,,,,,0.6440,0.5998,,,,,,
-260k,,,,,,,0.7056,0.6867,,,,,,,,,,,0.6322,0.5975,,,,,,
-265k,,,,,,,0.7040,0.6756,,,,,,,,,,,0.6338,0.6069,,,,,,
-270k,,,,,,,0.7111,0.6819,,,,,,,,,,,0.6314,0.5991,,,,,,
-275k,,,,,,,0.7127,0.6811,,,,,,,,,,,0.6306,0.6148,,,,,,
-280k,,,,,,,0.7064,0.6914,,,,,,,,,,,0.6251,0.6054,,,,,,
-285k,,,,,,,0.7096,0.6977,,,,,,,,,,,0.6385,,,,,,,
-290k,,,,,,,,,,,,,,,,,,,0.6338,0.6077,,,,,,
-300k,,,,,,,,,,,,,,,,,,,0.6227,0.6093,,,,,,
-305k,,,,,,,,,,,,,,,,,,,0.6290,0.6069,,,,,,
-310k,,,,,,,,,,,,,,,,,,,0.6267,0.6156,,,,,,
-315k,,,,,,,,,,,,,,,,,,,0.6314,0.6101,,,,,,
-320k,,,,,,,,,,,,,,,,,,,0.6401,0.5991,,,,,,
-325k,,,,,,,,,,,,,,,,,,,,,,,,,,
-330k,,,,,,,,,,,,,,,,,,,,,,,,,,
-335k,,,,,,,,,,,,,,,,,,,,,,,,,,
+,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
+0-shot: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
+5k,0.5691,0.5351,0.5367,0.5383
+10k,0.5904,0.5604,0.5817,0.5667
+15k,0.5927,0.5919,0.588,0.5896
+20k,0.6448,0.6006,0.618,0.5935
+25k,0.6196,0.6125,0.6062,0.6101
+30k,0.6488,,0.614,0.6322
+35k,0.644,0.603,0.6259,0.6212
+40k,0.6496,0.6338,0.6267,0.6433
+45k,0.6456,0.6172,0.6393,0.6393
+50k,0.6464,0.6401,0.6164,0.6472
+55k,0.6567,0.6235,0.6314,0.6464
+60k,0.648,0.6251,0.6219,0.6369
+65k,0.6654,0.6283,0.6401,0.6504
+70k,0.6709,0.6322,0.6417,0.6559
+75k,0.6709,0.648,0.6527,0.6527
+80k,0.6843,0.6504,0.6369,0.6519
+85k,0.6875,0.6409,0.6575,0.6393
+90k,0.674,0.6369,0.6488,0.6527
+95k,0.6835,0.6369,0.6654,0.6409
+100k,0.6756,,0.659,0.6511
+105k,0.6772,,0.6732,0.674
+110k,0.6669,0.6559,0.6567,0.6551
+115k,0.6732,0.6456,0.6661,0.6622
+120k,0.6764,0.6519,0.659,0.6519
+125k,0.6985,0.6393,0.6646,0.6803
+130k,0.6811,0.6614,0.659,0.6559
+135k,0.6827,,0.6551,0.6677
+140k,0.6867,,0.6567,0.6638
+145k,0.6819,,0.6669,0.6725
+150k,0.6835,,,0.6788
+155k,0.6748,,0.663,0.6922
+160k,0.6875,,0.6748,0.6811
+165k,0.6788,,0.6725,
+170k,0.6938,,0.6725,0.6717
+175k,0.6938,,0.6693,0.689
+180k,0.6977,,0.674,0.6685
+185k,0.6875,,0.6811,0.6851
+190k,0.6914,,,0.6693
+195k,0.6859,,,0.6756
+200k,0.6875,,,0.7017
+205k,0.7072,,,0.6827
+210k,0.6859,,,0.6882
+215k,0.7017,,,0.6922
+220k,0.704,,,0.6969
+225k,0.7111,,,0.6756
+230k,0.7103,,,0.7096
+235k,0.704,,,0.7096
+240k,0.708,,,
+245k,0.6985,,,
+250k,0.7127,,,
+255k,0.7119,,,
+260k,0.7056,,,
+265k,0.704,,,
+270k,0.7111,,,
+275k,0.7127,,,
+280k,0.7064,,,
+285k,0.7096,,,
+290k,,,,
+300k,,,,
+305k,,,,
+310k,,,,
+315k,,,,
+320k,,,,
+325k,,,,
+330k,,,,
+335k,,,,

main.py

@@ -54,7 +54,7 @@ front_matter = {
             "author": "Nikhil Ranjan",
             "authorURL": "https://huggingface.co/nikhilranjan",
             "affiliation": "MBZUAI",
-            "affiliationURL": "",
+            "affiliationURL": "LLM360.ai",
         },
         {
             "author": "Omkar Pangarkar",
@@ -62,6 +62,12 @@ front_matter = {
             "affiliation": "Petuum, Inc.",
             "affiliationURL": "",
         },
+        {
+            "author": "Xuezhi Liang",
+            "authorURL": "",
+            "affiliation": "MBZUAI",
+            "affiliationURL": "",
+        },
         {
             "author": "Zhen Wang",
             "authorURL": "",
@@ -74,6 +80,12 @@ front_matter = {
             "affiliation": "UCSD",
             "affiliationURL": "",
         },
+        {
+            "author": "Bhaskar Rao",
+            "authorURL": "",
+            "affiliation": "MBZUAI",
+            "affiliationURL": "",
+        },
         {
             "author": "Zhoujun Cheng",
             "authorURL": "https://huggingface.co/zhoujun",

results.py

@@ -25,10 +25,10 @@ for fname in os.listdir("data/txt360_eval"):
         df = pd.read_csv(os.path.join("data/txt360_eval", fname))
 
         # slimpajama_res = df.iloc[2:, 2].astype(float).fillna(0.0) # slimpajama
-        fineweb_res = df.iloc[2:, 4].astype(float).fillna(method="bfill") # fineweb
-        txt360_base = df.iloc[2:, 5].astype(float).fillna(method="bfill") # txt360-dedup-only
-        txt360_web_up = df.iloc[2:, 7].astype(float).fillna(method="bfill") # txt360-web-only-upsampled
-        txt360_all_up_stack = df.iloc[2:, 9].astype(float).fillna(method="bfill") # txt360-all-upsampled + stackv2
+        fineweb_res = df.iloc[2:, 1].astype(float).fillna(method="bfill") # fineweb
+        txt360_base = df.iloc[2:, 2].astype(float).fillna(method="bfill") # txt360-dedup-only
+        txt360_web_up = df.iloc[2:, 3].astype(float).fillna(method="bfill") # txt360-web-only-upsampled
+        txt360_all_up_stack = df.iloc[2:, 4].astype(float).fillna(method="bfill") # txt360-all-upsampled + stackv2
 
         # each row is 20B tokens.
         # all_eval_results[metric_name]["slimpajama"] = slimpajama_res
@@ -66,10 +66,6 @@ for metric_name, res in all_eval_results.items():
         mode='lines', name='TxT360 - Full Upsampled + Stack V2'
     ))
 
-    print(all_eval_results[metric_name]["token"])
-    print(all_eval_results[metric_name]["fineweb"].tolist())
-    print(all_eval_results[metric_name]["txt360-web-only-upsampled"].tolist())
-
     # Update layout
     fig_res.update_layout(
         title=f"{metric_name} Performance",
@@ -825,7 +821,7 @@ table_div_1 = Div(NotStr(table_html),
 intro_div = Div(
     H2("TxT360 Studies"),
     H3("What This Section Contains"),
-    P("This section shows the learning curve when pre-training on TxT360, with a proper upsampling approach. We compare several simple strategies and demonstrate that one particular upsampling method, inspired by the natural data distribution, performs exceptionally well. In our preliminary experiments, the model learns significantly faster on TxT360 compared to a similarly scaled dataset, FineWeb. We believe that a more carefully designed upsampling strategy could further enhance the use of our data."),
+    P("This section shows the learning curve when pre-training on TxT360, with a proper upsampling approach. We compare several simple strategies and demonstrate that one particular upsampling method, inspired by the natural data distribution, performs exceptionally well. In our preliminary experiments, the model learns significantly faster on TxT360 compared to a similarly scaled dataset, FineWeb, on several important evaluation metrics. We believe that a more carefully designed upsampling strategy could further enhance the use of our data."),
     P("In addition to the training results, we also provide an analysis of the dataset, including perplexity trends over time across the CommonCrawl snapshots. This section is organized into the following topic areas:"),
     Ul(
         Li("The Learning Curve of TxT360 with an Upsampling Recipe", style = "margin-bottom: 5px"),
@@ -865,17 +861,18 @@ upsampling_exp = Div(
             "Evaluation results are the most direct indicator of model quality. We assess the intermediate results of the models across multiple metrics and plot the learning curves. Our findings indicate that the model learns significantly faster with TxT360. For a fair comparison, we evaluate TxT360 against FineWeb using only the CommonCrawl data sources, and we also show the curves after incorporating the 14 curated sources and coding data (Stack V2), demonstrating the full potential of the dataset. Due to computation resource constraints, we stop running experiments when we can observe clear trends."
         ),
         P(
-            "Based on the metrics, we find that TxT360’s CommonCrawl portion consistently outperforms FineWeb after upsampling, particularly on challenging tasks like MMLU and generation tasks such as NQ. Similar to the findings in DCLM, adding non-CommonCrawl data sources produces mixed results, especially when testing with that specific version of the data. We have since updated the non-CC data to further reduce noise."
+            "Based on the metrics, we find that TxT360’s CommonCrawl portion with the umsampling strategy outperforms FineWeb on key metrics at MMLU, NQ, falls slightly behind on HellaSwag. Furhter, we show that by combining TxT360 with coding data (Stack V2), the learning curve is significantly more stable and we observe improved results across most all of the metrics. Apparently the dataset preference here may depend on the set of metrics one would use."
         ),
         plotly2fasthtml(all_eval_res_figs["MMLU"]),
         plotly2fasthtml(all_eval_res_figs["NQ"]),
-        # plotly2fasthtml(all_eval_res_figs["GSM8K"]),
         plotly2fasthtml(all_eval_res_figs["HellaSwag"]),
+        P(
+            "Similar to the findings in DCLM, adding the curated non-CommonCrawl data sources produces mixed results (some preliminary figures are not shown here). Yet such data can help with domain specific tasks like MedQA."
+        ),
         plotly2fasthtml(all_eval_res_figs["MedQA"]),
         plotly2fasthtml(all_eval_res_figs["PIQA"]),
         plotly2fasthtml(all_eval_res_figs["TriviaQA"]),
         plotly2fasthtml(all_eval_res_figs["WinoGrande"]),
-
         H3("Comparing the Loss Curves"),
         P(
             "We also plot the training and validation loss curves for each dataset, showing that TxT360 achieves both lower training and validation losses compared to FineWeb. Although training loss may not correlate directly with final model performance, we observe that the loss curve for TxT360 exhibits fewer spikes compared to FineWeb, indicating more stable training dynamics."