,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
0-shot: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
5k,0.5691,0.5351,0.5367,0.5383
10k,0.5904,0.5604,0.5817,0.5667
15k,0.5927,0.5919,0.588,0.5896
20k,0.6448,0.6006,0.618,0.5935
25k,0.6196,0.6125,0.6062,0.6101
30k,0.6488,,0.614,0.6322
35k,0.644,0.603,0.6259,0.6212
40k,0.6496,0.6338,0.6267,0.6433
45k,0.6456,0.6172,0.6393,0.6393
50k,0.6464,0.6401,0.6164,0.6472
55k,0.6567,0.6235,0.6314,0.6464
60k,0.648,0.6251,0.6219,0.6369
65k,0.6654,0.6283,0.6401,0.6504
70k,0.6709,0.6322,0.6417,0.6559
75k,0.6709,0.648,0.6527,0.6527
80k,0.6843,0.6504,0.6369,0.6519
85k,0.6875,0.6409,0.6575,0.6393
90k,0.674,0.6369,0.6488,0.6527
95k,0.6835,0.6369,0.6654,0.6409
100k,0.6756,,0.659,0.6511
105k,0.6772,,0.6732,0.674
110k,0.6669,0.6559,0.6567,0.6551
115k,0.6732,0.6456,0.6661,0.6622
120k,0.6764,0.6519,0.659,0.6519
125k,0.6985,0.6393,0.6646,0.6803
130k,0.6811,0.6614,0.659,0.6559
135k,0.6827,,0.6551,0.6677
140k,0.6867,,0.6567,0.6638
145k,0.6819,,0.6669,0.6725
150k,0.6835,,,0.6788
155k,0.6748,,0.663,0.6922
160k,0.6875,,0.6748,0.6811
165k,0.6788,,0.6725,
170k,0.6938,,0.6725,0.6717
175k,0.6938,,0.6693,0.689
180k,0.6977,,0.674,0.6685
185k,0.6875,,0.6811,0.6851
190k,0.6914,,,0.6693
195k,0.6859,,,0.6756
200k,0.6875,,,0.7017
205k,0.7072,,,0.6827
210k,0.6859,,,0.6882
215k,0.7017,,,0.6922
220k,0.704,,,0.6969
225k,0.7111,,,0.6756
230k,0.7103,,,0.7096
235k,0.704,,,0.7096
240k,0.708,,,
245k,0.6985,,,
250k,0.7127,,,
255k,0.7119,,,
260k,0.7056,,,
265k,0.704,,,
270k,0.7111,,,
275k,0.7127,,,
280k,0.7064,,,
285k,0.7096,,,
290k,,,,
300k,,,,
305k,,,,
310k,,,,
315k,,,,
320k,,,,
325k,,,,
330k,,,,
335k,,,,