|
,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
|
|
0-shot: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
|
|
5k,0.5691,0.5351,0.5367,0.5383
|
|
10k,0.5904,0.5604,0.5817,0.5667
|
|
15k,0.5927,0.5919,0.588,0.5896
|
|
20k,0.6448,0.6006,0.618,0.5935
|
|
25k,0.6196,0.6125,0.6062,0.6101
|
|
30k,0.6488,,0.614,0.6322
|
|
35k,0.644,0.603,0.6259,0.6212
|
|
40k,0.6496,0.6338,0.6267,0.6433
|
|
45k,0.6456,0.6172,0.6393,0.6393
|
|
50k,0.6464,0.6401,0.6164,0.6472
|
|
55k,0.6567,0.6235,0.6314,0.6464
|
|
60k,0.648,0.6251,0.6219,0.6369
|
|
65k,0.6654,0.6283,0.6401,0.6504
|
|
70k,0.6709,0.6322,0.6417,0.6559
|
|
75k,0.6709,0.648,0.6527,0.6527
|
|
80k,0.6843,0.6504,0.6369,0.6519
|
|
85k,0.6875,0.6409,0.6575,0.6393
|
|
90k,0.674,0.6369,0.6488,0.6527
|
|
95k,0.6835,0.6369,0.6654,0.6409
|
|
100k,0.6756,,0.659,0.6511
|
|
105k,0.6772,,0.6732,0.674
|
|
110k,0.6669,0.6559,0.6567,0.6551
|
|
115k,0.6732,0.6456,0.6661,0.6622
|
|
120k,0.6764,0.6519,0.659,0.6519
|
|
125k,0.6985,0.6393,0.6646,0.6803
|
|
130k,0.6811,0.6614,0.659,0.6559
|
|
135k,0.6827,,0.6551,0.6677
|
|
140k,0.6867,,0.6567,0.6638
|
|
145k,0.6819,,0.6669,0.6725
|
|
150k,0.6835,,,0.6788
|
|
155k,0.6748,,0.663,0.6922
|
|
160k,0.6875,,0.6748,0.6811
|
|
165k,0.6788,,0.6725,
|
|
170k,0.6938,,0.6725,0.6717
|
|
175k,0.6938,,0.6693,0.689
|
|
180k,0.6977,,0.674,0.6685
|
|
185k,0.6875,,0.6811,0.6851
|
|
190k,0.6914,,,0.6693
|
|
195k,0.6859,,,0.6756
|
|
200k,0.6875,,,0.7017
|
|
205k,0.7072,,,0.6827
|
|
210k,0.6859,,,0.6882
|
|
215k,0.7017,,,0.6922
|
|
220k,0.704,,,0.6969
|
|
225k,0.7111,,,0.6756
|
|
230k,0.7103,,,0.7096
|
|
235k,0.704,,,0.7096
|
|
240k,0.708,,,
|
|
245k,0.6985,,,
|
|
250k,0.7127,,,
|
|
255k,0.7119,,,
|
|
260k,0.7056,,,
|
|
265k,0.704,,,
|
|
270k,0.7111,,,
|
|
275k,0.7127,,,
|
|
280k,0.7064,,,
|
|
285k,0.7096,,,
|
|
290k,,,,
|
|
300k,,,,
|
|
305k,,,,
|
|
310k,,,,
|
|
315k,,,,
|
|
320k,,,,
|
|
325k,,,,
|
|
330k,,,,
|
|
335k,,,, |