,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
0-shot: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
5k,0.747,,0.7378,0.7318
10k,0.765,0.7573,0.7557,0.7612
15k,0.7775,0.7628,0.7769,0.7655
20k,0.7807,0.7671,0.7709,0.784
25k,0.7878,,0.7913,0.7791
30k,0.7862,0.778,0.7829,0.7889
35k,0.7933,0.7769,0.7824,0.7987
40k,0.7905,,0.7943,0.7878
45k,0.7982,0.7786,0.7829,0.7949
50k,0.7992,0.7775,0.7943,0.7933
55k,0.8079,0.7786,0.7884,0.7943
60k,0.7922,0.7818,0.7905,0.8003
65k,0.7976,0.79,0.7835,0.7943
70k,0.8052,0.7916,0.79,0.7976
75k,0.803,0.7878,0.8079,0.802
80k,0.7971,0.7829,0.7992,0.7933
85k,0.8003,0.8014,0.7949,0.7965
90k,0.7976,0.7873,0.7856,0.7998
95k,0.8041,0.7905,0.7954,0.8003
100k,0.8069,,0.7998,0.8009
105k,0.8074,,0.8063,0.796
110k,0.8085,0.7856,0.7938,0.7998
115k,0.8118,0.7911,0.8041,0.8052
120k,0.8074,0.7982,0.8025,0.7949
125k,0.8107,0.8009,0.8047,0.8003
130k,0.8079,0.7916,0.8014,0.7922
135k,0.8074,,0.8052,0.8014
140k,0.8123,,0.8063,0.7987
145k,0.8069,,0.8052,0.803
150k,0.8058,,,0.7987
155k,0.8096,,0.7954,0.8107
160k,0.8101,,0.802,0.8079
165k,0.8112,,0.8058,0.8101
170k,,,0.8041,0.8036
175k,0.8194,,0.7982,0.8118
180k,0.8118,,0.8025,0.8172
185k,0.8259,,0.8036,0.8096
190k,0.8139,,,0.8128
195k,0.8188,,,0.8161
200k,0.8112,,,0.8128
205k,0.8188,,,0.8177
210k,0.8188,,,0.8161
215k,0.8188,,,0.8085
220k,0.8199,,,0.8096
225k,0.8199,,,0.8134
230k,0.8172,,,0.8134
235k,0.8199,,,0.8205
240k,0.8166,,,
245k,0.8215,,,
250k,0.8172,,,
255k,0.8254,,,
260k,0.8215,,,
265k,0.821,,,
270k,0.8145,,,
275k,0.8161,,,
280k,0.8248,,,
285k,0.821,,,
290k,,,,
300k,,,,
305k,,,,
310k,,,,
315k,,,,
320k,,,,
325k,,,,
330k,,,,
335k,,,,