0-shot,FineWeb-1.5T,Ours-Base,Ours-Upsampling2,All-Upsampling1
time: 3 min,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192,Llama-8x8B-seq8192
5k,0.2152,,0.2482,0.2687
10k,0.238,0.2372,0.2616,0.2718
15k,0.227,,0.2797,0.2639
20k,0.2419,,0.2317,0.2757
25k,0.2184,,0.2569,0.2474
30k,0.2679,0.2522,0.2097,0.2608
35k,0.2647,0.2655,0.2467,0.2694
40k,0.2671,0.2396,0.2569,0.2482
45k,0.2742,0.2734,0.2255,0.2333
50k,0.2749,0.2537,0.2372,0.2655
55k,0.2797,0.2561,0.2294,0.2537
60k,0.2294,0.2325,,0.2639
65k,0.2663,0.2757,0.2749,0.2726
70k,0.2592,0.2757,0.2632,0.2435
75k,0.249,0.2679,0.2616,0.2765
80k,0.2797,0.2419,0.2522,0.2789
85k,0.2655,0.2844,0.2687,0.2553
90k,0.231,0.2364,0.2624,0.2679
95k,0.2742,0.282,0.2647,0.2749
100k,0.2679,,0.2702,0.2663
105k,0.2655,,0.2632,0.2726
110k,0.2718,0.2474,0.2537,0.2537
115k,0.2655,0.2718,0.2247,0.2867
120k,0.293,0.2537,,0.2844
125k,0.2624,0.2364,0.2145,0.2883
130k,0.2828,0.2412,0.2891,0.2922
135k,,,0.2765,0.2702
140k,0.2529,,0.2545,0.293
145k,0.249,,0.2718,0.3024
150k,,,,0.3244
155k,0.2608,,0.2624,
160k,0.2529,,0.2726,0.2852
165k,0.2388,,0.2742,0.2561
170k,0.2435,,0.2506,0.3056
175k,0.2632,,0.2647,0.3126
180k,0.2608,,0.2899,0.3166
185k,0.271,,0.2561,0.3268
190k,0.2812,,,0.304
195k,0.2482,,,0.3472
200k,0.2639,,,0.3339
205k,0.2514,,,0.3409
210k,0.2742,,,0.3378
215k,0.2592,,,0.3362
220k,0.2262,,,0.3559
225k,0.249,,,0.3213
230k,0.2357,,,0.3472
235k,0.2514,,,0.3614
240k,0.2624,,,
245k,0.2482,,,
250k,0.2592,,,
255k,0.2537,,,
260k,0.2639,,,
265k,0.2844,,,
270k,0.2624,,,
275k,0.2757,,,
280k,0.2852,,,
285k,0.2726,,,
290k,,,,
300k,,,,
305k,,,,
310k,,,,
315k,,,,
320k,,,,
325k,,,,
330k,,,,
335k,,,,