| {"data":{"FineWeb (ours)":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.3308933284133672,0.3552836012095213,0.3781493119895458,0.3866849727928638,0.4050675220787525,0.4032807648181915,0.4174600429832935,0.4206059761345386,0.427497424185276,0.4316632784903049,0.4385909177362919,0.4334069043397903,0.4360812865197658,0.4404293224215507,0.4385774843394756,0.4407080821692943,0.4467254020273685,0.4470436163246631,0.4486658610403538,0.4459679573774338,0.4454015754163265,0.4515932314097881,0.4482216536998749,0.4484201297163963,0.455057855695486,0.4526158757507801,0.453176885843277,0.450159091502428,0.4516039006412029,0.4549933448433876,0.4555377587676048,0.4575010798871517,0.4577344059944153,0.4540543705224991,0.4537974074482918,0.4611785635352134,0.4586966186761856,0.4594406597316265,0.4598931074142456,0.457538403570652,0.4591932781040668,0.4636382386088371,0.4582749158143997,0.4625946804881096,0.4633439630270004,0.4666871763765812,0.4649887941777706,0.4671247974038124,0.4665776938199997,0.4672530107200145,0.4666078947484493,0.4666155055165291,0.4727727174758911,0.467480719089508,0.4681386984884739,0.4651658721268177,0.4668439887464046,0.4671731516718864,0.4719251021742821,0.4699816256761551,0.4723306186497211,0.4686817973852157,0.468911949545145,0.4714248068630695,0.4724191203713417,0.4700912088155746,0.4685601107776165,0.4716645181179046,0.4724556542932987,0.4670086726546287,0.4703365340828895,0.4698334187269211,0.471625205129385,0.4688323326408863,0.4735309742391109,0.4729253277182579,0.4747676998376846,0.4723741039633751,0.4764323942363262,0.4737579710781574,0.4758132360875606,0.4755662642419338,0.4730159305036068,0.4787128046154976,0.4740134924650192,0.4785312972962856,0.4783577285706997,0.4752367511391639,0.474204134196043,0.4737414345145225,0.4780189953744411,0.477523285895586,0.4751617163419723,0.4776186011731624,0.4769949465990066,0.4790891669690609,0.479917362332344,0.4771673306822777,0.4825278185307979,0.4811677671968937,0.4787211790680885,0.4817796200513839,0.4819813556969166,0.4802381917834282,0.4810985140502453,0.481117732822895,0.4791575670242309,0.4798801243305206,0.4829155020415783,0.4822122864425182,0.4827562272548675,0.4839778505265713,0.4820474348962307,0.4858015961945057,0.4826803356409073,0.4831027314066887,0.4827458150684833,0.4819435514509678,0.4836879819631576,0.4835174195468426,0.4855972006917,0.4871680215001106,0.4840429238975048,0.4827739149332046,0.4881435632705688,0.4871019721031189,0.486987367272377,0.4836358055472374,0.4867987409234047,0.4869474284350872,0.4886575266718864,0.4855775311589241,0.4863000251352787,0.4841057248413563,0.488163661211729,0.4904011823236942,0.4870587214827537,0.4884037151932716,0.4873756393790245,0.4925794936716556,0.4874482750892639,0.4898910224437713,0.4893574342131614,0.4888269044458866,0.4887814335525036,0.4876748844981193,0.4853886738419533,0.4878034777939319,0.4911742769181728,0.4905468784272671,0.4896938055753708,0.4875142201781273,0.4900367334485054,0.4900274313986301,0.4905461706221103,0.4891181476414203,0.4881824217736721,0.4902780950069427,0.4895042479038238,0.4890727028250694,0.4897591508924961,0.4879062548279762,0.4897833876311779,0.4902243539690971,0.4884885586798191,0.4880276583135128,0.4927133433520794,0.4899616949260235],"label":"FineWeb (ours)"},"C4":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.3308933284133672,0.3551123738288879,0.3784352876245975,0.3877953104674816,0.3993203267455101,0.4099095128476619,0.4136184640228748,0.414347343146801,0.4190568178892135,0.4218908287584781,0.4231295175850391,0.4287125319242477,0.4297344498336315,0.4332403875887394,0.4319765940308571,0.4359219521284103,0.4346421919763088,0.4424981512129307,0.4414303861558437,0.444732554256916,0.4384104907512665,0.444181352853775,0.4503846168518066,0.4436752013862133,0.4460380338132381,0.4498310275375843,0.4561186209321022,0.4558072946965694,0.4517050087451935,0.45280721783638,0.4511029534041881,0.4538496136665344,0.4561821073293686,0.4553975760936737,0.4567296579480171,0.4564522393047809,0.4547750055789947,0.4563902877271175,0.4566588178277015,0.4564423710107803,0.4597547873854637,0.4611989110708236,0.458673931658268,0.4615549780428409,0.4594683423638344,0.459637489169836,0.4626501463353634,0.4586587473750114,0.4627759084105491,0.4645364582538605,0.4662919379770756,0.4669810235500335,0.4629590921103954,0.4611541703343391,0.4635159634053707,0.4617790356278419,0.4662701040506363,0.4603991322219372,0.4632007218897342,0.4649621024727821,0.4670373052358627,0.4684801176190376,0.4635576270520687,0.4640897139906883,0.4648366048932075,0.467750646173954,0.4661705493927002,0.4673017747700214,0.4670937992632389,0.4679652377963066,0.4677082300186157,0.4677698649466038,0.4682595692574978,0.4695373848080635,0.466829814016819,0.4746575504541397,0.4714464470744133,0.4712707810103893,0.4668035432696342,0.4732451289892196,0.4715676791965961,0.4724206030368805,0.4714202284812927,0.4740159995853901,0.4751642681658268,0.474029790610075,0.47599982842803,0.4718085192143917,0.4766124375164509,0.4732356742024421,0.4746463894844055,0.473610121756792,0.4765255078673363,0.4751369915902614,0.477124772965908,0.4740747064352035,0.4751597791910171,0.4746479839086532,0.4750863499939441,0.474069282412529,0.4766325205564499,0.4751966968178749,0.480234831571579,0.4829998835921287,0.4779119342565536,0.4755960702896118,0.4754855707287788,0.4781345650553703,0.4780046604573726,0.4784741662442684,0.4773235991597175,0.4779680110514164,0.4786801375448704,0.4763363562524318,0.4761211909353733,0.4790357053279876,0.4782482571899891,0.4772635623812675,0.4800884462893009,0.4762507900595665,0.4809304289519787,0.4789686501026153,0.4788952358067035,0.4807513877749443,0.4819636456668377,0.4813393056392669,0.4799563512206077,0.4804293029010296,0.4832860343158245,0.4821714237332344,0.485592633485794,0.4815996848046779,0.4831583350896835,0.4846071004867553,0.4833417683839798,0.4807553999125957,0.4866329692304134,0.4829108603298664,0.4853781275451183,0.4814088977873325,0.4803747944533825,0.4820726066827774,0.4840803518891334,0.4835550002753734,0.4805133901536464,0.4861049503087997,0.4835067577660084,0.4856174662709236,0.4859548546373844,0.4868494793772697,0.4867915287613868,0.4881403036415577,0.4869705513119697,0.4862060099840164,0.4862068481743336,0.4862393103539943,0.4863550774753093,0.4858086891472339,0.4857287481427192,0.4880005866289139,0.4846996292471885,0.4863993115723133,0.4876768328249454,0.4873999394476414,0.4885325841605663,0.4876705072820186,0.485390305519104,0.4839635267853737],"label":"C4"},"Dolma":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.3307658787816763,0.3523229286074638,0.377311572432518,0.3858824856579304,0.3951764293015003,0.402766715735197,0.4049470797181129,0.4120629839599132,0.4156050495803356,0.4159501679241657,0.4160743616521358,0.4245675876736641,0.4277330711483955,0.4266031645238399,0.4327213019132614,0.4314779937267303,0.4336568377912044,0.4369685687124729,0.4366284385323524,0.4392882287502289,0.4405222535133362,0.435525432229042,0.4436039961874485,0.4407493844628334,0.441445555537939,0.4410557225346565,0.4371193572878837,0.4451764188706875,0.4450704641640186,0.4473482742905617,0.4460285790264606,0.4495329968631267,0.4506766386330127,0.4496022351086139,0.4495359361171722,0.4517026245594024,0.4506071843206882,0.4504962339997291,0.4534401223063469,0.4526695124804973,0.4503647126257419,0.4501906409859657,0.4478448294103145,0.4534101262688637,0.4531553275883198,0.4552497416734695,0.4568525813519954,0.4575119316577911,0.4590418413281441,0.4530587382614612,0.4584306105971336,0.4584567248821258,0.4616208709776401,0.4618218578398227,0.4582518599927425,0.4551334418356418,0.4586207643151283,0.4616809487342834,0.4611648134887218,0.4634306691586971,0.4613924361765384,0.4588174410164356,0.4602113999426365,0.4626984223723411,0.4556249380111694,0.4615240134298801,0.4615156538784504,0.4632683917880058,0.4620018191635608,0.4673102460801601,0.4642679207026958,0.4684422351419925,0.4641293548047542,0.4620067216455936,0.463960450142622,0.4660330079495907,0.4692153520882129,0.4669661112129688,0.466122068464756,0.4682283326983452,0.4685749150812626,0.4688928835093975,0.465722806751728,0.4659770168364048,0.4635110311210155,0.4644729532301426,0.4663772545754909,0.4708714000880718,0.4700784012675285,0.465810552239418,0.4693214185535908,0.4660605490207672,0.4695423729717731,0.4688450992107391,0.4721916541457176,0.4698034971952438,0.4665254801511764,0.4703435115516186,0.4714724421501159,0.4702155850827694,0.4694059453904629,0.471682820469141,0.4736889898777008,0.4703876934945583,0.4701756276190281,0.4680277854204178,0.4743972420692444,0.4708037711679935,0.4717469103634357,0.4740131162106991,0.4727639146149158,0.4723498672246933,0.4724784828722477,0.4736377336084842,0.4709004536271095,0.4707547724246979,0.469988003373146,0.4711141660809517,0.4780926555395126,0.4735368900001049,0.4735525399446487,0.472091656178236,0.4721037782728672,0.4745764546096325,0.4754389114677906,0.4705884419381618,0.4724388048052788,0.4759692996740341,0.4773353897035122,0.4758949503302574,0.4740513376891613,0.4749866165220737,0.4769214987754822,0.4776637814939022,0.4756806269288063,0.4760831333696842,0.4764537550508976,0.4760236926376819,0.4765783883631229,0.4751994907855987,0.4746053963899612,0.4780152216553688,0.4798212572932243,0.4787088483572006,0.4781070649623871,0.478924810886383,0.4792283922433853,0.4830445684492588,0.477554626762867,0.4838457219302654,0.4780268892645836,0.4787273816764354,0.4787089973688125,0.4800758995115757,0.4792372398078441,0.480698712170124,0.4782215058803558,0.4793966896831989,0.4793987721204757,0.4804111570119858,0.4809239841997623,0.480060052126646,0.4797034226357937,0.4830792918801307,0.4778680615127086,0.4814067967236042,0.4812476821243763,0.4816214181482792],"label":"Dolma"},"RefinedWeb":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.3308933284133672,0.3534814938902855,0.3764607086777687,0.38782499730587,0.3981050960719585,0.4028486795723438,0.4125883243978023,0.4117814563214779,0.414029736071825,0.4197172522544861,0.4211113378405571,0.4279881417751312,0.4280137903988361,0.4280424378812313,0.4326301179826259,0.4371833503246307,0.4346669465303421,0.4336562640964985,0.4432648755609989,0.4401291646063328,0.4394684173166752,0.4476612061262131,0.4465444348752498,0.4472153298556804,0.4433343075215816,0.4510187618434429,0.4459567815065384,0.4460812956094742,0.4498684890568256,0.4529943652451038,0.4528274349868297,0.4551213420927524,0.4549156539142132,0.4564928151667118,0.4576693661510944,0.4557182416319847,0.4536240361630916,0.457439012825489,0.4570476822555065,0.4589823484420776,0.462024375796318,0.4540738053619861,0.4550252184271812,0.4576593860983848,0.4573238864541054,0.4575810581445694,0.4622134491801262,0.4592566937208175,0.4614734016358852,0.4637473002076149,0.4625372551381588,0.4613912180066108,0.4597448222339153,0.4594792164862156,0.4662549719214439,0.4634026065468788,0.4633508697152138,0.4635734222829342,0.4628961533308029,0.4670135043561458,0.4639505892992019,0.4631133340299129,0.4665167145431041,0.4672448337078094,0.4693268723785877,0.4630668573081493,0.4676454700529575,0.4646359197795391,0.4621579721570015,0.4692446552217006,0.4704835228621959,0.4663223996758461,0.4680556617677212,0.466339822858572,0.4682099223136902,0.4711195565760135,0.4722655527293682,0.4727961830794811,0.4676857478916645,0.4719390422105789,0.4713102728128433,0.4712141714990139,0.4721613004803657,0.4713456854224205,0.4682970903813839,0.4679934531450271,0.4685162976384163,0.4679946713149547,0.4681242071092129,0.4702276065945625,0.472664151340723,0.4730790853500366,0.4731674715876579,0.4718914777040481,0.4719801284372806,0.4761029370129108,0.4735167175531387,0.4730370938777923,0.4730173237621784,0.4735377207398414,0.4777223989367485,0.4796326830983162,0.4734170883893966,0.4739485755562782,0.4748299159109592,0.4765299335122108,0.4745025858283043,0.4754423759877682,0.4784592799842357,0.4761341325938701,0.4760282784700393,0.4769757278263569,0.47154351323843,0.4786738082766533,0.4804279990494251,0.4777076803147793,0.4798569902777672,0.4759011939167976,0.4784621745347976,0.479673832654953,0.4780617095530033,0.48076206818223,0.47995800152421,0.4790860973298549,0.4817167408764362,0.4811586998403072,0.482547752559185,0.4816697351634502,0.4809327870607376,0.4816545359790325,0.4804601892828941,0.4776877984404564,0.4813711903989315,0.4844604581594467,0.4819537848234176,0.4820829331874847,0.4778126627206802,0.482935007661581,0.48230691999197,0.4826001971960068,0.4823969900608063,0.4811219945549965,0.4789146520197391,0.484035175293684,0.4848698377609253,0.4855728335678577,0.4825376532971859,0.485215101391077,0.4824351668357849,0.4835342466831207,0.4822137206792831,0.4838785007596016,0.4837255179882049,0.4853012599050998,0.4857851006090641,0.4863366298377514,0.4856646582484245,0.4842503517866134,0.4838776960968971,0.4846346862614155,0.4837041422724724,0.4813097268342972,0.4873070046305656,0.4841253720223903,0.4837464913725853,0.483069509267807,0.4851242564618587,0.4861010462045669],"label":"RefinedWeb"},"SlimPajama":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.3304025065153837,0.3485326766967773,0.366294227540493,0.3754063658416271,0.380060039460659,0.3842084035277366,0.3941901586949825,0.396852757781744,0.4074499122798443,0.405584454536438,0.4073434360325336,0.4124714508652687,0.4134335741400718,0.4161443412303924,0.4188667088747024,0.4202426932752132,0.4203570708632469,0.4227871932089329,0.4275586046278476,0.4272583276033401,0.4236343242228031,0.4221611768007278,0.4248477220535278,0.4282951094210148,0.4292030818760395,0.4363042339682579,0.4320153258740902,0.4338943809270859,0.4330463260412216,0.4384616464376449,0.4358800426125526,0.4390238709747791,0.4387876056134701,0.4379991367459297,0.4407520480453968,0.4365722797811031,0.4373247250914573,0.4419326409697532,0.4368139393627643,0.4383439570665359,0.4416647292673588,0.4406861551105976,0.4423066079616546,0.4422457814216614,0.4484159983694553,0.4441563449800014,0.4470763392746448,0.4423048347234726,0.446647435426712,0.4459831714630127,0.4476513750851154,0.4466814696788788,0.4435278102755546,0.4461003206670284,0.4472127594053745,0.4489398226141929,0.4500008933246135,0.4516164027154445,0.4502749890089035,0.4490774273872375,0.4485588259994983,0.4482260681688785,0.4513203538954258,0.4532071612775326,0.4522806704044342,0.4535516202449798,0.4556184969842434,0.4512116685509681,0.4526651911437511,0.4534229151904583,0.4510319754481315,0.4549613930284977,0.4524864666163921,0.452243909239769,0.4522925950586796,0.4506081640720367,0.4515948034822941,0.4542848505079746,0.4496320001780987,0.4584742784500122,0.4562755897641182,0.4584121964871883,0.4543888121843338,0.4571178145706653,0.4539541527628898,0.4583615288138389,0.4560770355165005,0.4565840028226375,0.456016231328249,0.4589645341038704,0.4549507163465023,0.4590726271271705,0.4600294604897499,0.4643459767103195,0.4621279649436474,0.4603357166051864,0.4565914608538151,0.4588591121137142,0.4605486840009689,0.4612153358757496,0.4666493646800518,0.4652941562235355,0.4635471254587173,0.4646625965833664,0.4616324640810489,0.4620275981724262,0.4599555470049381,0.4569032154977321,0.463778618723154,0.4630071707069874,0.4675904884934425,0.467460885643959,0.4701412692666054,0.463244054466486,0.4643578268587589,0.4655150510370731,0.4679873175919056,0.4627011120319366,0.4645654186606407,0.4667633399367332,0.4651090800762176,0.4674227461218834,0.463763378560543,0.4683923609554767,0.4667338877916336,0.4650763012468815,0.4685031399130821,0.4682512991130352,0.4704002998769283,0.4690291061997413,0.4723447039723396,0.4700597859919071,0.4730553664267063,0.4665385261178016,0.4713114872574806,0.4723366685211658,0.4753480590879917,0.4702173508703708,0.4711348637938499,0.4730475284159183,0.4687437377870083,0.472866803407669,0.4722364842891693,0.4734555780887604,0.4718337096273899,0.474613182246685,0.4756719246506691,0.4706077054142952,0.4729462638497352,0.4753362536430359,0.4749811328947544,0.4718534983694553,0.4764903634786606,0.4737220667302608,0.473462775349617,0.4746402017772198,0.4707343839108944,0.4728488773107528,0.4778959937393665,0.4751430302858352,0.4726544991135597,0.4728592745959759,0.4748839288949966,0.4711641781032085,0.4739051833748817,0.4720781221985817,0.4751792773604393,0.4769017845392227],"label":"SlimPajama"},"The Pile":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.3304216358810663,0.3466135319322347,0.3567665815353393,0.3609300442039966,0.3754195682704448,0.3760548382997513,0.3804825097322464,0.3870464153587818,0.3920954465866089,0.3908583559095859,0.3928747698664665,0.3923895694315433,0.3968513160943985,0.404996283352375,0.4021198935806751,0.4044422507286072,0.4055487178266048,0.4043267257511616,0.4051039405167103,0.4053148292005062,0.4118910208344459,0.4129008501768112,0.4153450205922127,0.4157540574669838,0.4165645688772201,0.4158085733652115,0.4154625944793224,0.4205422177910805,0.4168646782636642,0.4214760512113571,0.4222277626395225,0.4179082997143268,0.416801854968071,0.4236912615597248,0.4248885214328766,0.4242431484162807,0.4248657301068306,0.4258512482047081,0.4243130981922149,0.4244629479944706,0.430818609893322,0.4303862266242504,0.4287605956196785,0.432219460606575,0.4292010366916656,0.4285963587462902,0.4321340434253216,0.4298995658755302,0.4349256418645382,0.4324938207864761,0.4351008906960487,0.4330397509038448,0.4322203621268272,0.4351420737802982,0.4295673854649067,0.4331545792520046,0.4371312223374843,0.4368601404130459,0.4334152191877365,0.435929175466299,0.4348413497209549,0.4362935796380043,0.4395272135734558,0.4395910315215587,0.4397818148136139,0.4375653453171253,0.4350125305354595,0.4383638128638267,0.4414386712014675,0.4406127110123634,0.4395736493170261,0.4400510974228382,0.4349951185286045,0.4433234259486198,0.4408213645219803,0.4408195316791534,0.4454181902110576,0.4432171359658241,0.4386095143854618,0.4424192681908607,0.4406412802636623,0.4441004805266857,0.4434664808213711,0.4409965090453625,0.4398404918611049,0.4429648593068123,0.4477721899747848,0.4449514634907245,0.4450686313211918,0.4429425410926342,0.4468786306679249,0.4498275183141231,0.4463700018823147,0.4454979188740253,0.4485073313117027,0.4498474262654781,0.4488017149269581,0.4474775716662407,0.4488831833004951,0.4498938769102096,0.4441156759858131,0.4483954235911369,0.454746376723051,0.4491635039448738,0.4470739066600799,0.451224073767662,0.4524858966469764,0.4501192942261696,0.4518791697919369,0.4503650553524494,0.4545648172497749,0.4524221830070019,0.4521367736160755,0.4533489346504211,0.4547304809093475,0.4522802866995334,0.451341975480318,0.4508697539567947,0.4542875029146671,0.4543761536478996,0.456620555371046,0.4563915356993675,0.4555570594966411,0.4565647505223751,0.4527720846235752,0.4573541656136513,0.4586507454514503,0.4574597366154194,0.4572464860975742,0.4601885713636875,0.4597031660377979,0.4591619409620762,0.457712460309267,0.4584441147744655,0.4599815532565117,0.4610339701175689,0.4604088515043258,0.4583280719816685,0.4597837403416633,0.4560487046837806,0.4594802744686603,0.4635319598019123,0.4591141827404499,0.4597685001790523,0.4643094949424267,0.4597949869930744,0.4616970308125019,0.4608079977333545,0.461796186864376,0.4613191820681095,0.4638922624289989,0.4597249217331409,0.4629687368869781,0.4651569910347461,0.4653593450784683,0.4654904417693615,0.4689781554043293,0.4642409011721611,0.466390497982502,0.4667546525597572,0.4650019332766533,0.4624537453055382,0.4668834805488586,0.4637890830636024,0.4627578742802143,0.465025994926691,0.4623493291437626,0.4645387642085552],"label":"The Pile"},"RedPajama2":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.3302722573280334,0.3470507562160492,0.364705353975296,0.3764069005846977,0.3825778141617775,0.3896523900330066,0.3974571377038955,0.397537350654602,0.4001355320215225,0.4036932997405529,0.4104340709745884,0.4086671769618988,0.413622997701168,0.4122740626335144,0.4195037446916103,0.4190866313874721,0.4231311045587063,0.4235090501606464,0.4237663596868515,0.4246972687542438,0.4299561083316803,0.4258239455521106,0.4314707778394222,0.4274507761001587,0.4299160614609718,0.4354668110609054,0.4302525334060192,0.4337679930031299,0.4337638393044472,0.4404422976076603,0.4347392916679382,0.4343290999531746,0.4375873804092407,0.4372701570391655,0.4380103722214699,0.4379842169582844,0.4400746375322342,0.4394551366567611,0.4451456516981125,0.4420723840594291,0.4404123835265636,0.4446186311542988,0.4421855062246322,0.44248116761446,0.4423649460077286,0.4426446110010147,0.4418415017426014,0.4442390464246273,0.4463883489370346,0.4474505893886089,0.446966927498579,0.4456985853612423,0.4466467499732971,0.4440925717353821,0.4471017979085445,0.4474578313529491,0.4471569992601871,0.449719063937664,0.453827504068613,0.449543334543705,0.451267059892416,0.4523019045591354,0.4507706724107265,0.4516403637826442,0.4510826356709003,0.4538759067654609,0.453817319124937,0.4493415988981724,0.4524289667606354,0.4563530124723911,0.4525565430521965,0.4559375420212745,0.4579579904675483,0.4522393532097339,0.4529666379094124,0.4572585858404636,0.4526477642357349,0.4548618048429489,0.4535989835858345,0.4557950794696808,0.4576374888420105,0.4536071494221687,0.4567363113164902,0.4559534676373005,0.455357126891613,0.4581229574978351,0.4584124349057674,0.459429319947958,0.4609483703970909,0.4585786387324333,0.4584725610911846,0.460013099014759,0.4675985686480999,0.4608690924942493,0.4615997225046158,0.4654040858149528,0.4610586836934089,0.4633209407329559,0.4612604938447475,0.460259061306715,0.4624678529798984,0.4620500393211841,0.459990244358778,0.4619648195803165,0.4650140479207039,0.4641274213790893,0.4607139900326729,0.466593112796545,0.4665224589407444,0.4634186550974846,0.4623883478343487,0.4616753794252872,0.46418297290802,0.4637203030288219,0.4639860466122627,0.4644428603351116,0.4654988572001457,0.4639867171645164,0.4652636311948299,0.4673995152115822,0.4623220227658748,0.4664290770888328,0.4666106030344963,0.4653977937996387,0.464808851480484,0.468203954398632,0.4619068317115307,0.461650725454092,0.4648414589464664,0.465693786740303,0.4655340574681759,0.4663631655275821,0.4668397903442383,0.4658713564276695,0.4650161266326904,0.4697218723595142,0.4669433757662773,0.4679506830871105,0.4668639451265335,0.4648850038647651,0.4669442251324653,0.4653140194714069,0.4670411869883537,0.4648204818367958,0.4657435193657875,0.4680184945464134,0.4664600379765033,0.4672395847737789,0.4657266996800899,0.4675477854907512,0.4670163989067077,0.4690168797969818,0.4676221422851085,0.4677147716283798,0.4668192155659199,0.4684425666928291,0.467963095754385,0.4691894799470901,0.4651848673820495,0.4664440341293812,0.467386495321989,0.4652351178228855,0.4669347554445267,0.4688444659113884,0.467147346585989,0.4683484248816967,0.4699504524469375,0.4709850251674652],"label":"RedPajama2"}},"layout":{"xaxis":{"title":{"text":"Training tokens (billions)"}},"yaxis":{"range":[0.39,0.5]},"title":{"text":"Dataset Ablations"}}} |