Merge branch 'main' of hf.co:spaces/HuggingFaceFW/blogpost

bibliography.bib

@@ -1,108 +1,129 @@
-@article{gregor2015draw,
-  title={DRAW: A recurrent neural network for image generation},
-  author={Gregor, Karol and Danihelka, Ivo and Graves, Alex and Rezende, Danilo Jimenez and Wierstra, Daan},
-  journal={arXiv preprint arXiv:1502.04623},
-  year={2015},
-  url ={https://arxiv.org/pdf/1502.04623.pdf}
-}
-@article{mercier2011humans,
-  title={Why do humans reason? Arguments for an argumentative theory},
-  author={Mercier, Hugo and Sperber, Dan},
-  journal={Behavioral and brain sciences},
-  volume={34},
-  number={02},
-  pages={57--74},
-  year={2011},
-  publisher={Cambridge Univ Press},
-  doi={10.1017/S0140525X10000968}
-}
-
-@article{dong2014image,
-  title={Image super-resolution using deep convolutional networks},
-  author={Dong, Chao and Loy, Chen Change and He, Kaiming and Tang, Xiaoou},
-  journal={arXiv preprint arXiv:1501.00092},
-  year={2014},
-  url={https://arxiv.org/pdf/1501.00092.pdf}
-}
-
-@article{dumoulin2016adversarially,
-  title={Adversarially Learned Inference},
-  author={Dumoulin, Vincent and Belghazi, Ishmael and Poole, Ben and Lamb, Alex and Arjovsky, Martin and Mastropietro, Olivier and Courville, Aaron},
-  journal={arXiv preprint arXiv:1606.00704},
-  year={2016},
-  url={https://arxiv.org/pdf/1606.00704.pdf}
-}
-
-@article{dumoulin2016guide,
-  title={A guide to convolution arithmetic for deep learning},
-  author={Dumoulin, Vincent and Visin, Francesco},
-  journal={arXiv preprint arXiv:1603.07285},
-  year={2016},
-  url={https://arxiv.org/pdf/1603.07285.pdf}
-}
-
-@article{gauthier2014conditional,
-  title={Conditional generative adversarial nets for convolutional face generation},
-  author={Gauthier, Jon},
-  journal={Class Project for Stanford CS231N: Convolutional Neural Networks for Visual Recognition, Winter semester},
-  volume={2014},
-  year={2014},
-  url={http://www.foldl.me/uploads/papers/tr-cgans.pdf}
-}
-
-@article{johnson2016perceptual,
-  title={Perceptual losses for real-time style transfer and super-resolution},
-  author={Johnson, Justin and Alahi, Alexandre and Fei-Fei, Li},
-  journal={arXiv preprint arXiv:1603.08155},
-  year={2016},
-  url={https://arxiv.org/pdf/1603.08155.pdf}
-}
-
-@article{mordvintsev2015inceptionism,
-  title={Inceptionism: Going deeper into neural networks},
-  author={Mordvintsev, Alexander and Olah, Christopher and Tyka, Mike},
-  journal={Google Research Blog},
-  year={2015},
-  url={https://research.googleblog.com/2015/06/inceptionism-going-deeper-into-neural.html}
-}
-
-@misc{mordvintsev2016deepdreaming,
-  title={DeepDreaming with TensorFlow},
-  author={Mordvintsev, Alexander},
-  year={2016},
-  url={https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/deepdream/deepdream.ipynb},
-}
-
-@article{radford2015unsupervised,
-  title={Unsupervised representation learning with deep convolutional generative adversarial networks},
-  author={Radford, Alec and Metz, Luke and Chintala, Soumith},
-  journal={arXiv preprint arXiv:1511.06434},
-  year={2015},
-  url={https://arxiv.org/pdf/1511.06434.pdf}
-}
-
-@inproceedings{salimans2016improved,
-  title={Improved techniques for training gans},
-  author={Salimans, Tim and Goodfellow, Ian and Zaremba, Wojciech and Cheung, Vicki and Radford, Alec and Chen, Xi},
-  booktitle={Advances in Neural Information Processing Systems},
-  pages={2226--2234},
-  year={2016},
-  url={https://arxiv.org/pdf/1606.03498.pdf}
-}
-
-@article{shi2016deconvolution,
-  title={Is the deconvolution layer the same as a convolutional layer?},
-  author={Shi, Wenzhe and Caballero, Jose and Theis, Lucas and Huszar, Ferenc and Aitken, Andrew and Ledig, Christian and Wang, Zehan},
-  journal={arXiv preprint arXiv:1609.07009},
-  year={2016},
-  url={https://arxiv.org/pdf/1609.07009.pdf}
-}
-
-@misc{openai2018charter,
-  author={OpenAI},
-  title={OpenAI Charter},
-  type={Blog},
-  number={April 9},
-  year={2018},
-  url={https://blog.openai.com/charter},
+@inproceedings{barbaresi-2021-trafilatura,
+  title = {Trafilatura: A Web Scraping Library and Command-Line Tool for Text Discovery and Extraction},
+  author = "Barbaresi, Adrien",
+  booktitle = "Proceedings of the Joint Conference of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing: System Demonstrations",
+  pages = "122--131",
+  publisher = "Association for Computational Linguistics",
+  url = "https://aclanthology.org/2021.acl-demo.15",
+  year = 2021,
+}
+@misc{penedo2023refinedweb,
+      title={The RefinedWeb Dataset for Falcon LLM: Outperforming Curated Corpora with Web Data, and Web Data Only},
+      author={Guilherme Penedo and Quentin Malartic and Daniel Hesslow and Ruxandra Cojocaru and Alessandro Cappelli and Hamza Alobeidli and Baptiste Pannier and Ebtesam Almazrouei and Julien Launay},
+      year={2023},
+      eprint={2306.01116},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@article{joulin2016fasttext,
+  title={FastText.zip: Compressing text classification models},
+  author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\'e}gou, H{\'e}rve and Mikolov, Tomas},
+  journal={arXiv preprint arXiv:1612.03651},
+  year={2016}
+}
+@article{joulin2016bag,
+  title={Bag of Tricks for Efficient Text Classification},
+  author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas},
+  journal={arXiv preprint arXiv:1607.01759},
+  year={2016}
+}
+@misc{penedo2024datatrove,
+  author = {Penedo, Guilherme and Kydlíček, Hynek and Cappelli, Alessandro and Wolf, Thomas and Sasko, Mario},
+  title = {DataTrove: large scale data processing},
+  year = {2024},
+  publisher = {GitHub},
+  journal = {GitHub repository},
+  url = {https://github.com/huggingface/datatrove}
+}
+@misc{chiang2024chatbot,
+      title={Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference},
+      author={Wei-Lin Chiang and Lianmin Zheng and Ying Sheng and Anastasios Nikolas Angelopoulos and Tianle Li and Dacheng Li and Hao Zhang and Banghua Zhu and Michael Jordan and Joseph E. Gonzalez and Ion Stoica},
+      year={2024},
+      eprint={2403.04132},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+@misc{rae2022scaling,
+      title={Scaling Language Models: Methods, Analysis & Insights from Training Gopher},
+      author={Jack W. Rae and Sebastian Borgeaud and Trevor Cai and Katie Millican and Jordan Hoffmann and Francis Song and John Aslanides and Sarah Henderson and Roman Ring and Susannah Young and Eliza Rutherford and Tom Hennigan and Jacob Menick and Albin Cassirer and Richard Powell and George van den Driessche and Lisa Anne Hendricks and Maribeth Rauh and Po-Sen Huang and Amelia Glaese and Johannes Welbl and Sumanth Dathathri and Saffron Huang and Jonathan Uesato and John Mellor and Irina Higgins and Antonia Creswell and Nat McAleese and Amy Wu and Erich Elsen and Siddhant Jayakumar and Elena Buchatskaya and David Budden and Esme Sutherland and Karen Simonyan and Michela Paganini and Laurent Sifre and Lena Martens and Xiang Lorraine Li and Adhiguna Kuncoro and Aida Nematzadeh and Elena Gribovskaya and Domenic Donato and Angeliki Lazaridou and Arthur Mensch and Jean-Baptiste Lespiau and Maria Tsimpoukelli and Nikolai Grigorev and Doug Fritz and Thibault Sottiaux and Mantas Pajarskas and Toby Pohlen and Zhitao Gong and Daniel Toyama and Cyprien de Masson d'Autume and Yujia Li and Tayfun Terzi and Vladimir Mikulik and Igor Babuschkin and Aidan Clark and Diego de Las Casas and Aurelia Guy and Chris Jones and James Bradbury and Matthew Johnson and Blake Hechtman and Laura Weidinger and Iason Gabriel and William Isaac and Ed Lockhart and Simon Osindero and Laura Rimell and Chris Dyer and Oriol Vinyals and Kareem Ayoub and Jeff Stanway and Lorrayne Bennett and Demis Hassabis and Koray Kavukcuoglu and Geoffrey Irving},
+      year={2022},
+      eprint={2112.11446},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{lee2022deduplicating,
+      title={Deduplicating Training Data Makes Language Models Better},
+      author={Katherine Lee and Daphne Ippolito and Andrew Nystrom and Chiyuan Zhang and Douglas Eck and Chris Callison-Burch and Nicholas Carlini},
+      year={2022},
+      eprint={2107.06499},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{carlini2023quantifying,
+      title={Quantifying Memorization Across Neural Language Models},
+      author={Nicholas Carlini and Daphne Ippolito and Matthew Jagielski and Katherine Lee and Florian Tramer and Chiyuan Zhang},
+      year={2023},
+      eprint={2202.07646},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@misc{raffel2023exploring,
+      title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
+      author={Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
+      year={2023},
+      eprint={1910.10683},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@misc{touvron2023llama,
+      title={LLaMA: Open and Efficient Foundation Language Models},
+      author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample},
+      year={2023},
+      eprint={2302.13971},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@article{dolma,
+  title = {Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research},
+  author={
+    Luca Soldaini and Rodney Kinney and Akshita Bhagia and Dustin Schwenk and David Atkinson and
+    Russell Authur and Ben Bogin and Khyathi Chandu and Jennifer Dumas and Yanai Elazar and
+    Valentin Hofmann and Ananya Harsh Jha and Sachin Kumar and Li Lucy and Xinxi Lyu and
+    Nathan Lambert and Ian Magnusson and Jacob Morrison and Niklas Muennighoff and Aakanksha Naik and
+    Crystal Nam and Matthew E. Peters and Abhilasha Ravichander and Kyle Richardson and Zejiang Shen and
+    Emma Strubell and Nishant Subramani and Oyvind Tafjord and Pete Walsh and Luke Zettlemoyer and
+    Noah A. Smith and Hannaneh Hajishirzi and Iz Beltagy and Dirk Groeneveld and Jesse Dodge and Kyle Lo
+  },
+  year = {2024},
+  journal={arXiv preprint},
+}
+@article{gao2020pile,
+  title={The {P}ile: An 800{GB} dataset of diverse text for language modeling},
+  author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and others},
+  journal={arXiv preprint arXiv:2101.00027},
+  year={2020}
+}
+@misc{cerebras2023slimpajama,
+  author = {Soboleva, Daria and Al-Khateeb, Faisal and Myers, Robert and Steeves, Jacob R and Hestness, Joel and Dey, Nolan},
+  title = {SlimPajama: A 627B token cleaned and deduplicated version of RedPajama},
+  month = {June},
+  year = 2023,
+  url = {https://huggingface.co/datasets/cerebras/SlimPajama-627B},
+}
+@software{together2023redpajama,
+  author = {Together Computer},
+  title = {RedPajama: an Open Dataset for Training Large Language Models},
+  month = {October},
+  year = 2023,
+  url = {https://github.com/togethercomputer/RedPajama-Data}
+}
+@article{jaccard1912distribution,
+  title={The distribution of the flora in the alpine zone. 1},
+  author={Jaccard, Paul},
+  journal={New phytologist},
+  volume={11},
+  number={2},
+  pages={37--50},
+  year={1912},
+  publisher={Wiley Online Library}
 }

index.html

@@ -55,7 +55,7 @@
                 border-right-color: rgba(0, 0, 0, 0.1);
                 position: -webkit-sticky; /* For Safari */
                 position: sticky;
-                top: 0; /* Adjust this value if needed */
+                top: 10px; /* Adjust this value if needed */
             }
         }
 
@@ -150,7 +150,7 @@
     </script>
 </d-front-matter>
 <d-title>
-    <figure style="grid-column: page; mix-blend-mode: multiply;">
+    <figure class="l-page">
         <img src="banner.png" alt="FineWeb">
     </figure>
 </d-title>
@@ -171,7 +171,7 @@
         recipe, why more deduplication is not always better and some interesting findings on the difference in
         quality of CommonCrawl dumps.</p>
 
-    <h2>Preamble</h2>
+    <h2>General considerations on web data</h2>
     <h3>Sourcing the data</h3>
     <p>A common question we see asked regarding web datasets used
         to train LLMs is “where do they even get all that data?” There are generally two options:</p>
@@ -191,14 +191,14 @@
         every 1 or 2 months, which can be freely downloaded. </p>
     <p>As an example, their latest crawl (2024-10) contains 3.16
         billion web pages, totaling 424.7 TiB of uncompressed content (the size changes from dump to dump). There
-        are 95 dumps since 2013 and 3 dumps from 2008 to 2012, which are in a different (older) format. </p>
+        are 95 dumps since 2013 and 3 dumps from 2008 to 2012, which are in a different (older) format.<d-footnote>We have not processed these 3 older dumps.</d-footnote> </p>
     <h3>Processing at scale</h3>
     <p>Given the sheer size of the data involved, one of the main
         challenges we had to overcome was having a modular, scalable codebase that would allow us to quickly iterate
         on our processing decisions and easily try out new ideas, while appropriately parallelizing our workloads
         and providing clear insights into the data. </p>
     <p>For this purpose, we developed <a
-            href="https://github.com/huggingface/datatrove"><code>datatrove</code></a>, an open-source data
+            href="https://github.com/huggingface/datatrove"><code>datatrove</code></a><d-cite bibtex-key="penedo2024datatrove"></d-cite>, an open-source data
         processing library that allowed us to seamlessly scale our filtering and deduplication setup to thousands of
         CPU cores. All the data processing steps involved in the creation of FineWeb used this <a
                 href="https://github.com/huggingface/datatrove">library</a>.</p>
@@ -215,7 +215,7 @@
         choose a diverse set of tasks and try not to overfit to any one individual benchmark.</p>
     <p>Another way to evaluate different datasets would be to
         train a model on each one and have humans rate and compare the outputs of each one (like on the <a
-                href="https://chat.lmsys.org/">LMSYS Chatbot Arena</a>). This would arguably provide the most
+                href="https://chat.lmsys.org/">LMSYS Chatbot Arena</a>)<d-cite bibtex-key="chiang2024chatbot"></d-cite>. This would arguably provide the most
         reliable results in terms of representing real model usage, but getting ablation results this way is too
         expensive and slow.</p>
     <p>The approach we ultimately went with was to train small
@@ -250,17 +250,19 @@
             (should not be too noisy)
         </li>
     </ul>
-    <p>You can find the full list of tasks and prompts we used <a
-            href="https://huggingface.co/datasets/HuggingFaceFW/fineweb/blob/main/lighteval_tasks.py">here</a>. To
+    <p>To
         have results quickly we capped longer benchmarks at 1000 samples (wall-clock evaluation taking less than 5
         min on a single node of 8 GPUs - done in parallel to the training).</p>
+    <aside>You can find the full list of tasks and prompts we used <a
+            href="https://huggingface.co/datasets/HuggingFaceFW/fineweb/blob/main/lighteval_tasks.py">here</a>.</aside>
     <h2>The FineWeb recipe</h2>
     <p>In the next subsections we will explain each of the steps
-        taken to produce the FineWeb dataset. You can find a full reproducible <code>datatrove</code> config <a
-                href="https://github.com/huggingface/datatrove/blob/main/examples/fineweb.py">here</a>.</p>
+        taken to produce the FineWeb dataset.</p>
     <figure class="l-body">
         <img src="plots/fineweb-recipe.png"/>
     </figure>
+    <aside>You can find a fully reproducible <code>datatrove</code> config <a
+                href="https://github.com/huggingface/datatrove/blob/main/examples/fineweb.py">here</a>.</aside>
     <h3>Starting point: text extraction</h3>
     <p>CommonCrawl data is available in two main formats: WARC
         and WET. <strong>WARC </strong>(Web ARChive format) files contain the raw data from the crawl, including the
@@ -270,8 +272,7 @@
         starting point. In our experience the default text extraction (extracting the main text of a webpage from
         its HTML) used to create these WET files is suboptimal and there are a variety of open-source libraries that
         provide better text extraction (by, namely, keeping less boilerplate content/navigation menus). We extracted
-        the text content from the WARC files using the <a href="https://trafilatura.readthedocs.io/en/latest/">trafilatura</a>
-        library. It is important to note, however, that text extraction is one of the most costly steps of our
+        the text content from the WARC files using the trafilatura library<d-cite bibtex-key="barbaresi-2021-trafilatura"></d-cite>. It is important to note, however, that text extraction is one of the most costly steps of our
         processing, so we believe that using the readily available WET data could be a reasonable trade-off for
         lower budget teams.</p>
     <p>To validate this decision, we processed the 2019-18 dump
@@ -287,7 +288,7 @@
         removes part of the data (be it words, lines, or full documents) that would harm performance and is thus
         deemed to be “lower quality”.</p>
     <p>As a basis for our filtering we used part of the setup
-        from <a href="https://arxiv.org/abs/2306.01116">RefinedWeb</a>. Namely, we:</p>
+        from RefinedWeb<d-cite bibtex-key="penedo2023refinedweb"></d-cite>. Namely, we:</p>
     <ul>
         <li>Applied URL filtering using a <a
                 href="https://dsi.ut-capitole.fr/blacklists/">blocklist</a> to remove adult content
@@ -295,13 +296,12 @@
     </ul>
     <ul>
         <li>Applied a <a
-                href="https://fasttext.cc/docs/en/language-identification.html">fastText language classifier</a> to
+                href="https://fasttext.cc/docs/en/language-identification.html">fastText language classifier</a><d-cite bibtex-key="joulin2016bag"></d-cite><d-cite bibtex-key="joulin2016fasttext"></d-cite> to
             keep only English text with a score ≥ 0.65
         </li>
     </ul>
     <ul>
-        <li>Applied quality and repetition filters from the <a
-                href="https://arxiv.org/abs/2112.11446">Gopher</a> paper (using the default thresholds)
+        <li>Applied quality and repetition filters from the Gopher<d-cite bibtex-key="rae2022scaling"></d-cite> paper (using the default thresholds)
         </li>
     </ul>
     <p>After applying this filtering to each of the text
@@ -315,9 +315,7 @@
     <p>The web has many aggregators, mirrors, templated pages or
         just otherwise repeated content spread over different domains and webpages. Often, these duplicated pages
         can be introduced by the crawler itself, when different links point to the same page. </p>
-    <p>Removing these duplicates (deduplicating) has been <a
-            href="https://arxiv.org/abs/2107.06499">linked to an improvement in model performance</a> and a <a
-            href="https://arxiv.org/abs/2202.07646">reduction in memorization of pretraining data</a>, which might
+    <p>Removing these duplicates (deduplicating) has been linked to an improvement in model performance<d-cite bibtex-key="lee2022deduplicating"></d-cite> and a reduction in memorization of pretraining data<d-cite bibtex-key="carlini2023quantifying"></d-cite>, which might
         allow for better generalization. Additionally, the performance uplift can also be tied to increased training
         efficiency: by removing duplicated content, for the same number of training tokens, a model will have seen
         more diverse data.</p>
@@ -326,14 +324,14 @@
         efficient data structures to index the data (like suffix arrays). Methods can also be “fuzzy”, by using some
         similarity metric to mark documents as duplicates, or “exact” by checking for exact matches between two
         documents (or lines, paragraphs, or whatever other granularity level being used).</p>
-    <h3>Our deduplication parameters</h3>
+    <h4>Our deduplication parameters</h4>
     <p>Similarly to RefinedWeb, we decided to apply MinHash, a
         fuzzy hash based deduplication technique. We chose to compute minhashes on each document’s 5-grams, using
         112 hash functions in total, split into 14 buckets of 8 hashes each — targeting documents that are at least
         75% similar. Documents with the same 8 minhashes in any bucket are considered a duplicate of each other.</p>
     <p>This would mean that for two documents with a similarity (<code>s</code>)
         of 0.7, 0.75, 0.8 and 0.85, the probability that they would be identified as duplicates would be 56%, 77%,
-        92% and 98.8% respectively (<code>1-(1-s^8)^14</code>). See the plot below for a match probability
+        92% and 98.8% respectively ($$1-(1-s^8)^{14}$$). See the plot below for a match probability
         comparison between our setup with 112 hashes and the one from RefinedWeb, with 9000 hashes, divided into 450
         buckets of 20 hashes (that requires a substantially larger amount of compute resources):</p>
     <figure><img src="plots/minhash_parameters_comparison.png"/>
@@ -341,7 +339,7 @@
     <p>While the high number of hash functions in RefinedWeb
         allows for a steeper, more well defined cut off, we believe the compute and storage savings are a reasonable
         trade off.</p>
-    <h3>More deduplication is always better, right?</h3>
+    <h4>More deduplication is always better, right?</h4>
     <p>Our initial approach was to take the entire dataset (all
         95 dumps) and deduplicate them as one big dataset using MinHash.</p>
     <p>We did this in an iterative manner: starting with the most
@@ -385,7 +383,7 @@
     <p>These results show that, for this older dump where we were
         removing over 90% of the original data, the data that was kept was actually <em>worse</em> than the data
         removed (considered independently of all the other dumps).</p>
-    <h3>Taking a step back: individual dump dedup</h3>
+    <h4>Taking a step back: individual dump dedup</h4>
     <p>We then tried an alternative approach: we deduplicated
         each dump with MinHash individually (without considering the other dumps). This resulted in 20 trillion
         tokens of data.</p>
@@ -482,8 +480,7 @@
     <figure><img src="plots/Untitled.png"/></figure>
     <h3>Additional filtering</h3>
     <p>By this point we had reached the same performance as
-        RefinedWeb, but on our aggregate of tasks, another heavily filtered dataset, <a
-                href="https://arxiv.org/abs/1910.10683">the C4 dataset</a>, still showed stronger performance (with
+        RefinedWeb, but on our aggregate of tasks, another heavily filtered dataset, the C4 dataset<d-cite bibtex-key="raffel2023exploring"></d-cite>, still showed stronger performance (with
         the caveat that it is a relatively small dataset for current web-scale standards).</p>
     <p>We therefore set out to find new filtering steps that
         would, at first, allow us to match the performance of C4 and eventually surpass it. A natural starting point
@@ -496,8 +493,8 @@
     <p>Despite its age and limited size (around 175B gpt2
         tokens), models trained on this dataset have strong performance, excelling in particular on the Hellaswag
         benchmark, one of the benchmarks in our “early signal” group with the stronger signal and highest
-        signal-over-noise ratio. As such, it has stayed a common sub-set of typical LLM training, for instance in in
-        <a href="https://arxiv.org/abs/2302.13971">the relatively recent Llama1 model</a>. We experimented applying
+        signal-over-noise ratio. As such, it has stayed a common sub-set of typical LLM training, for instance in
+        the relatively recent Llama1 model<d-cite bibtex-key="touvron2023llama"></d-cite>. We experimented applying
         each of the different filters used in C4 to a baseline of the independently deduped FineWeb 2019-18 dump
         (plot smoothed with a 3 checkpoints sliding window):</p>
     <figure><img src="plots/c4_filters.png"/></figure>
@@ -596,28 +593,28 @@
     <p>We compared 🍷 FineWeb with the following datasets:</p>
     <ul>
         <li><a
-                href="https://huggingface.co/datasets/tiiuae/falcon-refinedweb">RefinedWeb</a>
+                href="https://huggingface.co/datasets/tiiuae/falcon-refinedweb">RefinedWeb</a><d-cite bibtex-key="penedo2023refinedweb"></d-cite>
         </li>
     </ul>
     <ul>
-        <li><a href="https://huggingface.co/datasets/allenai/c4">C4</a></li>
+        <li><a href="https://huggingface.co/datasets/allenai/c4">C4</a><d-cite bibtex-key="raffel2023exploring"></d-cite></li>
     </ul>
     <ul>
         <li><a href="https://huggingface.co/datasets/allenai/dolma">Dolma v1.6</a> (the
-            CommonCrawl part)
+            CommonCrawl part) <d-cite bibtex-key="dolma"></d-cite>
         </li>
     </ul>
     <ul>
-        <li><a href="https://huggingface.co/datasets/EleutherAI/pile">The Pile</a></li>
+        <li><a href="https://huggingface.co/datasets/EleutherAI/pile">The Pile</a> <d-cite bibtex-key="gao2020pile"></d-cite></li>
     </ul>
     <ul>
         <li><a
-                href="https://huggingface.co/datasets/cerebras/SlimPajama-627B">SlimPajama</a>
+                href="https://huggingface.co/datasets/cerebras/SlimPajama-627B">SlimPajama</a> <d-cite bibtex-key="cerebras2023slimpajama"></d-cite>
         </li>
     </ul>
     <ul>
         <li><a
-                href="https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2">RedPajama2</a>
+                href="https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2">RedPajama2</a> <d-cite bibtex-key="together2023redpajama"></d-cite>
             (deduplicated)
         </li>
     </ul>
@@ -677,7 +674,7 @@
     <h3>Changes in the most frequent URLs [HAVE TO RECHECK]</h3>
     <p>For each crawl from 2021-10 onwards, we gathered a list of
         the 60k most frequent <strong>FQDNs</strong> (fully qualified domain name). We then calculated the <a
-                href="https://en.wikipedia.org/wiki/Jaccard_index">Jaccard similarity</a> between consecutive
+                href="https://en.wikipedia.org/wiki/Jaccard_index">Jaccard similarity</a><d-cite bibtex-key="jaccard1912distribution"></d-cite> between consecutive
         crawls. A high value means that a crawl/dump has many of the same FQDNs as the dump immediately preceding
         it, while a small value means that a considerable number of top 60k FQDNs were downsampled or removed, or
         that alternatively new FQDNs were added to the top 60k.</p>
@@ -736,12 +733,6 @@
 </d-article>
 
 <d-appendix>
-
-    <h3>Contributions</h3>
-    <p>Some text describing who did what.</p>
-    <h3>Reviewers</h3>
-    <p>Some text with links describing who reviewed the article.</p>
-
     <d-bibliography src="bibliography.bib"></d-bibliography>
 </d-appendix>
 
@@ -749,10 +740,10 @@
     const article = document.querySelector('d-article');
     const toc = document.querySelector('d-contents');
     if (toc) {
-        const headings = article.querySelectorAll('h2, h3');
+        const headings = article.querySelectorAll('h2, h3, h4');
         let ToC = `<nav role="navigation" class="l-text figcaption"><h3>Table of contents</h3>`;
+        let prevLevel = 0;
 
-        let elements = [];
         for (const el of headings) {
             // should element be included in TOC?
             const isInTitle = el.parentElement.tagName == 'D-TITLE';
@@ -760,20 +751,25 @@
             if (isInTitle || isException) continue;
             el.setAttribute('id', el.textContent.toLowerCase().replaceAll(" ", "_"))
             const link = '<a href="' + '#' + el.getAttribute('id') + '">' + el.textContent + '</a>';
-            if (el.tagName === 'H2')
-                elements.push([link, []]);
-            else {
-                if (elements.length === 0)
-                    elements.push([null, []])
-                elements[elements.length - 1][1].push(link)
+
+            const level = el.tagName === 'H2' ? 0 : (el.tagName === 'H3' ? 1 : 2);
+            while (prevLevel < level) {
+                ToC += '<ul>'
+                prevLevel++;
+            }
+            while (prevLevel > level) {
+                ToC += '</ul>'
+                prevLevel--;
             }
+            if (level === 0)
+                ToC += '<div>' + link + '</div>';
+            else
+                ToC += '<li>' + link + '</li>';
         }
 
-        for (const topLevel of elements) {
-            ToC += '<div>' + topLevel[0] + '</div><ul>';
-            for (const subLevel of topLevel[1])
-                ToC += '<li>' + subLevel + '</li>';
-            ToC += '</ul>';
+        while (prevLevel > 0) {
+            ToC += '</ul>'
+            prevLevel--;
         }
         ToC += '</nav>';
         toc.innerHTML = ToC;