Update curated.py

curated.py

@@ -445,14 +445,9 @@ filtering_process = Div(
     Section(
         H3("Wikipedia"),
         H4("Download and Extraction"),
-        Ol(
-            Li("The Wikimedia dataset was downloaded from the official snapshot on Huggingface", A("https://huggingface.co/datasets/wikimedia/wikipedia/tree/main", href="https://huggingface.co/datasets/wikimedia/wikipedia/tree/main")),
-            Li("Data is originally in parqet format so we used the", D_code("huggingface dataset.to_json"), " function to convert the data to the jsonl format"),
-        ),
+        P("The Wikimedia dataset was downloaded from the official snapshot on Huggingface: ", A("https://huggingface.co/datasets/wikimedia/wikipedia/tree/main", href="https://huggingface.co/datasets/wikimedia/wikipedia/tree/main"), ". The", D_code("huggingface dataset.to_json", language="python"), " function was used to convert the original parqet format to the jsonl format."),
         H4("Filtering"),
-        Ol(
-            Li("As we expect the dataset to be already of high quality so only one filter is applied which is to remove all documents (articles) with less than 10 words (not inclusive)"),
-        ),
+        P("Manual inspection of the dataset demostrated high quality content. Only one filter was used to remove articles with few words. Based normal sentence constructs, the article was kept if it contained 10 or more words. Any article with fewer than 10 words was removed."),
         H4("Local Deduplication Process"),
         Ol(
             Li("Whole wikipedia was deduped using minhash generation following Slim pajama code"),
@@ -463,12 +458,7 @@ filtering_process = Div(
     Section(
         H3("ArXiv"),
         H4("Download and Extraction"),
-        Ol(
-            Li("All the data was downloaded in original latex format from Arxiv official S3 dump s3://arxic/src"),
-            Li("We try to encode the downloaded data into utf-8 or guess encoding using chardet library"),
-            Li("After that pandoc was used to extract information from the latex files and saved as markdown format - code: pandoc -s {tex} -o out/{out_name}.md --wrap=none"),
-            Li("All markdowns were combined to create jsonl files"),
-        ),
+        P("All the data was downloaded in original latex format from Arxiv official S3 dump ", A("s3://arxic/src", href="s3://arxic/src"), "We try to encode the downloaded data into utf-8 or guess encoding using chardet library. After that pandoc was used to extract information from the latex files and saved as markdown format",  D_code("pandoc -s {tex} -o out/{out_name}.md --wrap=none", language="python"), ". All markdowns were combined to create jsonl files."),
         H4("Filtering"),
         P("Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset"),
         Ol(
@@ -1207,12 +1197,9 @@ def curated(request):
             H2("Curated Sources Defined"),
             table_desc,
             data_preprocessing_div,
-           # plotly2fasthtml(get_chart_28168342()),
-           # plotly2fasthtml(get_chart_new()),
-           # plotly2fasthtml(stacked_bar),
-           # plotly2fasthtml(diff_stacked_bar),
             plotly2fasthtml(diff2_stacked_bar),
             H2("Curated Sources Processing"),
+            H3("TALK ABOUT THE DIFFERENT FILTERS BEFORE HAND"),
             filtering_process,
             data_preparation_div,
             H2("Local Deduplication"),