Update curated.py

curated.py

@@ -551,6 +551,9 @@ filtering_process = Div(
         H3("ArXiv"),
         P("The ArXiv dataset is a vast collection of preprint research papers primarily in Mathematics, Computer Science, and Physics. Established in 1991, it offers high-quality text and mathematical knowledge, making it an invaluable resource for academic and scientific research. ArXiv papers are typically written in LaTeX, a popular typesetting system for these fields. We have extracted the information from latex and converted it into a text format."),
         P(B("Download and Extraction: "),"All the data was downloaded in original latex format from Arxiv official S3 dump ", A("s3://arxic/src", href="s3://arxic/src"), ". We try to encode the downloaded data into utf-8 or guess encoding using chardet library. After that pandoc was used to extract information from the latex files and saved as markdown format",  D_code("pandoc -s {tex} -o out/{out_name}.md --wrap=none", language="python"), ". All markdowns were combined to create jsonl files."),
+        Ul(
+            Li("Due to large amounts of meaningful data being contained in table formats, speical consideration was taken to extract the data and proper metadata.", style = "margin-bottom: -3px"),
+        ), 
         P(B(" Filters Applied: "), "multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset (citation needed)"),
         Ul(
             Li("Language Filter: any language other than English are discarded", style = "margin-bottom: -3px"),
@@ -639,6 +642,10 @@ filtering_process = Div(
         Div(
         H3("PubMed Central and PubMed Abstract"),
         P(B("Download and Extraction: "), "All files were downloaded from", A("ttps://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/",href="ttps://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/"),". PubMed Central (PMC) files are downloaded in an xml.tar format. The tar files are opened and converted to markdown format using pandoc", D_code("pandoc -f jats {nxml} -o {pmcid}.md", language="bash"),". The markdown files are combined to create jsonl files. PubMed Abstract (PMA) files  were downloaded in xml. The BeautifulSoup library was used to extract the abstract, title, and PMID. All files were stored in jsonl format."),
+        P(B("Unique Data Preperation Challenges: ")),
+        Ul(
+            Li("Due to large amounts of meaningful data being contained in table formats, speical consideration was taken to extract the data and proper metadata.", style = "margin-bottom: -3px"),
+        ), 
         P(B("Filters Applied: "), "Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset."),
         Ul(
             Li("Minimum Word Count Filter: PMC documents with less than 100 words (not inclusive) are discarded; PMA documents less than 20 words are discarded", style = "margin-bottom: -3px"),
@@ -699,7 +706,7 @@ filtering_process = Div(
     ),
     Section(
         Div(
-        H3("Europarl"),
+        H3("EuroParl"),
         P("A collection of multilingual parallel corpora of parliamentary debates from the European Parliament. This is a high-quality legacy dataset earlier used for translation tasks."),
         P(B("Download and Extraction: "), "Original dataset was downloaded from", A("http://www.statmt.org/europarl/v7/europarl.tgz", href="http://www.statmt.org/europarl/v7/europarl.tgz"),". The files were converted to jsonl lines for filtering."),
         P(B("Filters Applied: ") ,"EuroParl was initially filtered during the download process. Documents with fewer than 200 characters were removed. The documents also contained HTML tags which were removed."),
@@ -750,6 +757,11 @@ filtering_process = Div(
         P("High-quality dialog-based dataset where user comments on the links as the head post aggregated by Y Combinator."),
         P(B("Download and Extraction: "), "The dataset was downloaded from the HackerNews repo here:", A("https://hacker-news.firebaseio.com/v0/item/", href="https://hacker-news.firebaseio.com/v0/item/"), ". The dataset was parsed using the Story ID. In this dataset each post is a story, and each reply is considered subsequent story. Story IDs were considered between ID 1 to 37500000.  The URL for all Story IDs was pinged. If that ID returned an error, the ID was removed. Each request was given a 2 second wait to account for network time."),
         P("The HackerNews dataset contains a vast amount of stories and is known for lively discussions. Due to the number of replies a story may contain, only longest comment thread for each story was sampled past level 3. All stories included the title (1st level) and all direct replies (2nd level). Replies to the replies (3rd level) are only included for X STORIES."),
+        P(B("Unique Data Preperation Challenges: ")),
+        Ul(
+            Li("As discussed above, the comment heirarchies required a thoughful approach to extracting meaningful data. ", style = "margin-bottom: -3px"),
+            Li("In the comment thread heirarchy, relationships had to be assigned to between the comments, sub-comments, and original story ID. ", style = "margin-bottom: -3px"),
+        ),     
         P(B("Filters Applied: ")),
         Ul(
             Li("Language Filter: English", style = "margin-bottom: -3px"),
@@ -779,15 +791,20 @@ filtering_process = Div(
         P("Legal documents and court cases from various jurisdictions provided by US-registered non-profit firm Free Law Project. We have included data from CourtListener which included millions of legal opinions from federal and state courts."),
         P(B("Download and Extraction"), "The dataset was downloaded from:", A("https://storage.courtlistener.com/bulk-data/", href="https://storage.courtlistener.com/bulk-data/"), ". There are 19 CSV files which contain overlapping content. CSV files can contain content in multiple columns requiring a holistic extraction approach. Text was extracted from the following using html2text function. The block below shows how each text type was extracted."), 
         D_code("""
-        ("html", html2text),
-        ("html_lawbox", html2text),
-        ("html_columbia", html2text),
-        ("html_anon_2020", html2text),
-        ("html_with_citations", html2text),
-        ("xml_harvard", html2text),
+        ("html", html2text), ("html_lawbox", html2text), 
+        ("html_columbia", html2text), ("html_anon_2020", html2text), 
+        ("html_with_citations", html2text), ("xml_harvard", html2text), 
         plain_text
         """, language ="SQL"),
         P("All content was downloaded leading to high number of documents filtered during local deduplication. Following The Pile, priorty was given to plain_text first, followed by the columns in the table in reverse order."),
+        P(B("Unique Data Preperation Challenges: ")),
+            Ul(
+            Li("Consecutive whitespaces and tabs were found. Consecutive Whitespaces and tabes were reduce to one, single whitespace.", style = "margin-bottom: -3px"),
+            Li("Whitespaces were found between new lines with no addition text. These whitespaces were removed.", style = "margin-bottom: -3px"),
+            Li("Consecutive new lines were found in some documents without leading to a new paragraph. All consecutive newline to a single new line.", style = "margin-bottom: -3px"),
+            Li("Converted all single new lines to whitespace. If whitespace was found after a new line with no text, the whitespace was removed. All leading and trailing whitespace was removed.", style = "margin-bottom: -3px"),
+            Li("All \f characters were removed.", style = "margin-bottom: -3px"),
+        ), 
         P(B("Filters Applied: ")),
         Ul(
             Li("Language Filter: English", style = "margin-bottom: -3px"),
@@ -828,6 +845,12 @@ filtering_process = Div(
   8. Comment1:
   9. Comment2:
         """),
+        P(B("Unique Data Preperation Challenges: ")),
+        Ul(
+            Li("Handling code block was a required finding the specific blocks and exacting the details in one snippet.", style = "margin-bottom: -3px"),
+            Li("Question and Answer formatting had to be rewritten to match the question and the anwer.", style = "margin-bottom: -3px"),
+            Li("Occasionally a title was not included at the beginning of a question. For consistent formatting, a title was added.", style = "margin-bottom: -3px"),
+        ),  
         P(B("Filters Applied: ")),
         Ul(
             Li("Minimum Word Count Filter: 10", style = "margin-bottom: -3px"),
@@ -866,6 +889,10 @@ filtering_process = Div(
         def clean(x):
             return '\n'.join('* ' + line[4:] if line.startswith('===') else line[8:] for line in x.split('\n'))
         """, block="block", language="python" ),
+        P(B("Unique Data Preperation Challenges: ")),
+        Ul(
+            Li("Similar to the HackerNews challenges, we had to map comments and sub-comments to the original question.", style = "margin-bottom: -3px"),
+        ),  
         P(B("Filters Applied: ")),
         Ul(
             Li("Language Filter: English", style = "margin-bottom: -3px"),
@@ -883,6 +910,11 @@ filtering_process = Div(
         D_code(""" 
         Question: TEXT
         Answer: TEXT""", block="block", language="python"),
+        P(B("Unique Data Preperation Challenges: ")),
+        Ul(
+            Li("A byte string was included at the beginning of new lines", style = "margin-bottom: -3px"),
+            Li('No space before keyword "Answer:"', style = "margin-bottom: -3px"),
+        ),   
         P(B("Filters Applied: ")),
         Ul(
             Li("No filtering was applied to DM Math", style = "margin-bottom: -3px"),
@@ -908,6 +940,12 @@ filtering_process = Div(
         H3("PG-19"),
         P("A collection of books from Project Gutenberg, a digital library of public domain works. This contains all the books that were published before 1919."),
         P(B("Download and Extraction: "), "The dataset was downloaded directly from Huggingface: ", A("https://huggingface.co/datasets/deepmind/pg19", href="https://huggingface.co/datasets/deepmind/pg19"), "."),
+        P(B("Unique Data Preperation Challenges: ")),
+            Ul(
+            Li("Consecutive whitespaces were found spanning 10+ whitespace entries. These whitespaces were reduce to one, single whitespace.", style = "margin-bottom: -3px"),
+            Li("Consecutive new lines were found in some documents. All consecutive news over two were were reduce to two new lines.", style = "margin-bottom: -3px"),
+            Li("Delimiters such as * * * * * * * * ? were found. They were removed and replaced with whitespace.", style = "margin-bottom: -3px"),
+        ), 
         P(B("Filters Applied:")),
         Ul(
             Li("Language Filter: English", style = "margin-bottom: -3px"),