Update curated.py

curated.py

@@ -595,15 +595,6 @@ filtering_process = Div(
             Li("paragraph_count: The paper must have at least 5 paragraphs after removing paragraphs with less than -20 average log world probability"),
             Li("frequency: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
         ),
-        H4("Filtering - S2ORC Abstract"),
-        P("1. Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset. The frequency filter was not used as suggested by peS2o because it was removing good samples as inspected manually"),
-        Ol(
-            Li("title_abstract: must have title and abstract"),
-            Li("language: abstract must be in English"),
-            Li("word_count: less than 20 (not inclusive) are discarded"),
-            Li("Unigram log probablity"),
-            Li("frequency: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
-        ),
         H4("Local Deduplication Process"),
         Ol(
             Li("Local dedup was done with all papers combined."),
@@ -616,7 +607,7 @@ filtering_process = Div(
         Details(
             Summary("FreeLaw Filtering Examples -- need to update"),
             Div(
-                freelaw_examples,
+                P("examples are missing"),
                 style="background-color: white; padding: 15px; margin-top: 10px; margin-bottom: 10px; border-radius: 8px; border: none; "  # Styling for the DV2 part
             ),
             style="""
@@ -628,6 +619,28 @@ filtering_process = Div(
         ),
         ),
     ),
+    Section(
+        Div(
+        H3("S2ORC ABSTRACT"),
+        P("The Semantic Scholar Open Research Corpus (S2ORC) is a comprehensive dataset designed for natural language processing (NLP) and text-mining research over scientific papers. It includes rich metadata, and abstract and full-text content for millions of academic papers across various disciplines. This dataset is further divided into two components, S2ORC abstract and S2ORC full text."),
+        H4("Download and Extraction"),
+        Ol(
+            Li("This was downloaded directly in zip format using S2ORC api key and normal get request. code: response = urllib.request.urlopen(url)"),
+            Li("There were two kind of datasets that was downloaded S2ORC and S2ORC abstract"),
+        ),    
+        H4("Filtering - S2ORC Abstract"),
+        P("1. Multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset. The frequency filter was not used as suggested by peS2o because it was removing good samples as inspected manually"),
+        Ol(
+            Li("title_abstract: must have title and abstract"),
+            Li("language: abstract must be in English"),
+            Li("word_count: less than 20 (not inclusive) are discarded"),
+            Li("Unigram log probablity"),
+            Li("frequency: The most frequent word in the paper consists of alpha characters only, and it appears in less than 7.5% of the document. Words are obtained by splitting the text on whitespace."),
+        ),
+        )
+    ),
+    
+    
     Section(
         Div(
         H3("PubMed - need to update with abstract vs central"),
@@ -797,7 +810,7 @@ filtering_process = Div(
         H3("FreeLaw"),
         P("Legal documents and court cases from various jurisdictions provided by US-registered non-profit firm Free Law Project. We have included data from CourtListener which included millions of legal opinions from federal and state courts."),
         H4("Download and Extraction"),
-        #P("The dataset was downloaded from:" A("https://storage.courtlistener.com/bulk-data/", href="https://storage.courtlistener.com/bulk-data/"), )#". There are 19 CSV files which contain overlapping content. CSV files can contain content in multiple columns requiring a holistic extraction approach. Text was extracted from the following using html2text function. The block below shows how each text type was extracted."), 
+        P("The dataset was downloaded from:", A("https://storage.courtlistener.com/bulk-data/", href="https://storage.courtlistener.com/bulk-data/"), ". There are 19 CSV files which contain overlapping content. CSV files can contain content in multiple columns requiring a holistic extraction approach. Text was extracted from the following using html2text function. The block below shows how each text type was extracted."), 
         D_code("""
         ("html", html2text),
         ("html_lawbox", html2text),
@@ -839,7 +852,7 @@ filtering_process = Div(
         Div(
         H3("StackExchange"),
         P("A network of question-and-answer websites on various subjects, including programming, science, mathematics, and more. This is one of the largest publicly available repositories for question-answer pairs. We have included comments also to include an overall discussion on each post."),
-        P(B("Download and Extraction: "), "The archive dataset was used to download all data from StackExchange and StackExchange's sub URLs including: ", A("math.stackexchange.com", href="math.stackexchange.com"),". Raw data was extracted an XML format and only two files Posts.xml and Comments.xml were considered. To match the StackExchange hierarchy, each file was parsed using post_id to connect questions to answers and then to comments."),
+        P(B("Download and Extraction: "), "The archive dataset was used to download all data from StackExchange and 364 StackExchange's sub URLs including: ", A("math.stackexchange.com", href="math.stackexchange.com"),". Raw data was extracted an XML format and only two files Posts.xml and Comments.xml were considered. To match the StackExchange hierarchy, each file was parsed using post_id to connect questions to answers and then to comments. We will include the full list of sub URLs in when the code is released."),
         P("""
         1. Questions:
   2. Comment1:
@@ -937,9 +950,9 @@ filtering_process = Div(
         P(B("Download and Extraction: "), "The dataset was downloaded directly from Huggingface:", A("https://huggingface.co/datasets/deepmind/pg19", href="https://huggingface.co/datasets/deepmind/pg19"), "."),
         H4("Filtering"),
         Ol(
-            Li("Language Filter: ???", style = "margin-bottom: 2px"),
+            Li("Language Filter: English", style = "margin-bottom: 2px"),
             Li("Minimum Word Count Filter: 20", style = "margin-bottom: 2px"),
-            Li("Unigram Log Probability", style = "margin-bottom: 2px"),
+            Li("Unigram Log Probability: ", "-20", style = "margin-bottom: 2px"),
         ),
         table_div_pg19,
         Details(