Update curated.py

curated.py

@@ -9,6 +9,57 @@ from rich import print
 import uuid
 import plotly.express as px
 
+
+overview = Div(
+            H2("Curated Source Processing Overview"),
+            H3("What This Section Contains"),
+            P("This section provides a complete discussion on the filtering applied to the 14 curated sources that comprise the non-web data section of TxT360. The section is split into the following topic areas: "),
+            Ul(
+                Li("Curated Sources Data Processing Summary", style = "margin-bottom: 5px"),
+                Li("Individual Filtering Discussion for Each Source", style = "margin-bottom: 5px"),
+            ),
+        ),
+
+overview_text = P("Curated sources comprise high-quality datasets that contain domain-specificity. These sources, such as Arxiv, Wikipedia, and Stack Exchange, provide valuable data that is excluded from the web dataset mentioned above. Analyzing and processing non-web data can yield insights and opportunities for various applications. Details about each of the sources are provided below. ")
+copyright_disclaimer = P("We respect the copyright of the data sources and have not included the controversial data that was used in Pile like YouTube and Opensubtitles, Reddit threads, and books.")
+
+treemap_data = {
+  'Source': ['ArXiv', 'PubMed Central', 'PubMed Abstract', 'S2ORC Full Text', 'S2ORC Abstract', 'PhilPapers', 'Wikipedia', 'StackExchange', 'EuroParl', 'Ubuntu IRC', 'Freelaw', 'PG19', 'USPTO', 'HackerNews', 'DM Maths'],
+  'Category': ['Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Internet', 'Conversational', 'Legal/Formal', 'Conversational', 'Legal/Formal', 'Books', 'Legal/Formal', 'Conversational', 'Reasoning'],
+  'Count': [100, 200, 150, 120, 80, 90, 300, 250, 180, 150, 150, 250, 180, 120, 90],
+  'Details': [
+    'A repository of scientific papers in various disciplines, including computer science, physics, mathematics, and more.',
+    'A database of biomedical and life sciences research articles.',
+    'Abstracts of biomedical literature from various sources.',
+    'Full-text articles from the Semantic Scholar Open Research Corpus.',
+    'Abstracts of articles from the Semantic Scholar Open Research Corpus.',
+    'Papers from the PhilPapers database, a comprehensive index and bibliography of philosophy research.',
+    'A collaborative online encyclopedia that covers a wide range of topics.',
+    'A network of question-and-answer websites on various subjects, including programming, science, mathematics, and more.',
+    'A collection of multilingual parallel corpora of parliamentary debates from the European Parliament.',
+    'Chat logs from the Ubuntu Internet Relay Chat (IRC) channels.',
+    'Legal documents and court cases from various jurisdictions.',
+    'A collection of books from Project Gutenberg, a digital library of public domain works.',
+    'Patent documents from the United States Patent and Trademark Office.',
+    'User-generated news and discussion platform focused on technology and startups.',
+    'Deep Mind Maths dataset with generated questions.'
+  ]
+}
+# Calculate percentage for each data source
+total_count = sum(treemap_data['Count'])
+treemap_data['Percentage'] = [count / total_count * 100 for count in treemap_data['Count']]
+
+# Create treemap
+fig = px.treemap(treemap_data, path=['Category', 'Source'], values='Count', hover_data=['Details', 'Percentage'], hover_name='Source')
+
+# Set the size of the chart
+
+
+# Display treemap if you want to update the size.update_layout(width=800, height=600)
+treemap_chart = fig
+
+
+
 filtering_process = Div(
     Section(
           P("This section contains the specific steps taken to filter all 14 curated source datasets.")  
@@ -353,45 +404,11 @@ filtering_process = Div(
 
 
 
-overview_text = P("Curated sources comprise high-quality datasets that contain domain-specificity. These sources, such as Arxiv, Wikipedia, and Stack Exchange, provide valuable data that is excluded from the web dataset mentioned above. Analyzing and processing non-web data can yield insights and opportunities for various applications. Details about each of the sources are provided below. ")
-copyright_disclaimer = P("We respect the copyright of the data sources and have not included the controversial data that was used in Pile like YouTube and Opensubtitles, Reddit threads, and books.")
 
-local_dedup_text = P("Each curated data source has been prepared using its specific rules and has been locally deduped using min-hash near deduplication. Details about the dataset are shown below in the table:")
 
-treemap_data = {
-  'Source': ['ArXiv', 'PubMed Central', 'PubMed Abstract', 'S2ORC Full Text', 'S2ORC Abstract', 'PhilPapers', 'Wikipedia', 'StackExchange', 'EuroParl', 'Ubuntu IRC', 'Freelaw', 'PG19', 'USPTO', 'HackerNews', 'DM Maths'],
-  'Category': ['Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Papers', 'Internet', 'Conversational', 'Legal/Formal', 'Conversational', 'Legal/Formal', 'Books', 'Legal/Formal', 'Conversational', 'Reasoning'],
-  'Count': [100, 200, 150, 120, 80, 90, 300, 250, 180, 150, 150, 250, 180, 120, 90],
-  'Details': [
-    'A repository of scientific papers in various disciplines, including computer science, physics, mathematics, and more.',
-    'A database of biomedical and life sciences research articles.',
-    'Abstracts of biomedical literature from various sources.',
-    'Full-text articles from the Semantic Scholar Open Research Corpus.',
-    'Abstracts of articles from the Semantic Scholar Open Research Corpus.',
-    'Papers from the PhilPapers database, a comprehensive index and bibliography of philosophy research.',
-    'A collaborative online encyclopedia that covers a wide range of topics.',
-    'A network of question-and-answer websites on various subjects, including programming, science, mathematics, and more.',
-    'A collection of multilingual parallel corpora of parliamentary debates from the European Parliament.',
-    'Chat logs from the Ubuntu Internet Relay Chat (IRC) channels.',
-    'Legal documents and court cases from various jurisdictions.',
-    'A collection of books from Project Gutenberg, a digital library of public domain works.',
-    'Patent documents from the United States Patent and Trademark Office.',
-    'User-generated news and discussion platform focused on technology and startups.',
-    'Deep Mind Maths dataset with generated questions.'
-  ]
-}
-# Calculate percentage for each data source
-total_count = sum(treemap_data['Count'])
-treemap_data['Percentage'] = [count / total_count * 100 for count in treemap_data['Count']]
-
-# Create treemap
-fig = px.treemap(treemap_data, path=['Category', 'Source'], values='Count', hover_data=['Details', 'Percentage'], hover_name='Source')
-
-# Set the size of the chart
+local_dedup_text = P("Each curated data source has been prepared using its specific rules and has been locally deduped using min-hash near deduplication. Details about the dataset are shown below in the table:")
 
 
-# Display treemap if you want to update the size.update_layout(width=800, height=600)
-treemap_chart = fig
 
 
 
@@ -743,7 +760,7 @@ def curated(request):
     or modules dedicated to the dataset.""")
 
     data_preparation_div = Div(
-        H3("Data Preparation"),
+        H2("Data Preparation"),
         text,
         table_div,
         Div(
@@ -812,17 +829,18 @@ def curated(request):
     data_preprocessing_div = Div(H3("Data Preprocessing"), text, table_div)
     
     return Div(
+            overview    
             H2("Curated Sources: Overview"),
             overview_text,
             copyright_disclaimer,
             plotly2fasthtml(treemap_chart),
+            H2("Curated Sources Defined")
             table_desc,
+            data_preprocessing_div,
+            plotly2fasthtml(get_chart_28168342()),
             H2("Curated Sources Processing"),
             filtering_process,
             data_preparation_div,
-            H3("Data Filtering"),
-            data_preprocessing_div,
-            plotly2fasthtml(get_chart_28168342()),
             H2("Local Deduplication"),
             local_dedup_text,
             table_div_data_pipe,