Update curated.py

curated.py

@@ -1041,10 +1041,7 @@ def curated(request):
         ),
     )
 
-    text = P("""Data preprocessing is a crucial step in the data science
-    pipeline. It involves cleaning and transforming raw data into a format that
-    is suitable for analysis. This process includes handling missing values,
-    normalizing data, encoding categorical variables, and more.""")
+
 
     preprocessing_steps = pd.DataFrame(
         {
@@ -1098,7 +1095,26 @@ def curated(request):
 
     table_html = preprocessing_steps.to_html(index=False, border=0)
     table_div = Div(NotStr(table_html), style="margin: 40px;")
-    data_preprocessing_div = Div(H3("Data Preprocessing"), text, table_div)
+    data_preprocessing_div = Div(
+        H2("Data Preprocessing"),
+        P("Data preprocessing is a crucial step in the data science pipeline. It involves cleaning and transforming raw data into a format that is suitable for analysis. This process includes handling missing values, normalizing data, encoding categorical variables, and more."),
+        H3("Language Filter"),
+        P("The Language Filter removes documents in unwanted languages. This step improves data quality by removing irrelevant documents."),
+        H3("Minimum Word Count Filter"),
+        P("The Minimum Word Count Filter sets a threshold for required words within a document. This step filters out low-quality or incomplete documents. However, this step may remove documents that contain valuable information so a proper analysis is important for each datasource."),
+        H3("Unigram Log Probability"),
+        P("The Unigram Log Probability Filter calculates the log probability of each unigram to measure the significance of individual words. This step quantifies the importance of individual words but maay not capture the semantic meaning of words."),
+        H2("Data Processing for S2ORC"),
+        P("The formating of the S2ORC dataset required special filters to be applied. These filters were not applied to the other data sources.")
+        H3("Title Abstract Filter"),
+        P("The Title Abstract Filter extracts information from the title and abstract. This step provides additional information for analysis but may introduce bias in the analysis."),
+        H3("Majority Language Filter"),
+        P("The Majority Language Filter identifies the majority language in the dataset. This step displays the distribution of languages in the dataset to enable language-specific analysis and insights."),
+        H3("Paragraph Count Filter"),
+        P("The Paragraph Count Filter counts the number of paragraphs in each document. This step helps to analyze the structure and length of documents which can be a useful hueristic for document complexity."),
+        H3("Frequency Filter"),
+        P("The Frequency Filter calculates the frequency of each word in the dataset. This step serves to identify important words and topics in the dataset but may be sensitive to noise and outliers."),
+        )
     
     return Div(
             overview,