Update web.py

web.py

@@ -363,12 +363,41 @@ def web_data():
             ),
             style="margin-top: 20px;",
         ),
-        H2("Web Data Processing Overview"),
+        H2("Web Data Processing Summary"),
         P("The following section provides explicit details covering the reasoning and decisions behind each of the filters we applied. The table below provides a high-level comparison of TxT360's filtering compared to other commonly used pretraining datasets."),
         table_div_filter_data,
         P("Our filtering rate is illustrated below. Before deduplication, our filtering rate is comparable to RefinedWeb. During global deduplication, we removed approximately 85.89% of the data, significantly higher than previous works, indicating a large number of duplicates across dumps. "),
         Img(src="images/filter_rate.jpg", height = "300", width = "600" ),
         P("Note: All percentages are based on the number of documents. The gray bars represent the relative percentages of removed documents at each step, while the colorful bars represent the percentages of retained documents relative to the total number of documents in the raw Common Crawl."),
+        P("We also adopt rules from RefinedWeb [1] to remove lines if they satisfy any of the following criteria:"),
+        Ul(
+            Li("the line is only composed of uppercase characters"),
+            Li("the line is only composed of numerical characters"),
+            Li("the line matches the pattern “r'^\d+\s+likes$"),
+            Li("the line only contains one word."),
+        ),
+        P("We summarize other statistics-based rules originated from Gopher [7] in this section. The statistics can be used include:"),
+        Ul(
+            Li("the word count in the document"),
+            Li("the mean word length"),
+            Li("the number of sentences"),
+            Li("the symbol-to-word ratio"),
+            Li("the fraction of alphabetic words"),
+            Li("and the number of stop words"),
+        ),
+        P("Specifically, we remove any document which satisfies any of the following criteria:"),
+        Ul(
+            Li("it contains less than 50 words or more than 100,000 words"),
+            Li("its mean word length is outside the range of 3 to 10"),
+            Li("it contains less than 3 sentences"),
+            Li("its symbol-to-word ratio is greater than 0.1"),
+            Li("the words that contain at least one alphabetic character are less than 80% of the whole words"),
+            Li("it contains less than two of the stop words (the, be, to, of, and, that, have, with"),
+        ),
+
+        P("Following C4, we remove any page where the phrase “lorem ipsum” appears since some pages have placeholder “lorem ipsum” text."),
+        
+        
         H3("1. Document Preparation"),
         
         H4("1.1 Text Extraction"),