Update web.py

web.py

@@ -750,16 +750,29 @@ def web_data():
             Li("it contains less than two of the stop words (the, be, to, of, and, that, have, with", style = "margin-bottom: 5px"),
         ),
         H5("Word Count"),
-        P("""
-        Implementations from Dolma
-        Implementations from RedPajama-V2
-        Implementations from DataTrove
+        Details(
+            Summary("Implementations from Dolma"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
+        Details(
+            Summary("Implementations from RedPajama-V2"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
         
+        Details(
+            Summary("Implementations from DataTrove"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
+        P("""
         Both Dolma and RedPajama V2 split texts into words using white spaces and newline symbols. However, 
         DataTrove employs a tokenizer to split texts into words and ignore punctuations, resulting in a higher 
         word count compared to simple `text.split()`.
         We decided to use simple `len(text.split())` to compute the word count.
         """),
+        
         H5("Mean Word Length"),
         P("""
         There is minimal variation among existing pipeline implementations. We simply compute the mean word length as follows:
@@ -782,30 +795,97 @@ def web_data():
         The only publicly available implementation of this quality signal is from RedPajama V2, which uses regular expressions 
         to split text into sentences.
         """),
+        Details(
+            Summary("Implementations from RedPajama-V2"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
         P("""
         However, we found that this approach can mistakenly interpret periods in URLs as sentence endings. To address this, 
         we opted to use `nltk.tokenize.sent_tokenize` for more accurate sentence splitting.
         """),
+        Details(
+            Summary("TxT360 Implementation"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
+        
         H5("Symbol to Word Ratio"),
         P("""
-        Implementations from Dolma
-        Implementations from RedPajama-V2
-        Implementations from DataTrove
-       
         Following RedPajama-V2 and DataTrove, we use the symbols of ("#", "...", "…").
         We calculate the ratio as the number of symbols divided by the total number of words.
         """),
+        Details(
+            Summary("Implementations from Dolma"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
+        Details(
+            Summary("Implementations from RedPajama-V2"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
+        
+        Details(
+            Summary("Implementations from DataTrove"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
+        Details(
+            Summary("TxT360 Implementation"),
+            D_code("""
+            """, block="block", language="python"),
+        ),
+        
         H5("Fraction of Alphabetic Words"),
+        Details(
+            Summary("Implementations from Dolma"),
+            D_code("""
+            attrs.fraction_of_words_with_alpha_character = sum(
+            1 for word in words if any(c.isalpha() for c in word)
+        ) / max(word_count, 1)
+            """, block="block", language="python"),
+        ),
+        Details(
+            Summary("Implementations from RedPajama-V2"),
+            D_code("""
+            class RPS_Doc_Frac_No_Alph_Words(RPSBase):  # noqa
+                ALPH_REGEX = re.compile(r"[a-zA-Z]")
+            
+                __slots__ = ()
+            
+                def __call__(self, document: Document) -> SignalType:
+                    num_words = document.num_raw_words
+            
+                    if num_words == 0:
+                        return [(0, len(document), None)]
+            
+                    num_words_with_alpha = float(sum(
+                        int(self.ALPH_REGEX.search(word) is not None)
+                        for word in document.raw_words
+                    ))
+            
+                    score = 1.0 - num_words_with_alpha / num_words
+                    score = round(score, PRECISION)
+                    return [(0, len(document), score)]
+            """, block="block", language="python"),
+        ),
+        Details(
+            Summary("Implementations from DataTrove"),
+            D_code("""
+            # that 80 % of words in a document contain at least one alphabetic character
+            if (
+                self.max_non_alpha_words_ratio
+                and sum([any((c.isalpha() for c in w)) for w in words]) / n_words < self.max_non_alpha_words_ratio
+            ):
+                return False, "gopher_below_alpha_threshold"
+            """, block="block", language="python"),
+        ),
         P("""
-        Implementations from Dolma
-        Implementations from RedPajama-V2
-        Implementations from DataTrove
-       
         Both Dolma and DataTrove use `char.isalpha()` to detect whether a word contains alphabetic characters while 
         RedPajama-V2 employs regular expressions for this purpose. We opt to use regular expressions since `char.isalpha()` 
         can also match words in other languages as long as they are not punctuations.
         """),
-        Img(),
         H5("Number of Stop Words"),
         P("""
         The implementations across existing pipelines are largely identical. We adopt them and apply them to our pipeline.
@@ -841,11 +921,11 @@ def web_data():
         P("""
         After careful filtering, although data quality has improved, a large fraction of the content is repeated across documents. This may be due to the crawler indirectly hitting the same page multiple times, to boilerplate content being repeated (e.g., licences), or even to plagiarism. These duplicates can strongly impact models, favoring memorization instead of generalization.
         """),  # Add detailed content and images as needed
-        P("We perform two-level deduplication: local exact deduplication and global fuzzy deduplication")
-        P(B("Local Exact Deduplication"))
-        P("To reduce the expensive cost of global deduplication, we apply a local exact deduplication before it. Specifically, each dump is split into 70 splits. A bloom filter is applied within each split.")
-        P(B("Global Fuzzy Deduplication"))
-        P("NEED TO UPDATE")
+        P("We perform two-level deduplication: local exact deduplication and global fuzzy deduplication"),
+        P(B("Local Exact Deduplication")),
+        P("To reduce the expensive cost of global deduplication, we apply a local exact deduplication before it. Specifically, each dump is split into 70 splits. A bloom filter is applied within each split."),
+        P(B("Global Fuzzy Deduplication")),
+        P("NEED TO UPDATE"),
         H3("5. PII Removal"),
         P("..."),  # Add detailed content and images as needed
         H2("Reference"),