Update common.py

common.py

@@ -26,6 +26,14 @@ table_html_nfc = nfc_examples.to_html(index=False, border=0)
 table_div_nfc_examples = Div(NotStr(table_html_nfc), style="margin: 40px;")
 
 
+dask_algo = """
+dask.bag.from_sequence(doc_file_paths)
+.map_partitions(stream_docs)
+.groupby(lambda doc: doc["hash"])
+.map_partitions(make_doc_pairs)
+.compute()
+"""
+
 global_div = Div(
         Section(
             H2("Global Steps"),
@@ -49,13 +57,7 @@ global_div = Div(
             H3("Matching Pairs Generation"),
             P("We are using a Jaccard similarity threshold of 0.8 to identify near-duplicate documents. To do this, we divide the MinHashes into 9 bands, each with 13 hashes (also known as the range). To save memory during matching, we first store each band of MinHashes separately on disk. We then process each band individually. Within each band, documents are matched based on their hashes, and the matches are saved as document pairs. A document is considered a match if it matches another document in any of the 9 bands. Since we are looking for near-duplicates, a document may match multiple documents across different bands."),
             P("For partitioning and matching the hashes, we utilize Dask's bag data structure to load the document ids and MinHashes. The matching process is simply a group by operation on this bag data structure. This approach allows us to group matches efficiently and distribute the operation to multiple machines. Also doing a group by produces full components (documents that share the same signature) within a band which simplifies the later stages. The algorithm can be expressed using the Dask expression below:"),
-            Code("""
-                dask.bag.from_sequence(doc_file_paths)
-                .map_partitions(stream_docs)
-                .groupby(lambda doc: doc["hash"])
-                .map_partitions(make_doc_pairs)
-                .compute()
-            """), 
+            Div(Code(dask_algo), cls= "code-block"), 
             P("This step produced 9.2 TB of matching pairs from all bands."),
         ),
     Section(