view pipeline result

app.py

@@ -1,9 +1,18 @@
+import ast
+import glob
+import tempfile
+from dataclasses import asdict
 from itertools import islice
+from functools import partial
+from typing import Optional
 
 import gradio as gr
+import nltk
+import pandas as pd
+from datatrove.utils.typeshelper import Languages
 from datatrove.executor.local import LocalPipelineExecutor
-from datatrove.pipeline.base import PipelineStep
 from datatrove.pipeline.extractors import Trafilatura
+from datatrove.pipeline.filters.base_filter import BaseFilter
 from datatrove.pipeline.filters import (
     C4QualityFilter,
     FineWebQualityFilter,
@@ -12,13 +21,378 @@ from datatrove.pipeline.filters import (
     LanguageFilter,
     URLFilter,
 )
-from datatrove.pipeline.readers import WarcReader
+from datatrove.pipeline.formatters import PIIFormatter
+from datatrove.pipeline.readers import JsonlReader, WarcReader
 from datatrove.pipeline.writers.jsonl import JsonlWriter
+from difflib import Differ
 
 
-def run(input):
-    return "wip"
+nltk.download('punkt_tab')
+DUMP_TO_PROCESS = "CC-MAIN-2023-50"
+default_output_docs_2k = pd.read_json(f"output_all-2k/base_processing/output/{DUMP_TO_PROCESS}/00000.jsonl.gz", compression="gzip", lines=True).to_dict(orient="records")
+default_output_docs_200 = pd.read_json(f"output_all-200/base_processing/output/{DUMP_TO_PROCESS}/00000.jsonl.gz", compression="gzip", lines=True).to_dict(orient="records")
+
+make_gallery_image_buttons_js = """
+function load() {
+    class ClassWatcher {
+
+        constructor(targetNode, classToWatch, classAddedCallback, arg) {
+            this.targetNode = targetNode
+            this.classToWatch = classToWatch
+            this.classAddedCallback = classAddedCallback
+            this.arg = arg
+            this.observer = null
+            this.lastClassState = targetNode.classList.contains(this.classToWatch)
+
+            this.init()
+        }
+
+        init() {
+            this.observer = new MutationObserver(this.mutationCallback)
+            this.observe()
+        }
+
+        observe() {
+            this.observer.observe(this.targetNode, { attributes: true })
+        }
+
+        disconnect() {
+            this.observer.disconnect()
+        }
+
+        mutationCallback = mutationsList => {
+            for (let mutation of mutationsList) {
+                if (mutation.type === 'attributes' && mutation.attributeName === 'class') {
+                    let currentClassState = mutation.target.classList.contains(this.classToWatch)
+                    if(this.lastClassState !== currentClassState) {
+                        this.lastClassState = currentClassState
+                        if(currentClassState) {
+                            this.classAddedCallback(this.arg)
+                        }
+                    }
+                }
+            }
+        }
+    }
+    let buttons = document.getElementsByClassName("block-button");
+    function clickButton(i) {
+        buttons[i].click();
+    }
+    Array.from(document.getElementById("pipeline-gallery").getElementsByClassName("thumbnail-item")).map(
+        (b, i) => new ClassWatcher(b, 'selected', clickButton, i)
+    )
+}
+"""
+css = """
+tr:has(> td div span span div.diffInsertion) {
+    background: darkgreen;
+}
+tr:has(> td div span span div.diffDeletion) {
+    background: darkred;
+}
+tr td {
+    border-top: 1px solid black;
+}
+"""
+
+
+blocks = sorted(glob.glob("images/*.png"))
+
+
+def prepare_as_list_or_none(text: str) -> Optional[list[str]]:
+    return ([x.strip() for x in text.split(",") if x.strip()] or None) if text else None
+
+def build_code_snippet(steps, params=None):
+    # TODO
+    return (
+        "```python\n"
+        "TODO\n"
+        "```"
+    )
+
+
+with gr.Blocks(css=css, js=make_gallery_image_buttons_js) as demo:
+    state = gr.State({"selected_block": 0})
+    gr.Markdown("# Common Crawl Pipeline Creator")
+    gallery = gr.Gallery(
+        blocks,
+        columns=4,
+        rows=2,
+        label="Select step to edit",
+        object_fit="scale-down",
+        show_share_button=False,
+        show_download_button=False,
+        show_fullscreen_button=False,
+        elem_id="pipeline-gallery",
+        height="200px",
+        allow_preview=False,
+    )
+    gallery_image_buttons = [gr.Button(visible=False, elem_classes="block-button") for _ in blocks]  # hack to simulate each image galery as a button, see `make_gallery_image_buttons_js``
+    blocks_uis = []
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 1. URL Filtering \n\nPerforms filtering based on samples urls.")
+        with gr.Group():
+            url_filtering_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion("Parameters", open=True) as acc:
+                use_integrated_lists_checkbox = gr.Checkbox(True, label="use_integrated_lists", info="use the datatrove integrated lists of banned urls and words")
+                with gr.Row():
+                    with gr.Column():
+                        extra_domain_textbox = gr.Textbox("", label="extra_domains", info="remove if the domain is present in `extra_domains`")
+                        extra_domain_textbox.prepare_parameter = prepare_as_list_or_none
+                        extra_urls_textbox = gr.Textbox("", label="extra_urls", info="remove if the full url is present on `extra_urls`")
+                        extra_urls_textbox.prepare_parameter = prepare_as_list_or_none
+                    with gr.Column():
+                        banned_words_textbox = gr.Textbox("", label="banned_words", info="remove if any word from `banned_words` is in the url")
+                        banned_words_textbox.prepare_parameter = prepare_as_list_or_none
+                        banned_subwords_textbox = gr.Textbox("", label="banned_subwords", info="remove if any word from `banned_subwords` is a substring of the url")
+                        banned_subwords_textbox.prepare_parameter = prepare_as_list_or_none
+                    with gr.Column():
+                        soft_banned_words_textbox = gr.Textbox("", label="soft_banned_words", info="remove if there are at least `soft_word_threshold` words from `soft_banned_words` in the url")
+                        soft_banned_words_textbox.prepare_parameter = prepare_as_list_or_none
+                        soft_word_threshold_slider = gr.Slider(0, 5, value=2, step=1, label="soft_word_threshold", info="remove if there are at least `soft_word_threshold` words from `soft_banned_words` in the url")
+            url_filtering_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=url_filtering_checkbox, outputs=acc)
+        url_filtering_parameters_components = [use_integrated_lists_checkbox, extra_domain_textbox, extra_urls_textbox, banned_words_textbox, banned_subwords_textbox, soft_banned_words_textbox, soft_word_threshold_slider]
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 2. Text Extraction \n\nUses the [Trafilatura](https://trafilatura.readthedocs.io) extractor.")
+        with gr.Group():
+            text_extraction_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion("Parameters", open=True) as acc:
+                with gr.Row():
+                    favour_precision_checkbox = gr.Checkbox(True, label="favour_precision", info="prefer less text but correct extraction")
+                    timeout_slider = gr.Slider(0.05, 0.5, value=0.1, step=0.05, label="timeout", info="the timeout for extraction, per document, in seconds")
+                    deduplicate_checkbox = gr.Checkbox(True, label="deduplicate", info="trafilatura's deduplicate option")
+            text_extraction_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=text_extraction_checkbox, outputs=acc)
+        text_extraction_parameters_components = [favour_precision_checkbox, timeout_slider, deduplicate_checkbox]
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 3. Language Filtering \n\nUses the [fastext](https://fasttext.cc/docs/en/language-identification.html) language identification models.")
+        with gr.Group():
+            language_filtering_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion("Parameters", open=True) as acc:
+                with gr.Row():
+                    languages_textbox = gr.Textbox("", label="languages", info="list of languages to keep. empty for all")
+                    languages_textbox.prepare_parameter = prepare_as_list_or_none
+                    language_threshold_slider = gr.Slider(0, 1, value=0.65, step=0.05, label="language_threshold", info="minimum score to accept a document")
+            language_filtering_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=language_filtering_checkbox, outputs=acc)
+        language_filtering_parameters_components = [languages_textbox, language_threshold_slider]
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 4. Gopher Filtering (repetitions) \n\nUses the [Gopher](https://huggingface.co/papers/2112.11446) text repetition filters.")
+        with gr.Group():
+            gopher_filtering_repetitions_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion("Parameters", open=True) as acc:
+                with gr.Group():
+                    with gr.Row():
+                        language_dropdown1 = gr.Dropdown([v for k, v in vars(Languages).items() if not k.startswith("__")], value=Languages.english, label="language", info="tokenizer language")
+                        top_n_grams_textbox = gr.Textbox("(2, 0.2), (3, 0.18), (4, 0.16)", label="top_n_grams")
+                        top_n_grams_textbox.prepare_parameter = ast.literal_eval
+                        dup_n_grams_textbox = gr.Textbox("(5, 0.15), (6, 0.14), (7, 0.13), (8, 0.12), (9, 0.11), (10, 0.10)", label="dup_n_grams")
+                        dup_n_grams_textbox.prepare_parameter = ast.literal_eval
+                    with gr.Row():
+                        dup_line_frac_slider = gr.Slider(0, 1, value=0.3, step=0.05, label="dup_line_frac")
+                        dup_para_frac_slider = gr.Slider(0, 1, value=0.3, step=0.05, label="dup_para_frac")
+                        dup_line_char_frac_slider = gr.Slider(0, 1, value=0.2, step=0.05, label="dup_line_char_frac")
+                        dup_para_char_frac_slider = gr.Slider(0, 1, value=0.2, step=0.05, label="dup_para_char_frac")
+            gopher_filtering_repetitions_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=gopher_filtering_repetitions_checkbox, outputs=acc)
+        gopher_filtering_repetitions_parameters_components = [language_dropdown1, top_n_grams_textbox, dup_n_grams_textbox, dup_line_frac_slider, dup_para_frac_slider, dup_line_char_frac_slider, dup_para_char_frac_slider]
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 8. PII Removal \n\nReplaces email addresses and ip addresses in the document text.")
+        with gr.Group():
+            pii_removal_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion("Parameters", open=True) as acc:
+                    with gr.Row():
+                        remove_emails_checkbox = gr.Checkbox(True, label="remove_emails", info="Replace email addresses")
+                        remove_ips_checkbox = gr.Checkbox(True, label="remove_ips", info="Replace IP addresses")
+                        only_remove_public_ips_checkbox = gr.Checkbox(True, label="only_remove_public_ips", info="by default we only replace public (and thus PII) IPs")
+                    with gr.Row():
+                        email_replacement_textbox = gr.Textbox("email@example.com, firstname.lastname@example.org", label="email_replacement", info="strings to use as replacement. They will be used in a circular way")
+                        email_replacement_textbox.prepare_parameter = prepare_as_list_or_none
+                        ip_replacement_textbox = gr.Textbox("22.214.171.124, 126.96.36.199, 188.8.131.52, 184.108.40.206, 220.127.116.11, 18.104.22.168", label="ip_replacement", info="same as email_replacement but for IP addresses")
+                        ip_replacement_textbox.prepare_parameter = prepare_as_list_or_none
+            pii_removal_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=pii_removal_checkbox, outputs=acc)
+        pii_removal_parameters_components = [remove_emails_checkbox, remove_ips_checkbox, only_remove_public_ips_checkbox, email_replacement_textbox, ip_replacement_textbox]
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 7. Custom Filters \n\nUses the [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) custom text filters.")
+        with gr.Group():
+            custom_filters_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion("Parameters", open=True) as acc:
+                with gr.Row():
+                    line_punct_thr_slider = gr.Slider(0, 1, value=0.12, step=0.01, label="line_punct_thr")
+                    line_punct_exclude_zero = gr.Checkbox(False, label="line_punct_exclude_zero")
+                    short_line_thr_slider = gr.Slider(0, 1, value=0.67, step=0.01, label="short_line_thr")
+                    short_line_length_slider = gr.Slider(0, 100, value=30, step=1, label="short_line_length")
+                    char_duplicates_ratio_slider = gr.Slider(0, 1, value=0.01, step=0.01, label="char_duplicates_ratio")
+                    new_line_ratio_slider = gr.Slider(0, 1, value=0.3, step=0.01, label="new_line_ratio")
+            custom_filters_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=custom_filters_checkbox, outputs=acc)
+        custom_filters_parameters_components = [line_punct_thr_slider, line_punct_exclude_zero, short_line_thr_slider, short_line_length_slider, char_duplicates_ratio_slider, new_line_ratio_slider]
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 6. C4 Filters\n\nUses the [C4](https://huggingface.co/datasets/allenai/c4) text size and content filters.")
+        with gr.Group():
+            c4_filters_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion(" Parameters", open=True) as acc:
+                with gr.Group():
+                    with gr.Row():
+                        split_paragraph_checkbox = gr.Checkbox(True, label="split_paragraph", info="disable to apply the filters to each sentence instead of to each line")
+                    with gr.Row():
+                        language_dropdown2 = gr.Dropdown([v for k, v in vars(Languages).items() if not k.startswith("__")], value=Languages.english, label="language", info="tokenizer language")
+                        min_num_sentences_slider = gr.Slider(0, 10, value=5, step=1, label="min_num_sentences", info="remove documents that do not have at least this number of sentences (after line filtering)")
+                        min_words_per_line_slider = gr.Slider(0, 10, value=3, step=1, label="min_words_per_line", info="drop lines without this min number of words")
+                        max_word_length_slider = gr.Slider(0, 2000, value=1000, step=10, label="max_word_length", info=" drop lines where at least one word has more than this number of characters")
+                    with gr.Row():
+                        remove_citations_checkbox = gr.Checkbox(True, label="remove_citations", info="remove wikipedia style citations from the text")
+                        filter_no_terminal_punct_checkbox = gr.Checkbox(True, label="filter_no_terminal_punct", info="remove lines without terminal punctuation marks")
+                        filter_lorem_ipsum_checkbox = gr.Checkbox(True, label="filter_lorem_ipsum", info="drop documents that contain 'lorem ipsum'")
+                        filter_javascript_checkbox = gr.Checkbox(True, label="filter_javascript", info="drop lines mentioning 'javascript'")
+                        filter_curly_bracket = gr.Checkbox(True, label="filter_curly_bracket", info="drop documents containing {")
+                        filter_policy = gr.Checkbox(True, label="filter_policy", info="drop lines containing any of the policy phrases (e.g. 'terms of use', 'use cookies')")
+            c4_filters_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=c4_filters_checkbox, outputs=acc)
+        c4_filters_parameters_components = [split_paragraph_checkbox, language_dropdown2, min_num_sentences_slider, min_words_per_line_slider, max_word_length_slider, remove_citations_checkbox, filter_no_terminal_punct_checkbox, filter_lorem_ipsum_checkbox, filter_javascript_checkbox, filter_curly_bracket, filter_policy]
+    with gr.Column(visible=False) as col:
+        blocks_uis.append(col)
+        gr.Markdown("## 5. Gopher Filtering (quality) \n\nUses the [Gopher](https://huggingface.co/papers/2112.11446) text quality filters.")
+        with gr.Group():
+            gopher_filtering_quality_checkbox = gr.Checkbox(True, label="Enable")
+            with gr.Accordion("Parameters", open=True) as acc:
+                with gr.Group():
+                    with gr.Row():
+                        language_dropdown2 = gr.Dropdown([v for k, v in vars(Languages).items() if not k.startswith("__")], value=Languages.english, label="language", info="tokenizer language")
+                        min_doc_words_slider = gr.Slider(0, 1000, value=50, step=10, label="min_doc_words")
+                        max_doc_words_slider = gr.Slider(0, 200_000, value=100_000, step=10_000, label="max_doc_words")
+                    with gr.Row():
+                        min_avg_word_length_slider = gr.Slider(0, 20, value=3, step=1, label="min_avg_word_length")
+                        max_avg_word_length_slider = gr.Slider(0, 20, value=10, step=1, label="max_avg_word_length")
+                    with gr.Row():
+                        max_symbol_word_ratio_slider = gr.Slider(0, 1, value=0.1, step=0.05, label="max_symbol_word_ratio")
+                        max_bullet_lines_ratio_slider = gr.Slider(0, 1, value=0.9, step=0.05, label="max_bullet_lines_ratio")
+                        max_ellipsis_lines_ratio_slider = gr.Slider(0, 1, value=0.3, step=0.05, label="max_ellipsis_lines_ratio")
+                        max_non_alpha_words_ratio_slider = gr.Slider(0, 1, value=0.8, step=0.05, label="max_non_alpha_words_ratio")
+                    with gr.Row():
+                        min_stop_words_slider = gr.Slider(0, 10, value=2, step=1, label="min_stop_words")
+                        stop_words_textbox = gr.Textbox("the, be, to, of, and, that, have, with", label="stop_words")
+                        stop_words_textbox.prepare_parameter = prepare_as_list_or_none
+            gopher_filtering_quality_checkbox.change(lambda visible: gr.Accordion(visible=visible), inputs=gopher_filtering_quality_checkbox, outputs=acc)
+        gopher_filtering_quality_parameters_components = [language_dropdown2, min_doc_words_slider, max_doc_words_slider, min_avg_word_length_slider, max_avg_word_length_slider, max_symbol_word_ratio_slider, max_bullet_lines_ratio_slider, max_ellipsis_lines_ratio_slider, max_non_alpha_words_ratio_slider, min_stop_words_slider, stop_words_textbox]
+
+    view_pipeline_results_button = gr.Button("View Pipeline Results", variant="primary")
+
+    steps = [
+        URLFilter,
+        Trafilatura,
+        LanguageFilter,
+        GopherRepetitionFilter,
+        GopherQualityFilter,
+        C4QualityFilter,
+        FineWebQualityFilter,
+        PIIFormatter
+    ]
+    steps_parameters_components = [
+        url_filtering_parameters_components,
+        text_extraction_parameters_components,
+        language_filtering_parameters_components,
+        gopher_filtering_repetitions_parameters_components,
+        gopher_filtering_quality_parameters_components,
+        c4_filters_parameters_components,
+        custom_filters_parameters_components,
+        pii_removal_parameters_components
+    ]
+
+    with gr.Tab("Output") as output_tab:
+        output_dataframe_diff = gr.DataFrame(datatype="markdown")
+    with gr.Tab("Python code") as code_tab:
+        python_code_markdown = gr.Markdown(build_code_snippet(steps))
+
+
+    gr.Markdown("_powered by [datatrove](https://github.com/huggingface/datatrove)_")
+
+    def show_block_ui(i):
+        return {**{block_ui: gr.Column(visible=(j == i)) for j, block_ui in enumerate(blocks_uis)}, state: {"selected_block": i}}
+
+    for i, button in enumerate(gallery_image_buttons):
+        button.click(partial(show_block_ui, i), outputs=blocks_uis + [state])
+
+
+    inputs = [
+        url_filtering_checkbox,
+        text_extraction_checkbox,
+        language_filtering_checkbox,
+        gopher_filtering_repetitions_checkbox,
+        gopher_filtering_quality_checkbox,
+        c4_filters_checkbox,
+        custom_filters_checkbox,
+        pii_removal_checkbox
+    ] + sum(steps_parameters_components, [])
+
+    @view_pipeline_results_button.click(inputs=inputs, outputs=[output_tab, output_dataframe_diff])
+    def view_pipeline_results(*args):
+        enable_steps, steps_parameters = args[:len(steps)], args[len(steps):]
+        steps_parameters_iter = iter(steps_parameters)
+        steps_parameters = [
+            {
+                parameters_component.label: parameters_component.prepare_parameter(parameter) if hasattr(parameters_component, "prepare_parameter") else parameter
+                for parameters_component, parameter in zip(step_parameters_components, steps_parameters_iter)
+            }
+            for step_parameters_components in steps_parameters_components
+        ]
+        default_steps_parameters = [
+            {
+                parameters_component.label: parameters_component.prepare_parameter(parameters_component.value) if hasattr(parameters_component, "prepare_parameter") else parameters_component.value
+                for parameters_component in step_parameters_components
+            }
+            for step_parameters_components in steps_parameters_components
+        ]
+
+        with tempfile.TemporaryDirectory() as output_path:
+            steps_to_run = [
+                step(**step_parameters, **({"exclusion_writer": JsonlWriter(f"{output_path}/base_processing/removed/{step.__name__}/{DUMP_TO_PROCESS}")} if issubclass(step, BaseFilter) and False else {}))
+                for step, step_parameters, enable_step in zip(steps, steps_parameters, enable_steps)
+                if enable_step
+            ]
+            output_docs = []
+            if steps_parameters[:2] == default_steps_parameters[:2] and all(enable_steps[:2]):
+                num_warc_samples = 2000
+                default_output_docs = default_output_docs_2k
+                pipeline = LocalPipelineExecutor(
+                    pipeline=[
+                        JsonlReader(data_folder=f"output_text_extraction-2k/base_processing/output/{DUMP_TO_PROCESS}", glob_pattern="*.jsonl.gz")
+                    ] + steps_to_run[2:] + [
+                        lambda data, rank, world_size: map(output_docs.append, data)
+                    ],
+                    logging_dir="logs",
+                    skip_completed=False
+                )
+            else:
+                num_warc_samples = 200
+                default_output_docs = default_output_docs_200
+                pipeline = LocalPipelineExecutor(
+                    pipeline=[
+                        WarcReader(data_folder="data", glob_pattern="*.warc.gz"),
+                        lambda data, rank, world_size: islice(data, num_warc_samples),
+                    ] + steps_to_run + [
+                        lambda data, rank, world_size: map(output_docs.append, data)
+                    ],
+                    logging_dir="logs",
+                    skip_completed=False
+                )
+            pipeline.run()
+            out = [doc.text[:1_000] + f" [+{len(doc.text) - 1000} chars]" if len(doc.text) > 1_000 else doc.text for doc in output_docs]
+            default_out = [doc["text"][:1_000] + f" [+{len(doc['text']) - 1000} chars]" if len(doc["text"]) > 1_000 else doc["text"] for doc in default_output_docs]
+            output_diff = []
+            for text_diff in Differ().compare(default_out, out[:len(default_out) * 10]):
+                opcode, text = text_diff[0], text_diff[2:]
+                if opcode == "-":
+                    text = f'<div class="diffDeletion">\n\n{text}\n\n</div>'
+                elif opcode == "+":
+                    text = f'<div class="diffInsertion">\n\n{text}\n\n</div>'
+                output_diff.append(text)
+            return {
+                output_tab: gr.Tab(f"Output: kept {len(out)/num_warc_samples*100:.02f}% of data"),
+                output_dataframe_diff: pd.DataFrame({"text": output_diff}),
+            }
 
 
-demo = gr.Interface(run, inputs=[gr.Textbox()], outputs=[gr.Textbox()])
 demo.launch()

output_all-200/base_processing/output/CC-MAIN-2023-50/00000.jsonl.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7798c4f4cb22b19532ea4c5a58dbc94b72dd7627007a90dc0b2d83cf6ac56007
+size 28010

output_all-2k/base_processing/output/CC-MAIN-2023-50/00000.jsonl.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e32e8d71f3bcf050fe2d0a744b21e411c66b95ee4f88a4fa8eac5f6b459ce345
+size 299465

output_text_extraction-2k/base_processing/output/CC-MAIN-2023-50/00000.jsonl.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17adea750d20daae760f1ed8f4ebde38e6e2e7ad6d0e2b6e98d044f459a5ac8
+size 1922310

requirements.txt

@@ -1 +1,2 @@
 datatrove[all]
+lxml_html_clean