Upload app.py

app.py

@@ -1,47 +1,93 @@
 import gradio as gr
 from haystack import Document
 from haystack.nodes import PreProcessor
-from langchain.text_splitter import CharacterTextSplitter
+from langchain.text_splitter import (
+    CharacterTextSplitter,
+    RecursiveCharacterTextSplitter,
+)
 
 preprocessor = PreProcessor(
     clean_empty_lines=True,
     clean_whitespace=True,
     clean_header_footer=True,
-  	remove_substrings=None,
-  	max_chars_check=10_000
+    remove_substrings=None,
+    max_chars_check=10_000,
 )
+import difflib
+
+
+def separate_overlap(s1, s2):
+    for i in range(len(s1) - len(s2), len(s1)):
+        if s1[i:] == s2[: len(s1) - i]:
+            overlap = s1[i:]
+            return [s1[:i], overlap, s2[len(s1) - i :]]
+    # if no overlap is found, return the strings
+    return [s1, s2]
+
 
-def chunk(text, words, splitter_selection):
+def extract_overlaps(list):
+    i = 0
+    annotated_list = [[el, i] for i, el in enumerate(list)]
+
+    while i < len(annotated_list) - 1:
+        separated = separate_overlap(annotated_list[i][0], annotated_list[i + 1][0])
+        if len(separated) == 2:
+            i += 1
+        elif len(separated) == 3:
+            annotated_list[i][0] = separated[0]
+            annotated_list.insert(i + 1, [separated[1], "overlap"])
+            annotated_list[i + 2][0] = separated[2]
+            i += 2
+    return annotated_list
+
+
+def chunk(text, words, splitter_selection, slider_overlap):
     if "Word" in splitter_selection:
         splits = preprocessor.split(
             Document(text),
             split_length=words,
             split_by="word",
-            split_overlap=0,
-            split_respect_sentence_boundary=("respect sentence boundaries" in splitter_selection),
+            split_overlap=slider_overlap,
+            split_respect_sentence_boundary=(
+                "respect sentence boundaries" in splitter_selection
+            ),
         )
         text_splits = [split.content for split in splits]
-    if splitter_selection == "Character":
-        text_splitter=CharacterTextSplitter(
+    elif splitter_selection == "Character":
+        text_splitter = CharacterTextSplitter(
             separator="",
             chunk_size=words,
-            chunk_overlap=0,
+            chunk_overlap=slider_overlap,
             length_function=len,
             is_separator_regex=False,
         )
         splits = text_splitter.create_documents([text])
         text_splits = [split.page_content for split in splits]
+    elif splitter_selection == "Recursive Character Text Splitter":
+        text_splitter = RecursiveCharacterTextSplitter(
+            separator="",
+            chunk_size=words,
+            chunk_overlap=slider_overlap,
+            length_function=len,
+            add_start_index=True,
+            separators=["\n\n", "\n", ".", " ", ""],
+        )
+        splits = text_splitter.create_documents([text])
+        text_splits = [split.page_content for split in splits]
 
+    if slider_overlap > 0:
+        output = extract_overlaps(text_splits)
 
-
-    output = [(split, str(i)) for i, split in enumerate(text_splits)]
-    return(output)
+    else:
+        output = [(split, str(i)) for i, split in enumerate(text_splits)]
+    return output
 
 
 ESSAY = """
 Chapter 6
 WHAT SORT OF DESPOTISM DEMOCRATIC NATIONS HAVE TO FEAR
-I HAD remarked during my stay in the United States that a democratic state of society, similar to that of the Americans, might offer singular facilities for the establishment of despotism; and I perceived, upon my return to Europe, how much use had already been made, by most of our rulers, of the notions, the sentiments, and the wants created by this same social condition, for the purpose of extending the circle of their power. This led me to think that the nations of Christendom would perhaps eventually undergo some oppression like that which hung over several of the nations of the ancient world. .
+
+I had remarked during my stay in the United States that a democratic state of society, similar to that of the Americans, might offer singular facilities for the establishment of despotism; and I perceived, upon my return to Europe, how much use had already been made, by most of our rulers, of the notions, the sentiments, and the wants created by this same social condition, for the purpose of extending the circle of their power. This led me to think that the nations of Christendom would perhaps eventually undergo some oppression like that which hung over several of the nations of the ancient world. .
 
 A more accurate examination of the subject, and five years of further meditation, have not diminished my fears, but have changed their object.
 
@@ -49,23 +95,79 @@ No sovereign ever lived in former ages so absolute or so powerful as to undertak
 
 When the Roman emperors were at the height of their power, the different nations of the empire still preserved usages and customs of great diversity; although they were subject to the same monarch, most of the provinces were separately administered; they abounded in powerful and active municipalities; and although the whole government of the empire was centered in the hands of the Emperor alone and he always remained, in case of need, the supreme arbiter in all matters, yet the details of social life and private occupations lay for the most part beyond his control. The emperors possessed, it is true, an immense and unchecked power, which allowed them to gratify all their whimsical tastes and to employ for that purpose the whole strength of the state. They frequently abused that power arbitrarily to deprive their subjects of property or of life; their tyranny was extremely onerous to the few, but it did not reach the many; it was confined to some few main objects and neglected the rest; it was violent, but its range was limited.
 
-It would seem that if despotism were to be established among the democratic nations of our days, it might assume a different character; it would be more extensive and more mild; it would degrade men without tormenting them. I do not question that, in an age of instruction and equality like our own, sovereigns might more easily succeed in collecting all political power into their own hands and might interfere more habitually and decidedly with the circle of private interests than any sovereign of antiquity could ever do. But this same principle of equality which facilitates despotism tempers its rigor. We have seen how the customs of society become more humane and gentle in proportion as men become more equal and alike. When no member of the community has much power or much wealth, tyranny is, as it were, without opportunities and a field of action. As all fortunes are scanty, the passions of men are naturally circumscribed, their imagination limited, their pleasures simple. This universal moderation moderates the sovereign himself and checks within certain limits the inordinate stretch of his desires.
+---
+
+Then you can [Create a dataset repository](../huggingface_hub/quick-start#create-a-repository), for example using:
+
+```python
+from huggingface_hub import HfApi
+
+HfApi().create_repo(repo_id="username/my_dataset", repo_type="dataset")
+```
+
+Finally, you can use [Hugging Face paths]([Hugging Face paths](https://huggingface.co/docs/huggingface_hub/guides/hf_file_system#integrations)) in Pandas:
+
+```python
+import pandas as pd
+
+df.to_parquet("hf://datasets/username/my_dataset/data.parquet")
+
+# or write in separate files if the dataset has train/validation/test splits
+df_train.to_parquet("hf://datasets/username/my_dataset/train.parquet")
+df_valid.to_parquet("hf://datasets/username/my_dataset/validation.parquet")
+df_test .to_parquet("hf://datasets/username/my_dataset/test.parquet")
+```
 
-Independently of these reasons, drawn from the nature of the state of society itself, I might add many others arising from causes beyond my subject; but I shall keep within the limits I have laid down.
 
-Democratic governments may become violent and even cruel at certain periods of extreme effervescence or of great danger, but these crises will be rare and brief. When I consider the petty passions of our contemporaries, the mildness of their manners, the extent of their education, the purity of their religion, the gentleness of their morality, their regular and industrious habits, and the restraint which they almost all observe in their vices no less than in their virtues, I have no fear that they will meet with tyrants in their rulers, but rather with guardians.1
 """
+
+
 with gr.Blocks(theme=gr.themes.Soft()) as demo:
     text = gr.Textbox(label="Your text 🪶", value=ESSAY)
-    split_selection = gr.Radio(["Word - respect sentence boundaries", "Word - no sentence boundaries", "Character"], value='Character', label='Chunking method 🍞', info='How should we split our chunks?')
-    slider = gr.Slider(20, 500, value=50, label="Count 🧮", info="Chunk size, in the chosen unit.")
+    split_selection = gr.Radio(
+        [
+            "Word - respect sentence boundaries",
+            "Word - no sentence boundaries",
+            "Recursive Character Text Splitter",
+            "Character",
+        ],
+        value="Character",
+        label="Chunking method ",
+        info="How should we split our chunks",
+    )
+    slider_count = gr.Slider(
+        20, 500, value=50, label="Count 🧮", info="Chunk size, in the chosen unit."
+    )
+    slider_overlap = gr.Slider(
+        0,
+        100,
+        value=0,
+        label="Overlap 🔀",
+        info="Size of overlap between adjacent chunks.",
+    )
     out = gr.HighlightedText(
         label="Output",
         show_legend=True,
         show_label=False,
     )
-    text.change(fn=chunk, inputs=[text, slider, split_selection], outputs=out)
-    split_selection.change(fn=chunk, inputs=[text, slider, split_selection], outputs=out)
-    slider.change(fn=chunk, inputs=[text, slider, split_selection], outputs=out)
-
-demo.launch()
+    text.change(
+        fn=chunk,
+        inputs=[text, slider_count, split_selection, slider_overlap],
+        outputs=out,
+    )
+    split_selection.change(
+        fn=chunk,
+        inputs=[text, slider_count, split_selection, slider_overlap],
+        outputs=out,
+    )
+    slider_count.change(
+        fn=chunk,
+        inputs=[text, slider_count, split_selection, slider_overlap],
+        outputs=out,
+    )
+    slider_overlap.change(
+        fn=chunk,
+        inputs=[text, slider_count, split_selection, slider_overlap],
+        outputs=out,
+    )
+demo.launch()