Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -2,6 +2,7 @@ import gradio as gr
|
|
2 |
from langchain.text_splitter import (
|
3 |
CharacterTextSplitter,
|
4 |
RecursiveCharacterTextSplitter,
|
|
|
5 |
)
|
6 |
from transformers import AutoTokenizer
|
7 |
from overlap import unoverlap_list
|
@@ -28,6 +29,7 @@ def extract_separators_from_string(separators_str):
|
|
28 |
def change_split_selection(text, slider_count, split_selection, separators_selection, length_unit_selection, chunk_overlap):
|
29 |
return (
|
30 |
gr.Textbox.update(visible=(split_selection==LABEL_RECURSIVE)),
|
|
|
31 |
chunk(text, slider_count, split_selection, separators_selection, length_unit_selection, chunk_overlap)
|
32 |
)
|
33 |
|
@@ -134,10 +136,11 @@ with gr.Blocks(theme=gr.themes.Soft(text_size='lg', font=["monospace"], primary_
|
|
134 |
show_label=False, # or set label to an empty string if you want to keep its space
|
135 |
visible=False,
|
136 |
)
|
137 |
-
|
138 |
-
['Default', 'Python', 'Markdown'],
|
139 |
label="Choose a preset",
|
140 |
-
info="This will
|
|
|
141 |
)
|
142 |
with gr.Row():
|
143 |
length_unit_selection = gr.Dropdown(
|
@@ -165,12 +168,12 @@ with gr.Blocks(theme=gr.themes.Soft(text_size='lg', font=["monospace"], primary_
|
|
165 |
split_selection.change(
|
166 |
fn=change_split_selection,
|
167 |
inputs=[text, slider_count, split_selection, separators_selection, length_unit_selection, chunk_overlap],
|
168 |
-
outputs=[separators_selection, out],
|
169 |
)
|
170 |
-
|
171 |
fn=change_preset_separators,
|
172 |
-
inputs=
|
173 |
-
outputs=separators_selection,
|
174 |
)
|
175 |
gr.on(
|
176 |
[text.change, length_unit_selection.change, separators_selection.change, slider_count.change, chunk_overlap.change],
|
|
|
2 |
from langchain.text_splitter import (
|
3 |
CharacterTextSplitter,
|
4 |
RecursiveCharacterTextSplitter,
|
5 |
+
Language,
|
6 |
)
|
7 |
from transformers import AutoTokenizer
|
8 |
from overlap import unoverlap_list
|
|
|
29 |
def change_split_selection(text, slider_count, split_selection, separators_selection, length_unit_selection, chunk_overlap):
|
30 |
return (
|
31 |
gr.Textbox.update(visible=(split_selection==LABEL_RECURSIVE)),
|
32 |
+
gr.Radio.update(visible=(split_selection==LABEL_RECURSIVE)),
|
33 |
chunk(text, slider_count, split_selection, separators_selection, length_unit_selection, chunk_overlap)
|
34 |
)
|
35 |
|
|
|
136 |
show_label=False, # or set label to an empty string if you want to keep its space
|
137 |
visible=False,
|
138 |
)
|
139 |
+
separator_preset_selection = gr.Radio(
|
140 |
+
['Default recursive', 'Python', 'Markdown'],
|
141 |
label="Choose a preset",
|
142 |
+
info="This will apply a specific set of separators to RecursiveCharacterTextSplitter.",
|
143 |
+
visible=False,
|
144 |
)
|
145 |
with gr.Row():
|
146 |
length_unit_selection = gr.Dropdown(
|
|
|
168 |
split_selection.change(
|
169 |
fn=change_split_selection,
|
170 |
inputs=[text, slider_count, split_selection, separators_selection, length_unit_selection, chunk_overlap],
|
171 |
+
outputs=[separators_selection, separator_preset_selection, out],
|
172 |
)
|
173 |
+
separator_preset_selection.change(
|
174 |
fn=change_preset_separators,
|
175 |
+
inputs=separator_preset_selection,
|
176 |
+
outputs=[separators_selection],
|
177 |
)
|
178 |
gr.on(
|
179 |
[text.change, length_unit_selection.change, separators_selection.change, slider_count.change, chunk_overlap.change],
|