Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -35,6 +35,7 @@ def change_split_selection(split_selection):
|
|
35 |
|
36 |
def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
|
37 |
separators = extract_separators_from_string(separators_str)
|
|
|
38 |
length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
|
39 |
if splitter_selection == LABEL_TEXTSPLITTER:
|
40 |
text_splitter = CharacterTextSplitter(
|
@@ -63,12 +64,14 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
|
|
63 |
|
64 |
def change_preset_separators(choice):
|
65 |
text_splitter = RecursiveCharacterTextSplitter()
|
66 |
-
if choice == "Default":
|
67 |
return ["\n\n", "\n", " ", ""]
|
68 |
elif choice == "Markdown":
|
69 |
return text_splitter.get_separators_for_language(Language.MARKDOWN)
|
70 |
elif choice == "Python":
|
71 |
return text_splitter.get_separators_for_language(Language.PYTHON)
|
|
|
|
|
72 |
|
73 |
|
74 |
EXAMPLE_TEXT = """### Chapter 6
|
|
|
35 |
|
36 |
def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
|
37 |
separators = extract_separators_from_string(separators_str)
|
38 |
+
print('Extracted separators:', ',,'.join(separators))
|
39 |
length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
|
40 |
if splitter_selection == LABEL_TEXTSPLITTER:
|
41 |
text_splitter = CharacterTextSplitter(
|
|
|
64 |
|
65 |
def change_preset_separators(choice):
|
66 |
text_splitter = RecursiveCharacterTextSplitter()
|
67 |
+
if choice == "Default recursive":
|
68 |
return ["\n\n", "\n", " ", ""]
|
69 |
elif choice == "Markdown":
|
70 |
return text_splitter.get_separators_for_language(Language.MARKDOWN)
|
71 |
elif choice == "Python":
|
72 |
return text_splitter.get_separators_for_language(Language.PYTHON)
|
73 |
+
else:
|
74 |
+
raise gr.Error("Choice of preset not recognized.")
|
75 |
|
76 |
|
77 |
EXAMPLE_TEXT = """### Chapter 6
|