m-ric HF staff commited on
Commit
6161180
·
verified ·
1 Parent(s): 5193146

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -3
app.py CHANGED
@@ -40,7 +40,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
40
  if splitter_selection == LABEL_TEXTSPLITTER:
41
  text_splitter = CharacterTextSplitter(
42
  chunk_size=length,
43
- chunk_overlap=chunk_overlap,
44
  length_function=length_function,
45
  strip_whitespace=False,
46
  is_separator_regex=False,
@@ -49,7 +49,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
49
  elif splitter_selection == LABEL_RECURSIVE:
50
  text_splitter = RecursiveCharacterTextSplitter(
51
  chunk_size=length,
52
- chunk_overlap=chunk_overlap,
53
  length_function=length_function,
54
  strip_whitespace=False,
55
  separators=separators,
@@ -59,7 +59,7 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
59
 
60
  unoverlapped_text_splits = unoverlap_list(text_splits)
61
 
62
- output = [((split[0], 0) if split[1] else (split[0], str(i+1))) for i, split in enumerate(unoverlapped_text_splits)]
63
  return output
64
 
65
 
 
40
  if splitter_selection == LABEL_TEXTSPLITTER:
41
  text_splitter = CharacterTextSplitter(
42
  chunk_size=length,
43
+ chunk_overlap=0,
44
  length_function=length_function,
45
  strip_whitespace=False,
46
  is_separator_regex=False,
 
49
  elif splitter_selection == LABEL_RECURSIVE:
50
  text_splitter = RecursiveCharacterTextSplitter(
51
  chunk_size=length,
52
+ chunk_overlap=0,
53
  length_function=length_function,
54
  strip_whitespace=False,
55
  separators=separators,
 
59
 
60
  unoverlapped_text_splits = unoverlap_list(text_splits)
61
 
62
+ output = [(split[0], str(i+1)) for i, split in enumerate(unoverlapped_text_splits)]
63
  return output
64
 
65