no message

main.py

@@ -108,31 +108,33 @@ def reduce_tokens(text: str):
     token_count = len(reduced_doc)
     return reduced_text, token_count
 
-def segment_text(text: str, max_length=512):
+def segment_text(text: str, max_tokens=500):  # Slightly less than 512 for safety
     # Use spaCy to divide the document into sentences
     doc = nlp(text)
-    sentences = [sent.text for sent in doc.sents]
+    sentences = [sent.text.strip() for sent in doc.sents]
     
-    # Group sentences into segments of approximately max_length tokens
     segments = []
     current_segment = []
     current_length = 0
     
     for sentence in sentences:
-        sentence_length = len(sentence.split())
-        if current_length + sentence_length > max_length:
-            segments.append(' '.join(current_segment))
+        sentence_length = len(sentence.split())  # Simple word count
+        if current_length + sentence_length > max_tokens:
+            if current_segment:  # Make sure there's something to add
+                segments.append(' '.join(current_segment))
             current_segment = [sentence]
             current_length = sentence_length
         else:
             current_segment.append(sentence)
             current_length += sentence_length
     
+    # Add the last segment if any
     if current_segment:
         segments.append(' '.join(current_segment))
     
     return segments
 
+
 classifier = pipeline("text-classification", model="distilbert-base-uncased-finetuned-sst-2-english")
 
 def classify_segments(segments):
@@ -146,11 +148,25 @@ async def summarize(request: TextRequest):
         processed_text = preprocess_text(request.text)
         segments = segment_text(processed_text)
 
-        # Classify each segment
-        classified_segments = classify_segments(segments)
-
-        # Optionally, reduce tokens for some specific task or summarize
-        reduced_texts = [reduce_tokens(segment)[0] for segment in segments]
+        # Classify each segment safely
+        classified_segments = []
+        for segment in segments:
+            try:
+                result = classifier(segment)
+                classified_segments.append(result)
+            except Exception as e:
+                print(f"Error classifying segment: {e}")
+                classified_segments.append({"error": str(e)})
+
+        # Optional: Reduce tokens or summarize
+        reduced_texts = []
+        for segment in segments:
+            try:
+                reduced_text, token_count = reduce_tokens(segment)
+                reduced_texts.append((reduced_text, token_count))
+            except Exception as e:
+                print(f"Error during token reduction: {e}")
+                reduced_texts.append(("Error", 0))
 
         return {
             "classified_segments": classified_segments,