truncation bug fix

.gitignore

@@ -4,8 +4,5 @@
 *.xlsx
 /testing/
 /modules/__pycache__/
-/logs/
 app.log
-logs
-logs/
 /sandbox/

app.py

@@ -131,7 +131,7 @@ def main():
                 logger.info(f"File uploaded: {uploaded_file.name}")
                 
                 if not st.session_state['data_processed']:
-                    logger.info("Starting data processing...")
+                    logger.info("Starting data processing")
                     try:
                         st.session_state['df'] = process_data(uploaded_file, sens_level)
                         logger.info("Data processing completed successfully")
@@ -141,8 +141,6 @@ def main():
                         raise
 
                 df = st.session_state['df']
-                logger.info(f"DataFrame columns: {list(df.columns)}")
-                logger.info(f"DataFrame shape: {df.shape}")
 
                 current_datetime = datetime.now().strftime('%d-%m-%Y_%H-%M-%S')
                 output_filename = f'processed_applications_{current_datetime}.csv'

modules/utils.py

@@ -88,32 +88,47 @@ def predict_category(df, model_name, progress_bar, repo, profile, multilabel=Fal
         col_name = re.sub(r'_(.*)', r'_txt', model_name)
         model = SetFitModel.from_pretrained(profile+"/"+repo)
         model.to(device)
+        # Get tokenizer from the model
+        tokenizer = model.model_body.tokenizer
     else:
         col_name = 'scope_txt'
-        model = pipeline("text-classification", model=profile+"/"+repo, device=device, return_all_scores=multilabel)
+        model = pipeline("text-classification", 
+                        model=profile+"/"+repo, 
+                        device=device, 
+                        return_all_scores=multilabel,
+                        truncation=True,
+                        max_length=512)   
     predictions = []
     total = len(df)
     for i, text in enumerate(df[col_name]):
-        prediction = model(text)
-        if model_name in model_names_sf:
-            predictions.append(0 if prediction == 'NEGATIVE' else 1)
-        elif model_name == 'ADAPMIT':
-            predictions.append(re.sub('Label$', '', prediction[0]['label']))
-        elif model_name == 'SECTOR':
-            predictions.append(extract_predicted_labels(prediction[0], threshold=0.5))
-        elif model_name == 'LANG':
-            predictions.append(prediction[0]['label'])
+        try:
+            if model_name in model_names_sf:
+                # Truncate text for SetFit models
+                encoded = tokenizer(text, truncation=True, max_length=512)
+                truncated_text = tokenizer.decode(encoded['input_ids'])
+                prediction = model(truncated_text)
+                predictions.append(0 if prediction == 'NEGATIVE' else 1)
+            else:
+                prediction = model(text)
+                if model_name == 'ADAPMIT':
+                    predictions.append(re.sub('Label$', '', prediction[0]['label']))
+                elif model_name == 'SECTOR':
+                    predictions.append(extract_predicted_labels(prediction[0], threshold=0.5))
+                elif model_name == 'LANG':
+                    predictions.append(prediction[0]['label'])
+        except Exception as e:
+            logger.error(f"Error processing sample {df['id'][i]}: {str(e)}")
+            st.error("Application Error. Please contact support.")
         # Update progress bar with each iteration
         progress = (i + 1) / total
         progress_bar.progress(progress)
-    # st.write(predictions)
     return predictions
 
 
 # Main function to process data
 def process_data(uploaded_file, sens_level):
     df = pd.read_excel(uploaded_file)
-    logger.info(f"data import successful")
+    logger.info(f"Data import successful")
     # Rename columns
     df.rename(columns={
         'id': 'id',
@@ -147,6 +162,7 @@ def process_data(uploaded_file, sens_level):
     step_count = 0
     total_steps = len(model_names)
     for model_name in model_names:
+        logger.info(f"Loading: {model_name}")
         step_count += 1
         model_processing_text = st.empty()
         model_processing_text.markdown(f'**Current Task: Processing with model "{model_name}"**')
@@ -165,6 +181,7 @@ def process_data(uploaded_file, sens_level):
         elif model_name == 'LANG':
             df[model_name] = predict_category(df, model_name, progress_bar, repo='51-languages-classifier', profile='qanastek')
         
+        logger.info(f"Completed: {model_name}")
         model_progress.empty()
 
         progress_count += len(df)