tests/test_integration.py

from course_search.search_system.data_pipeline import DataPipeline
import logging

def test_pipeline():
    pipeline = DataPipeline()
    
    # Run pipeline and save results
    df = pipeline.run_pipeline(save_path='data/courses_with_embeddings.pkl')
    
    # Print results
    print("\nDataFrame Info:")
    print(df.info())
    print("\nSample Course:")
    print(df.iloc[0][['title', 'description']].to_dict())
    print("\nSample Embedding Shape:", df.iloc[0]['embeddings'].shape)

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    test_pipeline()