Update app.py

app.py

@@ -139,38 +139,19 @@ topic_extractor = TopicExtractor()
 chatbot = Chatbot()
 
 # Load the yt-commons dataset
-def load_youtube_data():
-    try:
-        # Load the dataset from Hugging Face Hub
-        dataset = load_dataset("PleIAs/YouTube-Commons", data_files="*.parquet")  # Load all .parquet files
-        return dataset
-    except Exception as e:
-        print(f"Error loading yt-commons dataset: {e}")
-        return None
-
-
-# Load the dataset
-youtube_data = load_youtube_data()
-
-if youtube_data:
-    print("Dataset loaded successfully!")
-    print(f"Number of splits: {len(youtube_data)}")
-    print(f"Available splits: {list(youtube_data.keys())}")
-
-    # Inspect the first few rows of the dataset
-    print("\nSample data from the 'train' split:")
-    print(youtube_data["train"][0])  # Adjust based on the dataset structure
-else:
-    print("Failed to load dataset.")
-
-
-# Preprocess and build search index
-youtube_data = load_youtube_data()
-if youtube_data:
-    print("Dataset loaded successfully!")
-    print(f"Number of samples: {len(youtube_data)}")
-else:
-    print("Failed to load dataset.")
+from datasets import load_dataset
+
+# Load specific .parquet files
+dataset = load_dataset("PleIAs/YouTube-Commons", data_files=["cctube_0.parquet", "cctube_1.parquet"], streaming=True)
+
+# Extract specific columns
+for example in dataset["train"]:
+    title = example["title"]  # Replace 'title' with the correct column name
+    description = example["description"]  # Replace 'description' with the correct column name
+    print(f"Title: {title}")
+    print(f"Description: {description}")
+    break  # Stop after the first example for demonstration
+
 
 # API Endpoints
 @app.route("/classify", methods=["POST"])