Adding remove filtered rows functionality

filter_ids.json

@@ -0,0 +1,3 @@
+[
+  "qirdpd"
+]

main.py

@@ -6,7 +6,7 @@ import pandas as pd
 import schedule
 from datasets import Dataset
 
-from utilities.user_defined_functions import get_latest_data, merge_data, load_or_create_dataset
+from utilities.user_defined_functions import get_latest_data, merge_data, load_or_create_dataset, remove_filtered_rows
 from utilities.my_logger import setup_logger
 from utilities.readme_update import update_dataset_readme
 
@@ -44,6 +44,7 @@ def main():
         df['new'] = True
         df['updated'] = False
         new_rows = len(new_df)
+    df = remove_filtered_rows(df)
     dataset['train'] = Dataset.from_pandas(df, preserve_index=False)
 
     # Update README

utilities/user_defined_functions.py

@@ -1,5 +1,6 @@
 import os
 from datetime import datetime
+import json
 
 import pandas as pd
 from datasets import Dataset, DatasetDict, load_dataset, DownloadMode
@@ -104,6 +105,24 @@ def merge_data(old_df: pd.DataFrame, new_df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
+def remove_filtered_rows(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Removes rows from the DataFrame where the 'id' is present in filter_ids.json.
+
+    :param df: Input DataFrame to be filtered.
+    :return: DataFrame with rows containing IDs present in filter_ids.json removed.
+    """
+    # Load filter IDs from JSON file
+    with open('filter_ids.json', 'r') as file:
+        filter_ids = json.load(file)
+
+    # Remove the rows with IDs present in filter_ids
+    filtered_df = df[~df['id'].isin(filter_ids)]
+    logger.info(f"Filtered {len(df) - len(filtered_df)} rows from the DataFrame")
+
+    return filtered_df
+
+
 def get_latest_data():
     submissions = praw_downloader()
     df = preprocess_praw_data(submissions=submissions)