use subsets

main.py

@@ -22,6 +22,7 @@ subreddit = os.environ["SUBREDDIT"]
 username = os.environ["USERNAME"]
 dataset_name = f"{username}/reddit-{subreddit}"
 comment_dataset_name = f"{username}/reddit-comments-{subreddit}"
+subset = f"year_{datetime.now().year}"
 
 dataset_readme_path = "README.md"
 
@@ -38,16 +39,10 @@ def upload(new_df, dataset, hf_dataset_name):
     date = datetime.now().strftime('%Y-%m-%d')
     
     # Using dataset from hub
-    if 'train' in dataset.keys():
-        old_df = dataset['train'].to_pandas() if 'train' in dataset.keys() else pd.DataFrame()
-        df = merge_data(old_df=old_df, new_df=new_df)
-        new_rows = len(df) - len(old_df)
-    # New dataset
-    else:
-        df = new_df
-        df['new'] = True
-        df['updated'] = False
-        new_rows = len(new_df)
+    old_df = dataset['train'].to_pandas() if 'train' in dataset.keys() else pd.DataFrame()
+    df = merge_data(old_df=old_df, new_df=new_df)
+    new_rows = len(df) - len(old_df)
+
     df = remove_filtered_rows(df)
     dataset['train'] = Dataset.from_pandas(df, preserve_index=False)
 
@@ -56,7 +51,7 @@ def upload(new_df, dataset, hf_dataset_name):
 
     # Push the augmented dataset to the Hugging Face hub
     logger.debug(f"Pushing data for {date} to {hf_dataset_name}")
-    dataset.push_to_hub(hf_dataset_name, token=auth_token)
+    dataset.push_to_hub(hf_dataset_name, subset, token=auth_token)
     logger.info(f"Processed and pushed data for {date} to {hf_dataset_name}")
     update_dataset_readme(dataset_name=hf_dataset_name, subreddit=subreddit, new_rows=new_rows)
     logger.info(f"Updated README.")

utilities/praw_downloader.py

@@ -49,7 +49,7 @@ def extract_comment_data(comment: praw.models.Comment) -> Dict[str, Any]:
         'permalink': comment.permalink,
         'depth': comment.depth,
         'link_id': comment.link_id,
-        'submission_id': comment._submission.id,
+        'parent_id': comment.parent_id,
         'id': comment.id
     }
 

utilities/user_defined_functions.py

@@ -16,6 +16,7 @@ subreddit = os.environ["SUBREDDIT"]
 username = os.environ["USERNAME"]
 dataset_name = f"{username}/reddit-{subreddit}"
 comment_dataset_name = f"{username}/reddit-comments-{subreddit}"
+subset = f"year_{datetime.now().year}"
 
 frequency = os.environ.get("FREQUENCY", '').lower()
 if frequency not in ["daily", "hourly"]:
@@ -56,7 +57,7 @@ dummy_comment_data = {
     "new": [False],
     "depth": [2],
     "link_id": ["eqrkhgbjeh"],
-    "submission_id": ["eqrkhgbjeh"]
+    "parent_id": ["eqrkhgbjeh"]
 }
 
 
@@ -77,7 +78,7 @@ def load_or_create_dataset():
     # Load the existing dataset from the Hugging Face hub or create a new one
     try:
         logger.debug(f"Trying to download {dataset_name}")
-        dataset = load_dataset(dataset_name, download_mode=DownloadMode.FORCE_REDOWNLOAD)
+        dataset = load_dataset(dataset_name, subset, download_mode=DownloadMode.FORCE_REDOWNLOAD)
         logger.debug("Loading existing dataset")
     except FileNotFoundError:
         logger.warning("Creating new dataset")
@@ -85,10 +86,10 @@ def load_or_create_dataset():
         # Creating Initial Repo
         dataset = DatasetDict()
         dataset['train'] = Dataset.from_dict(dummy_data)
-        dataset.push_to_hub(repo_id=dataset_name, token=auth_token)
+        dataset.push_to_hub(dataset_name, subset, token=auth_token)
 
         # Pulling from Initial Repo
-        dataset = load_dataset(dataset_name)
+        dataset = load_dataset(dataset_name, subset)
 
         # Remove dummy data
         del dataset['train']
@@ -99,7 +100,7 @@ def load_or_create_comment_dataset():
     # Load the existing dataset from the Hugging Face hub or create a new one
     try:
         logger.debug(f"Trying to download {comment_dataset_name}")
-        dataset = load_dataset(comment_dataset_name, download_mode=DownloadMode.FORCE_REDOWNLOAD)
+        dataset = load_dataset(comment_dataset_name, subset, download_mode=DownloadMode.FORCE_REDOWNLOAD)
         logger.debug("Loading existing comment dataset")
     except FileNotFoundError:
         logger.warning("Creating new comment dataset")
@@ -107,10 +108,10 @@ def load_or_create_comment_dataset():
         # Creating Initial Repo
         dataset = DatasetDict()
         dataset['train'] = Dataset.from_dict(dummy_comment_data)
-        dataset.push_to_hub(repo_id=comment_dataset_name, token=auth_token)
+        dataset.push_to_hub(comment_dataset_name, subset, token=auth_token)
 
         # Pulling from Initial Repo
-        dataset = load_dataset(comment_dataset_name)
+        dataset = load_dataset(comment_dataset_name, subset)
 
         # Remove dummy data
         del dataset['train']