Removing nsfw content via the nsfw column as well!

src/build_nomic.py

@@ -87,9 +87,9 @@ def preprocess_markdown(text):
 def build_nomic(dataset):
     df = dataset['train'].to_pandas()
 
-    # Filter df for nsfw content for displaying in Nomic
+    # For nomig: filter out rows that contain 'nsfw' in specified text columns or where 'nsfw' column is True
     df = df[~df[['content', 'title', 'flair', 'permalink']].apply(
-        lambda x: x.str.contains('nsfw', case=False, na=False)).any(axis=1)]
+            lambda x: x.str.contains('nsfw', case=False, na=False)).any(axis=1) & ~df['nsfw']]
 
     non_embedding_columns = ['date_utc', 'title', 'flair', 'poster', 'url', 'id', 'word_count',
                              'score', 'score_percentile', 'html_content', 'subreddit']