# pv ~/data/bignews/processed/bignews_val_text.txt > /dev/shm/news_val_text.txt
# pv ~/data/bignews/processed/bignews_train_text.txt > /dev/shm/news_train_text.txt
python3 process_bignews.py ~/data/bignews/bignews_train.json /dev/shm/news_train
python3 process_bignews.py ~/data/bignews/bignews_val.json /dev/shm/news_val
python3 create_dataset.py /dev/shm/news /dev/shm/bignews_flattened

# python3 process_bignews.py ~/data/bignews/bignews_$SPLIT.json ~/data/bignews/processed_lite/bignews_$SPLIT --ratio 0.05
# pv ~/data/bignews/processed_lite/bignews_val_text.txt > /dev/shm/news_lite_val_text.txt
# pv ~/data/bignews/processed_lite/bignews_train_text.txt > /dev/shm/news_lite_train_text.txt
# python3 create_dataset.py /dev/shm/news_lite /dev/shm/bignews_lite_flattened