ncoop57 commited on
Commit
3e6eddc
1 Parent(s): 3e4a220

Fix data_files arg and add token auth usage

Browse files
Files changed (1) hide show
  1. app.py +22 -22
app.py CHANGED
@@ -5,28 +5,28 @@ from functools import partial
5
  import datasets
6
  from datasets import load_dataset
7
 
8
- ai4code_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/AI4Code/data.json")
9
- amps_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/AMPS/data.json")
10
- apache_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/ASFPublicMail/data.json")
11
- books3_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Books3/data.json")
12
- cp_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/CPDataset/data.json")
13
- dmmath_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/DMMath/data.json")
14
- discourse_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Discourse/data.json")
15
- wiki_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Enwiki/data.json")
16
- euro_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/EuroParliamentProceedings/data.json")
17
- freelaw_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/FreeLaw_Options/data.json")
18
- ghdiffs_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/GitHubDiff/data.json")
19
- ghissues_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/GitHubIssues/data.json")
20
- gutenberg_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Gutenberg/data.json")
21
- leet_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/LeetCode/data.json")
22
- pileoflaw_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/PileOfLaw/data.json")
23
- pubmed_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/PubMed/data.json")
24
- s2orc_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/S2ORC/data.json")
25
- se_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/StackExchange/data.json")
26
- usenet_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/USENET/data.json")
27
- uspto_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/USPTO/data.json")
28
- ubuntuirc_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/UbuntuIRC/data.json")
29
- arxiv_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/arXiv/data.json")
30
 
31
 
32
  dataset_data = {
 
5
  import datasets
6
  from datasets import load_dataset
7
 
8
+ ai4code_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/AI4Code/data.json", use_auth_token=True)
9
+ amps_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/AMPS/data.json", use_auth_token=True)
10
+ apache_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/ASFPublicMail/data.json", use_auth_token=True)
11
+ books3_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/Books3/data.json", use_auth_token=True)
12
+ cp_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/CPDataset/data.json", use_auth_token=True)
13
+ dmmath_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/DMMath/data.json", use_auth_token=True)
14
+ discourse_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/Discourse/data.json", use_auth_token=True)
15
+ wiki_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/Enwiki/data.json")
16
+ euro_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/EuroParliamentProceedings/data.json", use_auth_token=True)
17
+ freelaw_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/FreeLaw_Options/data.json", use_auth_token=True)
18
+ ghdiffs_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/GitHubDiff/data.json", use_auth_token=True)
19
+ ghissues_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/GitHubIssues/data.json", use_auth_token=True)
20
+ gutenberg_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/Gutenberg/data.json", use_auth_token=True)
21
+ leet_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/LeetCode/data.json", use_auth_token=True)
22
+ pileoflaw_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/PileOfLaw/data.json", use_auth_token=True)
23
+ pubmed_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/PubMed/data.json", use_auth_token=True)
24
+ s2orc_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/S2ORC/data.json", use_auth_token=True)
25
+ se_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/StackExchange/data.json", use_auth_token=True)
26
+ usenet_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/USENET/data.json", use_auth_token=True)
27
+ uspto_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/USPTO/data.json", use_auth_token=True)
28
+ ubuntuirc_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/UbuntuIRC/data.json", use_auth_token=True)
29
+ arxiv_ds = load_dataset("CarperAI/pile-v2-small", data_files="data/arXiv/data.json", use_auth_token=True)
30
 
31
 
32
  dataset_data = {