config_dict.json

{"dataset_repo": "togethercomputer/RedPajama-Data-1T-Sample", "dataset_split": "train[12%:20%]", "dataset_name": "", "dataset_row": "text", "batch_size": 8, "ctx_len": 256, "n_tokens": 10000000, "n_splits": 5, "noise_std": 1.0, "embedding_type": "pure_gaussian_noise", "num_layers": 12}