| { | |
| "n_features": 100, | |
| "d_model": 768, | |
| "lr_exp": -10, | |
| "disable_comet": false, | |
| "per_neuron_reinit_interval": 0, | |
| "reservoir_time_discount": 0.995, | |
| "reinit_interval": 800, | |
| "max_reinit_neurons": 5000, | |
| "reservoir_size": 5000, | |
| "n_piles": 292, | |
| "log_interval": 200, | |
| "reinit_input_norm": "target_scaled", | |
| "reinit_input": "x", | |
| "reinit_norm_alpha": 0.3, | |
| "data_loc": "mlp_data", | |
| "reinit_threshold": -6, | |
| "scheduler": "wsd", | |
| "layer_idx": 0, | |
| "l1_exp": -9, | |
| "neuron_reinit_percent": 0.85, | |
| "beta1": 1, | |
| "beta2": 4, | |
| "reinit_target": "error", | |
| "sparse_adam": false, | |
| "run_template": "Mo{layer_idx}_N{n_features}_S{l1_exp}", | |
| "project_name": "small_mlp_out", | |
| "decoder_bias": true, | |
| "l1_beta": 0.99, | |
| "alt_sparsity_loss": "log", | |
| "l1_ratio": 1, | |
| "l1_p": 0, | |
| "optimizer": "sparse_adam", | |
| "model_type": "mlp_out", | |
| "adam_beta1": 0.5, | |
| "adam_beta2": 0.9375, | |
| "run_name": "Mo0_N100_S-9" | |
| } |