Upload folder using huggingface_hub

mlp_out/Mo0_S-1_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:984f9c2f528344a97f876ecdaf9a730806b2aa665693139fbc7cc0b7d51637a4
+size 153705088

mlp_out/Mo0_S-1_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 0,
+  "l1_exp": -1,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo0_S-1_R1_P0"
+}

mlp_out/Mo0_S-2_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:429aef35f42a694db09f2901dbf3c732d7b24b52afdbae3effd903782a44ca6d
+size 153705088

mlp_out/Mo0_S-2_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 0,
+  "l1_exp": -2,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo0_S-2_R1_P0"
+}

mlp_out/Mo0_S-3_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:297a24116992b6c6f735b9064e18265eb914bb465ae81b3ad031543f9230759a
+size 153705088

mlp_out/Mo0_S-3_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 0,
+  "l1_exp": -3,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo0_S-3_R1_P0"
+}

mlp_out/Mo0_S-4_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8200bc51b8954b40aeda4f528d92be0ed3a348016102a597914ca1fac648a56f
+size 153705088

mlp_out/Mo0_S-4_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 0,
+  "l1_exp": -4,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo0_S-4_R1_P0"
+}

mlp_out/Mo0_S-5_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efaa3f22bb757acfa4545e348bc6326eeeaccbe146593c6a94e362e588358bd1
+size 153705088

mlp_out/Mo0_S-5_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 0,
+  "l1_exp": -5,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo0_S-5_R1_P0"
+}

mlp_out/Mo0_S-6_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2e1dd7d6865fd36d257ecf0b0d8872b479696222438d5b1992ba98bc7934add
+size 153705088

mlp_out/Mo0_S-6_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 0,
+  "l1_exp": -6,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo0_S-6_R1_P0"
+}

mlp_out/Mo1_S-1_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9718878757233965224570259ddaf12358f47825884d881b53d0e6b80e65618c
+size 153705088

mlp_out/Mo1_S-1_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 1,
+  "l1_exp": -1,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo1_S-1_R1_P0"
+}

mlp_out/Mo1_S-2_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2412c5d38e478b9f9bb310b2860826a501681695fe43e3205b9a5958f7007d40
+size 153705088

mlp_out/Mo1_S-2_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 1,
+  "l1_exp": -2,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo1_S-2_R1_P0"
+}

mlp_out/Mo1_S-3_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05a4b444d92c1ec5568d5322a3073029e4440be0c4e35ace336124b1da8a9257
+size 153705088

mlp_out/Mo1_S-3_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 1,
+  "l1_exp": -3,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo1_S-3_R1_P0"
+}

mlp_out/Mo1_S-4_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89d5e4b6273a3652848cd7d587e1c0c0a14ceb2bcb657950d891ba8e7b799c60
+size 153705088

mlp_out/Mo1_S-4_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 1,
+  "l1_exp": -4,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo1_S-4_R1_P0"
+}

mlp_out/Mo1_S-5_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:114b6f11baefbd6c360ad5fe3d873a6f0985721d56162565f0c32b1e8cc864e3
+size 153705088

mlp_out/Mo1_S-5_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 1,
+  "l1_exp": -5,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo1_S-5_R1_P0"
+}

mlp_out/Mo1_S-6_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cde27f39f1355ef34d67a3ca9b5a3debe48a4d8b30250f7da5920554959c8a2
+size 153705088

mlp_out/Mo1_S-6_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 1,
+  "l1_exp": -6,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo1_S-6_R1_P0"
+}

mlp_out/Mo2_S-1_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e95ab67d644595fbcbf83f7d3f753de49929bc43024535cfa42a614f1af1d26
+size 153705088

mlp_out/Mo2_S-1_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 2,
+  "l1_exp": -1,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo2_S-1_R1_P0"
+}

mlp_out/Mo2_S-2_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a70c5b17e7edf63c23a21f562672fb8cc6ecd27debe8b86844fe2251e9bb31c5
+size 153705088

mlp_out/Mo2_S-2_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 2,
+  "l1_exp": -2,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo2_S-2_R1_P0"
+}

mlp_out/Mo2_S-3_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f809c5f87096de73512d647f615ba4eb9bae1d71d52a226ba566216968d428fd
+size 153705088

mlp_out/Mo2_S-3_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 2,
+  "l1_exp": -3,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo2_S-3_R1_P0"
+}

mlp_out/Mo2_S-4_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9561e5148574aca7b92c977fcc1240620cb9c6d548d7a699c2bf6dff0f882b3d
+size 153705088

mlp_out/Mo2_S-4_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 2,
+  "l1_exp": -4,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo2_S-4_R1_P0"
+}

mlp_out/Mo2_S-5_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:042a5a4090bb7803f4cc663f2a9bea9691b9d32df6f2643516971142a879d638
+size 153705088

mlp_out/Mo2_S-5_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 2,
+  "l1_exp": -5,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo2_S-5_R1_P0"
+}

mlp_out/Mo2_S-6_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38a5a864e743df7638f33891da4e3ee21bbd488feebaed1b45a38f8252b7f038
+size 153705088

mlp_out/Mo2_S-6_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 2,
+  "l1_exp": -6,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo2_S-6_R1_P0"
+}

mlp_out/Mo3_S-1_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb3e057ec42506222a618429b605c657e47067f05cf9f94dff06239ab3ecc8cf
+size 153705088

mlp_out/Mo3_S-1_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 3,
+  "l1_exp": -1,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo3_S-1_R1_P0"
+}

mlp_out/Mo3_S-2_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec5231d051f63b3f92cbc90a88dbb400ab0395121f0300f254f7c6a95a5692c9
+size 153705088

mlp_out/Mo3_S-2_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 3,
+  "l1_exp": -2,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo3_S-2_R1_P0"
+}

mlp_out/Mo3_S-3_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81b11e9e8cd9562a57488bb0def2b8f6ae56b42b08d21d80f1b8b3593d8e7867
+size 153705088

mlp_out/Mo3_S-3_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 3,
+  "l1_exp": -3,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo3_S-3_R1_P0"
+}

mlp_out/Mo3_S-4_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f38a16a2a5f53f6d0372559b8317110b554b5cf406c34e42ac3ec7e8526d2ca1
+size 153705088

mlp_out/Mo3_S-4_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 3,
+  "l1_exp": -4,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo3_S-4_R1_P0"
+}

mlp_out/Mo3_S-5_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f62bb1ee986cecc58864bfd3d21ddd15f96f04eb81a2f51391b0a21c19dd3772
+size 153705088

mlp_out/Mo3_S-5_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 3,
+  "l1_exp": -5,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo3_S-5_R1_P0"
+}

mlp_out/Mo3_S-6_R1_P0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e43971c8fed0446b128b196b204c1c4857e16745a1096a9f8f3e97074fbfc73
+size 153705088

mlp_out/Mo3_S-6_R1_P0_config.json

@@ -0,0 +1,38 @@
+{
+  "n_features": 25000,
+  "d_model": 768,
+  "lr_exp": -10,
+  "disable_comet": false,
+  "per_neuron_reinit_interval": 0,
+  "reservoir_time_discount": 0.995,
+  "reinit_interval": 800,
+  "max_reinit_neurons": 5000,
+  "reservoir_size": 5000,
+  "n_piles": 292,
+  "log_interval": 200,
+  "reinit_input_norm": "target_scaled",
+  "reinit_input": "x",
+  "reinit_norm_alpha": 0.3,
+  "data_loc": "mlp_data",
+  "reinit_threshold": -6,
+  "scheduler": "wsd",
+  "layer_idx": 3,
+  "l1_exp": -6,
+  "neuron_reinit_percent": 0.85,
+  "beta1": 1,
+  "beta2": 4,
+  "reinit_target": "error",
+  "sparse_adam": false,
+  "run_template": "Mo{layer_idx}_S{l1_exp}_R{l1_ratio}_P{l1_p}",
+  "project_name": "mlp_out_test",
+  "decoder_bias": true,
+  "l1_beta": 0.99,
+  "alt_sparsity_loss": "log",
+  "l1_ratio": 1,
+  "l1_p": 0,
+  "optimizer": "sparse_adam",
+  "model_type": "mlp_out",
+  "adam_beta1": 0.5,
+  "adam_beta2": 0.9375,
+  "run_name": "Mo3_S-6_R1_P0"
+}