Hzfinfdu commited on
Commit
d3b09c7
1 Parent(s): 6e76064

Upload folder using huggingface_hub

Browse files
Llama3_1Base-L25M-8x/checkpoints/final.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a53644dc8b2f2e6f8feea47c59f37d88c787e3be39bf088927103bf290ab094b
3
- size 537076272
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96cf55c3ae2cb3a23e304c8c7890a4a0c2a2b07f9ef913f1ba772b5084afd00f
3
+ size 536945024
Llama3_1Base-L25M-8x/hyperparams.json CHANGED
@@ -1,4 +1,7 @@
1
  {
 
 
 
2
  "hook_point_in": "blocks.25.hook_mlp_out",
3
  "hook_point_out": "blocks.25.hook_mlp_out",
4
  "use_decoder_bias": true,
@@ -6,17 +9,26 @@
6
  "expansion_factor": 8,
7
  "d_model": 4096,
8
  "d_sae": 32768,
9
- "norm_activation": "token-wise",
10
- "dataset_average_activation_norm": null,
 
 
 
 
 
 
11
  "decoder_exactly_fixed_norm": false,
12
  "sparsity_include_decoder_norm": true,
13
  "use_glu_encoder": false,
14
- "init_decoder_norm": null,
15
  "init_encoder_norm": null,
16
  "init_encoder_with_decoder_transpose": true,
17
- "l1_coefficient": 1.6e-05,
18
- "l1_coefficient_warmup_steps": 14648,
19
  "lp": 1,
 
 
 
 
 
20
  "use_ghost_grads": false,
21
  "tp_size": 1,
22
  "ddp_size": 1
 
1
  {
2
+ "device": "cuda:0",
3
+ "seed": 42,
4
+ "dtype": "torch.bfloat16",
5
  "hook_point_in": "blocks.25.hook_mlp_out",
6
  "hook_point_out": "blocks.25.hook_mlp_out",
7
  "use_decoder_bias": true,
 
9
  "expansion_factor": 8,
10
  "d_model": 4096,
11
  "d_sae": 32768,
12
+ "bias_init_method": "all_zero",
13
+ "act_fn": "jumprelu",
14
+ "jump_relu_threshold": 0.27734375,
15
+ "norm_activation": "dataset-wise",
16
+ "dataset_average_activation_norm": {
17
+ "in": 7.875,
18
+ "out": 7.875
19
+ },
20
  "decoder_exactly_fixed_norm": false,
21
  "sparsity_include_decoder_norm": true,
22
  "use_glu_encoder": false,
23
+ "init_decoder_norm": 0.5,
24
  "init_encoder_norm": null,
25
  "init_encoder_with_decoder_transpose": true,
 
 
26
  "lp": 1,
27
+ "l1_coefficient": 8e-05,
28
+ "l1_coefficient_warmup_steps": 39062,
29
+ "top_k": 50,
30
+ "k_warmup_steps": 39062,
31
+ "use_batch_norm_mse": true,
32
  "use_ghost_grads": false,
33
  "tp_size": 1,
34
  "ddp_size": 1
Llama3_1Base-L25M-8x/lm_config.json CHANGED
@@ -1,4 +1,7 @@
1
  {
 
 
 
2
  "model_name": "meta-llama/Meta-Llama-3.1-8B",
3
  "use_flash_attn": false,
4
  "cache_dir": null,
 
1
  {
2
+ "device": "cuda",
3
+ "seed": 42,
4
+ "dtype": "torch.bfloat16",
5
  "model_name": "meta-llama/Meta-Llama-3.1-8B",
6
  "use_flash_attn": false,
7
  "cache_dir": null,