Hzfinfdu commited on
Commit
13155f5
1 Parent(s): 5811169

Upload folder using huggingface_hub

Browse files
Llama3_1Base-L4M-8x/checkpoints/final.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c67082fed1f988c81f119469039b8bf395355bb3dc64f6e50b2ffd797af46a82
3
- size 537076272
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ef17f65cd4a26b3f1ae253a2b631a89402e9547ae510608b30987f5cf8162a5
3
+ size 536945024
Llama3_1Base-L4M-8x/hyperparams.json CHANGED
@@ -1,4 +1,7 @@
1
  {
 
 
 
2
  "hook_point_in": "blocks.4.hook_mlp_out",
3
  "hook_point_out": "blocks.4.hook_mlp_out",
4
  "use_decoder_bias": true,
@@ -6,17 +9,26 @@
6
  "expansion_factor": 8,
7
  "d_model": 4096,
8
  "d_sae": 32768,
9
- "norm_activation": "token-wise",
10
- "dataset_average_activation_norm": null,
 
 
 
 
 
 
11
  "decoder_exactly_fixed_norm": false,
12
  "sparsity_include_decoder_norm": true,
13
  "use_glu_encoder": false,
14
- "init_decoder_norm": null,
15
  "init_encoder_norm": null,
16
  "init_encoder_with_decoder_transpose": true,
17
- "l1_coefficient": 1.6e-05,
18
- "l1_coefficient_warmup_steps": 14648,
19
  "lp": 1,
 
 
 
 
 
20
  "use_ghost_grads": false,
21
  "tp_size": 1,
22
  "ddp_size": 1
 
1
  {
2
+ "device": "cuda:0",
3
+ "seed": 42,
4
+ "dtype": "torch.bfloat16",
5
  "hook_point_in": "blocks.4.hook_mlp_out",
6
  "hook_point_out": "blocks.4.hook_mlp_out",
7
  "use_decoder_bias": true,
 
9
  "expansion_factor": 8,
10
  "d_model": 4096,
11
  "d_sae": 32768,
12
+ "bias_init_method": "all_zero",
13
+ "act_fn": "jumprelu",
14
+ "jump_relu_threshold": 0.07470703125,
15
+ "norm_activation": "dataset-wise",
16
+ "dataset_average_activation_norm": {
17
+ "in": 2.296875,
18
+ "out": 2.296875
19
+ },
20
  "decoder_exactly_fixed_norm": false,
21
  "sparsity_include_decoder_norm": true,
22
  "use_glu_encoder": false,
23
+ "init_decoder_norm": 0.5,
24
  "init_encoder_norm": null,
25
  "init_encoder_with_decoder_transpose": true,
 
 
26
  "lp": 1,
27
+ "l1_coefficient": 8e-05,
28
+ "l1_coefficient_warmup_steps": 39062,
29
+ "top_k": 50,
30
+ "k_warmup_steps": 39062,
31
+ "use_batch_norm_mse": true,
32
  "use_ghost_grads": false,
33
  "tp_size": 1,
34
  "ddp_size": 1
Llama3_1Base-L4M-8x/lm_config.json CHANGED
@@ -1,4 +1,7 @@
1
  {
 
 
 
2
  "model_name": "meta-llama/Meta-Llama-3.1-8B",
3
  "use_flash_attn": false,
4
  "cache_dir": null,
 
1
  {
2
+ "device": "cuda",
3
+ "seed": 42,
4
+ "dtype": "torch.bfloat16",
5
  "model_name": "meta-llama/Meta-Llama-3.1-8B",
6
  "use_flash_attn": false,
7
  "cache_dir": null,