Add metadata files (#1)

- Add metadata files (a1d74d9edcc5fe196be04281417a40c81dbfae63)


Co-authored-by: Ivan Arcuschin <iarcuschin@users.noreply.huggingface.co>

benchmark_cases_metadata.csv

@@ -0,0 +1,19 @@
+case_id,task_description,max_seq_len,min_seq_len,training_args.atol,training_args.lr,training_args.use_single_loss,training_args.iit_weight,training_args.behavior_weight,training_args.strict_weight,training_args.epochs,training_args.act_fn,training_args.clip_grad_norm,training_args.lr_scheduler,transformer_cfg.n_layers,transformer_cfg.d_model,transformer_cfg.n_ctx,transformer_cfg.d_head,transformer_cfg.model_name,transformer_cfg.n_heads,transformer_cfg.d_mlp,transformer_cfg.act_fn,transformer_cfg.d_vocab,transformer_cfg.eps,transformer_cfg.use_attn_result,transformer_cfg.use_attn_scale,transformer_cfg.use_split_qkv_input,transformer_cfg.use_hook_mlp_in,transformer_cfg.use_attn_in,transformer_cfg.use_local_attn,transformer_cfg.original_architecture,transformer_cfg.from_checkpoint,transformer_cfg.checkpoint_index,transformer_cfg.checkpoint_label_type,transformer_cfg.checkpoint_value,transformer_cfg.tokenizer_name,transformer_cfg.window_size,transformer_cfg.attn_types,transformer_cfg.init_mode,transformer_cfg.normalization_type,transformer_cfg.device,transformer_cfg.n_devices,transformer_cfg.attention_dir,transformer_cfg.attn_only,transformer_cfg.seed,transformer_cfg.initializer_range,transformer_cfg.init_weights,transformer_cfg.scale_attn_by_inverse_layer_idx,transformer_cfg.positional_embedding_type,transformer_cfg.final_rms,transformer_cfg.d_vocab_out,transformer_cfg.parallel_attn_mlp,transformer_cfg.rotary_dim,transformer_cfg.n_params,transformer_cfg.use_hook_tokens,transformer_cfg.gated_mlp,transformer_cfg.default_prepend_bos,transformer_cfg.dtype,transformer_cfg.tokenizer_prepends_bos,transformer_cfg.n_key_value_heads,transformer_cfg.post_embedding_ln,transformer_cfg.rotary_base,transformer_cfg.trust_remote_code,transformer_cfg.rotary_adjacent_pairs
+11,Counts the number of words in a sequence based on their length.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1460593486680443,True,False,standard,False,5.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+13,"Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.1460593486680443,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+18,"Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.12344267996967354,True,False,standard,False,3.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+19,Removes consecutive duplicate tokens from a sequence.,15.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,32.0,15.0,8.0,custom,4.0,128.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.15689290811054724,True,False,standard,False,3.0,False,,24576.0,False,False,True,torch.float32,,,False,10000.0,False,False
+20,Detect spam messages based on appearance of spam keywords.,10.0,4.0,0.05,0.001,False,1.0,1.0,1.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,14.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.16,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+21,Extract unique tokens from a string,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+24,Identifies the first occurrence of each token in a sequence.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+3,Returns the fraction of 'x' in the input up to the i-th position for all i.,5.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,12.0,5.0,3.0,custom,4.0,48.0,gelu,6.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.22188007849009167,True,False,standard,False,1.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+33,Checks if each token's length is odd or even.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17457431218879393,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+34,Calculate the ratio of vowels to consonants in each word.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.16329931618554522,True,False,standard,False,5.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+35,Alternates capitalization of each character in words.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+36,"Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",10.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.19402850002906638,True,False,standard,False,3.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+37,Reverses each word in the sequence except for specified exclusions.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+38,Checks if tokens alternate between two types.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,2.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+4,Return fraction of previous open tokens minus the fraction of close tokens.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17056057308448835,True,False,standard,False,1.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+8,Fills gaps between tokens with a specified filler.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.13333333333333333,True,False,standard,False,8.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+ioi,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+ioi_next_token,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

benchmark_cases_metadata.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01f473326ab84033784efd049808e3f0395bf577e23311a60f9b05eff32f5ca3
+size 49942

benchmark_metadata.json

@@ -0,0 +1,1362 @@
+{
+  "name": "InterpBench",
+  "version": "1.0.0",
+  "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.",
+  "cases": [
+    {
+      "case_id": "11",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Counts the number of words in a sequence based on their length.",
+      "vocab": [
+        "J",
+        "oCLrZaW",
+        "no",
+        "poiVg",
+        "V",
+        "b",
+        "LB",
+        "TPSI"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 12,
+        "n_ctx": 10,
+        "d_head": 3,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 48,
+        "act_fn": "gelu",
+        "d_vocab": 10,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.1460593486680443,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 5,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 3456,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.01,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 500,
+        "act_fn": "gelu",
+        "clip_grad_norm": 1.0,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "13",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",
+      "vocab": [
+        0,
+        1,
+        2
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 20,
+        "n_ctx": 10,
+        "d_head": 5,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 80,
+        "act_fn": "gelu",
+        "d_vocab": 5,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "bidirectional",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.1460593486680443,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 3,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 9600,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.01,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 500,
+        "act_fn": "gelu",
+        "clip_grad_norm": 1.0,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "18",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",
+      "vocab": [
+        "c",
+        "e",
+        "b",
+        "d",
+        "a"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 12,
+        "n_ctx": 10,
+        "d_head": 3,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 48,
+        "act_fn": "gelu",
+        "d_vocab": 7,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "bidirectional",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.12344267996967354,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 3,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 3456,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "19",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Removes consecutive duplicate tokens from a sequence.",
+      "vocab": [
+        "b",
+        "a",
+        "c"
+      ],
+      "max_seq_len": 15,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 32,
+        "n_ctx": 15,
+        "d_head": 8,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 128,
+        "act_fn": "gelu",
+        "d_vocab": 5,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.15689290811054724,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 3,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 24576,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "20",
+      "files": [
+        "edges.pkl",
+        "ll_model_1110.pth",
+        "ll_model_cfg_1110.pkl",
+        "meta_1110.json"
+      ],
+      "task_description": "Detect spam messages based on appearance of spam keywords.",
+      "vocab": [
+        "J",
+        "spam",
+        "offer",
+        "click",
+        "oCLrZaW",
+        "no",
+        "poiVg",
+        "V",
+        "b",
+        "LB",
+        "now",
+        "TPSI"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 4,
+        "n_ctx": 10,
+        "d_head": 1,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 16,
+        "act_fn": "gelu",
+        "d_vocab": 14,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cuda",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.16,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 2,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 384,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 1.0,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "21",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Extract unique tokens from a string",
+      "vocab": [
+        "b",
+        "a",
+        "c"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 20,
+        "n_ctx": 10,
+        "d_head": 5,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 80,
+        "act_fn": "gelu",
+        "d_vocab": 5,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.1885618083164127,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 3,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 9600,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.01,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 500,
+        "act_fn": "gelu",
+        "clip_grad_norm": 1.0,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "24",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Identifies the first occurrence of each token in a sequence.",
+      "vocab": [
+        "b",
+        "a",
+        "c"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 20,
+        "n_ctx": 10,
+        "d_head": 5,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 80,
+        "act_fn": "gelu",
+        "d_vocab": 5,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.1885618083164127,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 3,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 9600,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.01,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 500,
+        "act_fn": "gelu",
+        "clip_grad_norm": 1.0,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "3",
+      "files": [
+        "edges.pkl",
+        "ll_model_10110.pth",
+        "ll_model_cfg_10110.pkl",
+        "meta_10110.json"
+      ],
+      "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.",
+      "vocab": [
+        "x",
+        "b",
+        "a",
+        "c"
+      ],
+      "max_seq_len": 5,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 12,
+        "n_ctx": 5,
+        "d_head": 3,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 48,
+        "act_fn": "gelu",
+        "d_vocab": 6,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.22188007849009167,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 1,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 3456,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 10.0,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "33",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Checks if each token's length is odd or even.",
+      "vocab": [
+        "J",
+        "oCLrZaW",
+        "no",
+        "poiVg",
+        "V",
+        "b",
+        "LB",
+        "TPSI"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 4,
+        "n_ctx": 10,
+        "d_head": 1,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 16,
+        "act_fn": "gelu",
+        "d_vocab": 10,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.17457431218879393,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 2,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 384,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "34",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Calculate the ratio of vowels to consonants in each word.",
+      "vocab": [
+        "J",
+        "oCLrZaW",
+        "no",
+        "poiVg",
+        "V",
+        "b",
+        "LB",
+        "TPSI"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 4,
+        "n_ctx": 10,
+        "d_head": 1,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 16,
+        "act_fn": "gelu",
+        "d_vocab": 10,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.16329931618554522,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 5,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 384,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "35",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Alternates capitalization of each character in words.",
+      "vocab": [
+        "J",
+        "oCLrZaW",
+        "no",
+        "poiVg",
+        "V",
+        "b",
+        "LB",
+        "TPSI"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 4,
+        "n_ctx": 10,
+        "d_head": 1,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 16,
+        "act_fn": "gelu",
+        "d_vocab": 10,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.1539600717839002,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 8,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 384,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "36",
+      "files": [
+        "edges.pkl",
+        "ll_model_10110.pth",
+        "ll_model_cfg_10110.pkl",
+        "meta_10110.json"
+      ],
+      "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",
+      "vocab": [
+        "\ud83d\ude22",
+        "\ud83d\udcd8",
+        "\ud83d\ude0a"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 4,
+        "n_ctx": 10,
+        "d_head": 1,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 16,
+        "act_fn": "gelu",
+        "d_vocab": 5,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cuda",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.19402850002906638,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 3,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 384,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 10.0,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "37",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Reverses each word in the sequence except for specified exclusions.",
+      "vocab": [
+        "J",
+        "oCLrZaW",
+        "no",
+        "poiVg",
+        "V",
+        "b",
+        "LB",
+        "TPSI"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 4,
+        "n_ctx": 10,
+        "d_head": 1,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 16,
+        "act_fn": "gelu",
+        "d_vocab": 10,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.1539600717839002,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 8,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 384,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "38",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Checks if tokens alternate between two types.",
+      "vocab": [
+        "b",
+        "a",
+        "c"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 20,
+        "n_ctx": 10,
+        "d_head": 5,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 80,
+        "act_fn": "gelu",
+        "d_vocab": 5,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.1539600717839002,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 2,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 9600,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "4",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.",
+      "vocab": [
+        "b",
+        "(",
+        "c",
+        ")",
+        "a"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 20,
+        "n_ctx": 10,
+        "d_head": 5,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 80,
+        "act_fn": "gelu",
+        "d_vocab": 7,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.17056057308448835,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 1,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 9600,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.001,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 2000,
+        "act_fn": "gelu",
+        "clip_grad_norm": 0.1,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "8",
+      "files": [
+        "edges.pkl",
+        "ll_model_510.pth",
+        "ll_model_cfg_510.pkl",
+        "meta_510.json"
+      ],
+      "task_description": "Fills gaps between tokens with a specified filler.",
+      "vocab": [
+        "J",
+        "oCLrZaW",
+        "no",
+        "poiVg",
+        "V",
+        "b",
+        "LB",
+        "TPSI"
+      ],
+      "max_seq_len": 10,
+      "min_seq_len": 4,
+      "transformer_cfg": {
+        "n_layers": 2,
+        "d_model": 20,
+        "n_ctx": 10,
+        "d_head": 5,
+        "model_name": "custom",
+        "n_heads": 4,
+        "d_mlp": 80,
+        "act_fn": "gelu",
+        "d_vocab": 10,
+        "eps": 1e-05,
+        "use_attn_result": true,
+        "use_attn_scale": true,
+        "use_split_qkv_input": true,
+        "use_hook_mlp_in": true,
+        "use_attn_in": false,
+        "use_local_attn": false,
+        "original_architecture": null,
+        "from_checkpoint": false,
+        "checkpoint_index": null,
+        "checkpoint_label_type": null,
+        "checkpoint_value": null,
+        "tokenizer_name": null,
+        "window_size": null,
+        "attn_types": null,
+        "init_mode": "gpt2",
+        "normalization_type": null,
+        "device": "cpu",
+        "n_devices": 1,
+        "attention_dir": "causal",
+        "attn_only": false,
+        "seed": 0,
+        "initializer_range": 0.13333333333333333,
+        "init_weights": true,
+        "scale_attn_by_inverse_layer_idx": false,
+        "positional_embedding_type": "standard",
+        "final_rms": false,
+        "d_vocab_out": 8,
+        "parallel_attn_mlp": false,
+        "rotary_dim": null,
+        "n_params": 9600,
+        "use_hook_tokens": false,
+        "gated_mlp": false,
+        "default_prepend_bos": true,
+        "dtype": "torch.float32",
+        "tokenizer_prepends_bos": null,
+        "n_key_value_heads": null,
+        "post_embedding_ln": false,
+        "rotary_base": 10000,
+        "trust_remote_code": false,
+        "rotary_adjacent_pairs": false
+      },
+      "training_args": {
+        "atol": 0.05,
+        "lr": 0.01,
+        "use_single_loss": false,
+        "iit_weight": 1.0,
+        "behavior_weight": 1.0,
+        "strict_weight": 0.4,
+        "epochs": 500,
+        "act_fn": "gelu",
+        "clip_grad_norm": 1.0,
+        "lr_scheduler": ""
+      }
+    },
+    {
+      "case_id": "ioi",
+      "files": [
+        "corr_100_100_40.json",
+        "ll_model_100_100_40.pth"
+      ]
+    },
+    {
+      "case_id": "ioi_next_token",
+      "files": [
+        "corr_100_100_40.json",
+        "ll_model_100_100_40.pth",
+        "training_args.json"
+      ]
+    }
+  ]
+}