{ "name": "InterpBench", "version": "1.0.0", "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.", "cases": [ { "case_id": "11", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Counts the number of words in a sequence based on their length.", "vocab": [ "J", "oCLrZaW", "no", "poiVg", "V", "b", "LB", "TPSI" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1460593486680443, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 5, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "13", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.", "vocab": [ 0, 1, 2 ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "bidirectional", "attn_only": false, "seed": 0, "initializer_range": 0.1460593486680443, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "18", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.", "vocab": [ "c", "e", "b", "d", "a" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 7, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "bidirectional", "attn_only": false, "seed": 0, "initializer_range": 0.12344267996967354, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "19", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Removes consecutive duplicate tokens from a sequence.", "vocab": [ "b", "a", "c" ], "max_seq_len": 15, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 32, "n_ctx": 15, "d_head": 8, "model_name": "custom", "n_heads": 4, "d_mlp": 128, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.15689290811054724, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 24576, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "20", "files": [ "edges.pkl", "ll_model_1110.pth", "ll_model_cfg_1110.pkl", "meta_1110.json" ], "task_description": "Detect spam messages based on appearance of spam keywords.", "vocab": [ "J", "spam", "offer", "click", "oCLrZaW", "no", "poiVg", "V", "b", "LB", "now", "TPSI" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 14, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cuda", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.16, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 1.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "21", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Extract unique tokens from a string", "vocab": [ "b", "a", "c" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1885618083164127, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "24", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Identifies the first occurrence of each token in a sequence.", "vocab": [ "b", "a", "c" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1885618083164127, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "3", "files": [ "edges.pkl", "ll_model_10110.pth", "ll_model_cfg_10110.pkl", "meta_10110.json" ], "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.", "vocab": [ "x", "b", "a", "c" ], "max_seq_len": 5, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 5, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 6, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.22188007849009167, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 1, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 10.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "33", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Checks if each token's length is odd or even.", "vocab": [ "J", "oCLrZaW", "no", "poiVg", "V", "b", "LB", "TPSI" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.17457431218879393, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "34", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Calculate the ratio of vowels to consonants in each word.", "vocab": [ "J", "oCLrZaW", "no", "poiVg", "V", "b", "LB", "TPSI" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.16329931618554522, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 5, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "35", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Alternates capitalization of each character in words.", "vocab": [ "J", "oCLrZaW", "no", "poiVg", "V", "b", "LB", "TPSI" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "36", "files": [ "edges.pkl", "ll_model_10110.pth", "ll_model_cfg_10110.pkl", "meta_10110.json" ], "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.", "vocab": [ "\ud83d\ude22", "\ud83d\udcd8", "\ud83d\ude0a" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cuda", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.19402850002906638, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 10.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "37", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Reverses each word in the sequence except for specified exclusions.", "vocab": [ "J", "oCLrZaW", "no", "poiVg", "V", "b", "LB", "TPSI" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "38", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Checks if tokens alternate between two types.", "vocab": [ "b", "a", "c" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "4", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.", "vocab": [ "b", "(", "c", ")", "a" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 7, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.17056057308448835, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 1, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "8", "files": [ "edges.pkl", "ll_model_510.pth", "ll_model_cfg_510.pkl", "meta_510.json" ], "task_description": "Fills gaps between tokens with a specified filler.", "vocab": [ "J", "oCLrZaW", "no", "poiVg", "V", "b", "LB", "TPSI" ], "max_seq_len": 10, "min_seq_len": 4, "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.13333333333333333, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "ioi", "files": [ "corr_100_100_40.json", "ll_model_100_100_40.pth" ] }, { "case_id": "ioi_next_token", "files": [ "corr_100_100_40.json", "ll_model_100_100_40.pth", "training_args.json" ] } ] }