Update metadata files (#3)

- Update metadata files (39e730430334236aabcb01801e7af2c568d1e397)


Co-authored-by: Ivan Arcuschin <iarcuschin@users.noreply.huggingface.co>

benchmark_cases_metadata.csv

@@ -1,19 +1,19 @@
-case_id,task_description,max_seq_len,min_seq_len,training_args.atol,training_args.lr,training_args.use_single_loss,training_args.iit_weight,training_args.behavior_weight,training_args.strict_weight,training_args.epochs,training_args.act_fn,training_args.clip_grad_norm,training_args.lr_scheduler,transformer_cfg.n_layers,transformer_cfg.d_model,transformer_cfg.n_ctx,transformer_cfg.d_head,transformer_cfg.model_name,transformer_cfg.n_heads,transformer_cfg.d_mlp,transformer_cfg.act_fn,transformer_cfg.d_vocab,transformer_cfg.eps,transformer_cfg.use_attn_result,transformer_cfg.use_attn_scale,transformer_cfg.use_split_qkv_input,transformer_cfg.use_hook_mlp_in,transformer_cfg.use_attn_in,transformer_cfg.use_local_attn,transformer_cfg.original_architecture,transformer_cfg.from_checkpoint,transformer_cfg.checkpoint_index,transformer_cfg.checkpoint_label_type,transformer_cfg.checkpoint_value,transformer_cfg.tokenizer_name,transformer_cfg.window_size,transformer_cfg.attn_types,transformer_cfg.init_mode,transformer_cfg.normalization_type,transformer_cfg.device,transformer_cfg.n_devices,transformer_cfg.attention_dir,transformer_cfg.attn_only,transformer_cfg.seed,transformer_cfg.initializer_range,transformer_cfg.init_weights,transformer_cfg.scale_attn_by_inverse_layer_idx,transformer_cfg.positional_embedding_type,transformer_cfg.final_rms,transformer_cfg.d_vocab_out,transformer_cfg.parallel_attn_mlp,transformer_cfg.rotary_dim,transformer_cfg.n_params,transformer_cfg.use_hook_tokens,transformer_cfg.gated_mlp,transformer_cfg.default_prepend_bos,transformer_cfg.dtype,transformer_cfg.tokenizer_prepends_bos,transformer_cfg.n_key_value_heads,transformer_cfg.post_embedding_ln,transformer_cfg.rotary_base,transformer_cfg.trust_remote_code,transformer_cfg.rotary_adjacent_pairs
-11,Counts the number of words in a sequence based on their length.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1460593486680443,True,False,standard,False,5.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
-13,"Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.1460593486680443,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-18,"Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.12344267996967354,True,False,standard,False,3.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
-19,Removes consecutive duplicate tokens from a sequence.,15.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,32.0,15.0,8.0,custom,4.0,128.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.15689290811054724,True,False,standard,False,3.0,False,,24576.0,False,False,True,torch.float32,,,False,10000.0,False,False
-20,Detect spam messages based on appearance of spam keywords.,10.0,4.0,0.05,0.001,False,1.0,1.0,1.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,14.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.16,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-21,Extract unique tokens from a string,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-24,Identifies the first occurrence of each token in a sequence.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-3,Returns the fraction of 'x' in the input up to the i-th position for all i.,5.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,12.0,5.0,3.0,custom,4.0,48.0,gelu,6.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.22188007849009167,True,False,standard,False,1.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
-33,Checks if each token's length is odd or even.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17457431218879393,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-34,Calculate the ratio of vowels to consonants in each word.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.16329931618554522,True,False,standard,False,5.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-35,Alternates capitalization of each character in words.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-36,"Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",10.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.19402850002906638,True,False,standard,False,3.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-37,Reverses each word in the sequence except for specified exclusions.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-38,Checks if tokens alternate between two types.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,2.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-4,Return fraction of previous open tokens minus the fraction of close tokens.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17056057308448835,True,False,standard,False,1.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-8,Fills gaps between tokens with a specified filler.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.13333333333333333,True,False,standard,False,8.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-ioi,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-ioi_next_token,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+case_id,url,task_description,max_seq_len,min_seq_len,training_args.atol,training_args.lr,training_args.use_single_loss,training_args.iit_weight,training_args.behavior_weight,training_args.strict_weight,training_args.epochs,training_args.act_fn,training_args.clip_grad_norm,training_args.lr_scheduler,transformer_cfg.n_layers,transformer_cfg.d_model,transformer_cfg.n_ctx,transformer_cfg.d_head,transformer_cfg.model_name,transformer_cfg.n_heads,transformer_cfg.d_mlp,transformer_cfg.act_fn,transformer_cfg.d_vocab,transformer_cfg.eps,transformer_cfg.use_attn_result,transformer_cfg.use_attn_scale,transformer_cfg.use_split_qkv_input,transformer_cfg.use_hook_mlp_in,transformer_cfg.use_attn_in,transformer_cfg.use_local_attn,transformer_cfg.original_architecture,transformer_cfg.from_checkpoint,transformer_cfg.checkpoint_index,transformer_cfg.checkpoint_label_type,transformer_cfg.checkpoint_value,transformer_cfg.tokenizer_name,transformer_cfg.window_size,transformer_cfg.attn_types,transformer_cfg.init_mode,transformer_cfg.normalization_type,transformer_cfg.device,transformer_cfg.n_devices,transformer_cfg.attention_dir,transformer_cfg.attn_only,transformer_cfg.seed,transformer_cfg.initializer_range,transformer_cfg.init_weights,transformer_cfg.scale_attn_by_inverse_layer_idx,transformer_cfg.positional_embedding_type,transformer_cfg.final_rms,transformer_cfg.d_vocab_out,transformer_cfg.parallel_attn_mlp,transformer_cfg.rotary_dim,transformer_cfg.n_params,transformer_cfg.use_hook_tokens,transformer_cfg.gated_mlp,transformer_cfg.default_prepend_bos,transformer_cfg.dtype,transformer_cfg.tokenizer_prepends_bos,transformer_cfg.n_key_value_heads,transformer_cfg.post_embedding_ln,transformer_cfg.rotary_base,transformer_cfg.trust_remote_code,transformer_cfg.rotary_adjacent_pairs
+11,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11,Counts the number of words in a sequence based on their length.,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1460593486680443,True,False,standard,False,5.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+13,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13,"Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.1460593486680443,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+18,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18,"Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.12344267996967354,True,False,standard,False,3.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+19,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19,Removes consecutive duplicate tokens from a sequence.,15,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,32.0,15.0,8.0,custom,4.0,128.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.15689290811054724,True,False,standard,False,3.0,False,,24576.0,False,False,True,torch.float32,,,False,10000.0,False,False
+20,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20,Detect spam messages based on appearance of spam keywords.,10,4,0.05,0.001,False,1.0,1.0,1.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,14.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.16,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+21,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21,Extract unique tokens from a string,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+24,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/24,Identifies the first occurrence of each token in a sequence.,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+3,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3,Returns the fraction of 'x' in the input up to the i-th position for all i.,5,4,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,12.0,5.0,3.0,custom,4.0,48.0,gelu,6.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.22188007849009167,True,False,standard,False,1.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+33,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33,Checks if each token's length is odd or even.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17457431218879393,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+34,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34,Calculate the ratio of vowels to consonants in each word.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.16329931618554522,True,False,standard,False,5.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+35,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35,Alternates capitalization of each character in words.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+36,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36,"Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",10,4,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.19402850002906638,True,False,standard,False,3.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+37,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37,Reverses each word in the sequence except for specified exclusions.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+38,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/38,Checks if tokens alternate between two types.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,2.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+4,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4,Return fraction of previous open tokens minus the fraction of close tokens.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17056057308448835,True,False,standard,False,1.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+8,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/8,Fills gaps between tokens with a specified filler.,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.13333333333333333,True,False,standard,False,8.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+ioi,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi,Indirect object identification,16,16,,,True,,,,,,,,,,,,,,,,,,True,True,True,True,True,True,,True,,,,,,,,,,,,True,,,True,True,,True,,True,,,True,True,True,,,,True,,True,True
+ioi_next_token,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token,Indirect object identification,16,16,,,True,,,,,,,,,,,,,,,,,,True,True,True,True,True,True,,True,,,,,,,,,,,,True,,,True,True,,True,,True,,,True,True,True,,,,True,,True,True

benchmark_cases_metadata.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01f473326ab84033784efd049808e3f0395bf577e23311a60f9b05eff32f5ca3
-size 49942
+oid sha256:12652b82dbded2521f44e1219ade14c88e6bd787db5a2141803db257fb375e87
+size 51034

benchmark_metadata.json

@@ -2,28 +2,43 @@
   "name": "InterpBench",
   "version": "1.0.0",
   "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.",
+  "license": "https://creativecommons.org/licenses/by/4.0/",
+  "url": "https://huggingface.co/cybershiptrooper/InterpBench",
   "cases": [
     {
       "case_id": "11",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11",
       "task_description": "Counts the number of words in a sequence based on their length.",
       "vocab": [
         "J",
-        "oCLrZaW",
-        "no",
-        "poiVg",
+        "LB",
+        "TPSI",
         "V",
         "b",
-        "LB",
-        "TPSI"
+        "no",
+        "oCLrZaW",
+        "poiVg"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 12,
@@ -91,12 +106,7 @@
     },
     {
       "case_id": "13",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13",
       "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",
       "vocab": [
         0,
@@ -105,6 +115,24 @@
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 20,
@@ -172,22 +200,35 @@
     },
     {
       "case_id": "18",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18",
       "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",
       "vocab": [
-        "c",
-        "e",
+        "a",
         "b",
+        "c",
         "d",
-        "a"
+        "e"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 12,
@@ -255,20 +296,33 @@
     },
     {
       "case_id": "19",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19",
       "task_description": "Removes consecutive duplicate tokens from a sequence.",
       "vocab": [
-        "b",
         "a",
+        "b",
         "c"
       ],
       "max_seq_len": 15,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 32,
@@ -336,29 +390,42 @@
     },
     {
       "case_id": "20",
-      "files": [
-        "edges.pkl",
-        "ll_model_1110.pth",
-        "ll_model_cfg_1110.pkl",
-        "meta_1110.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20",
       "task_description": "Detect spam messages based on appearance of spam keywords.",
       "vocab": [
         "J",
-        "spam",
-        "offer",
-        "click",
-        "oCLrZaW",
-        "no",
-        "poiVg",
+        "LB",
+        "TPSI",
         "V",
         "b",
-        "LB",
+        "click",
+        "no",
         "now",
-        "TPSI"
+        "oCLrZaW",
+        "offer",
+        "poiVg",
+        "spam"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_1110.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_1110.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_1110.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg_1110.pkl"
+        },
+        {
+          "file_name": "meta_1110.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta_1110.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 4,
@@ -426,20 +493,33 @@
     },
     {
       "case_id": "21",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21",
       "task_description": "Extract unique tokens from a string",
       "vocab": [
-        "b",
         "a",
+        "b",
         "c"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 20,
@@ -507,20 +587,33 @@
     },
     {
       "case_id": "24",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/24",
       "task_description": "Identifies the first occurrence of each token in a sequence.",
       "vocab": [
-        "b",
         "a",
+        "b",
         "c"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 20,
@@ -588,21 +681,34 @@
     },
     {
       "case_id": "3",
-      "files": [
-        "edges.pkl",
-        "ll_model_10110.pth",
-        "ll_model_cfg_10110.pkl",
-        "meta_10110.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3",
       "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.",
       "vocab": [
-        "x",
-        "b",
         "a",
-        "c"
+        "b",
+        "c",
+        "x"
       ],
       "max_seq_len": 5,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_10110.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_10110.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_10110.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg_10110.pkl"
+        },
+        {
+          "file_name": "meta_10110.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta_10110.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 12,
@@ -670,25 +776,38 @@
     },
     {
       "case_id": "33",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33",
       "task_description": "Checks if each token's length is odd or even.",
       "vocab": [
         "J",
-        "oCLrZaW",
-        "no",
-        "poiVg",
+        "LB",
+        "TPSI",
         "V",
         "b",
-        "LB",
-        "TPSI"
+        "no",
+        "oCLrZaW",
+        "poiVg"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 4,
@@ -756,25 +875,38 @@
     },
     {
       "case_id": "34",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34",
       "task_description": "Calculate the ratio of vowels to consonants in each word.",
       "vocab": [
         "J",
-        "oCLrZaW",
-        "no",
-        "poiVg",
+        "LB",
+        "TPSI",
         "V",
         "b",
-        "LB",
-        "TPSI"
+        "no",
+        "oCLrZaW",
+        "poiVg"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 4,
@@ -842,25 +974,38 @@
     },
     {
       "case_id": "35",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35",
       "task_description": "Alternates capitalization of each character in words.",
       "vocab": [
         "J",
-        "oCLrZaW",
-        "no",
-        "poiVg",
+        "LB",
+        "TPSI",
         "V",
         "b",
-        "LB",
-        "TPSI"
+        "no",
+        "oCLrZaW",
+        "poiVg"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 4,
@@ -928,20 +1073,33 @@
     },
     {
       "case_id": "36",
-      "files": [
-        "edges.pkl",
-        "ll_model_10110.pth",
-        "ll_model_cfg_10110.pkl",
-        "meta_10110.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36",
       "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",
       "vocab": [
-        "\ud83d\ude22",
         "\ud83d\udcd8",
-        "\ud83d\ude0a"
+        "\ud83d\ude0a",
+        "\ud83d\ude22"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_10110.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_10110.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_10110.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg_10110.pkl"
+        },
+        {
+          "file_name": "meta_10110.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta_10110.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 4,
@@ -1009,25 +1167,38 @@
     },
     {
       "case_id": "37",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37",
       "task_description": "Reverses each word in the sequence except for specified exclusions.",
       "vocab": [
         "J",
-        "oCLrZaW",
-        "no",
-        "poiVg",
+        "LB",
+        "TPSI",
         "V",
         "b",
-        "LB",
-        "TPSI"
+        "no",
+        "oCLrZaW",
+        "poiVg"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 4,
@@ -1095,20 +1266,33 @@
     },
     {
       "case_id": "38",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/38",
       "task_description": "Checks if tokens alternate between two types.",
       "vocab": [
-        "b",
         "a",
+        "b",
         "c"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 20,
@@ -1176,22 +1360,35 @@
     },
     {
       "case_id": "4",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4",
       "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.",
       "vocab": [
-        "b",
         "(",
-        "c",
         ")",
-        "a"
+        "a",
+        "b",
+        "c"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 20,
@@ -1259,25 +1456,38 @@
     },
     {
       "case_id": "8",
-      "files": [
-        "edges.pkl",
-        "ll_model_510.pth",
-        "ll_model_cfg_510.pkl",
-        "meta_510.json"
-      ],
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/8",
       "task_description": "Fills gaps between tokens with a specified filler.",
       "vocab": [
         "J",
-        "oCLrZaW",
-        "no",
-        "poiVg",
+        "LB",
+        "TPSI",
         "V",
         "b",
-        "LB",
-        "TPSI"
+        "no",
+        "oCLrZaW",
+        "poiVg"
       ],
       "max_seq_len": 10,
       "min_seq_len": 4,
+      "files": [
+        {
+          "file_name": "edges.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/edges.pkl"
+        },
+        {
+          "file_name": "ll_model_510.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/ll_model_510.pth"
+        },
+        {
+          "file_name": "ll_model_cfg_510.pkl",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/ll_model_cfg_510.pkl"
+        },
+        {
+          "file_name": "meta_510.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/meta_510.json"
+        }
+      ],
       "transformer_cfg": {
         "n_layers": 2,
         "d_model": 20,
@@ -1345,17 +1555,40 @@
     },
     {
       "case_id": "ioi",
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi",
+      "task_description": "Indirect object identification",
+      "max_seq_len": 16,
+      "min_seq_len": 16,
       "files": [
-        "corr_100_100_40.json",
-        "ll_model_100_100_40.pth"
+        {
+          "file_name": "corr_100_100_40.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/corr_100_100_40.json"
+        },
+        {
+          "file_name": "ll_model_100_100_40.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_100_100_40.pth"
+        }
       ]
     },
     {
       "case_id": "ioi_next_token",
+      "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token",
+      "task_description": "Indirect object identification",
+      "max_seq_len": 16,
+      "min_seq_len": 16,
       "files": [
-        "corr_100_100_40.json",
-        "ll_model_100_100_40.pth",
-        "training_args.json"
+        {
+          "file_name": "corr_100_100_40.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/corr_100_100_40.json"
+        },
+        {
+          "file_name": "ll_model_100_100_40.pth",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_100_100_40.pth"
+        },
+        {
+          "file_name": "training_args.json",
+          "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/training_args.json"
+        }
       ]
     }
   ]

benchmark_metadata_croissant.json

@@ -0,0 +1,1103 @@
+{
+  "@context": {
+    "@language": "en",
+    "@vocab": "https://schema.org/",
+    "citeAs": "cr:citeAs",
+    "column": "cr:column",
+    "conformsTo": "dct:conformsTo",
+    "cr": "http://mlcommons.org/croissant/",
+    "rai": "http://mlcommons.org/croissant/RAI/",
+    "data": {
+      "@id": "cr:data",
+      "@type": "@json"
+    },
+    "dataType": {
+      "@id": "cr:dataType",
+      "@type": "@vocab"
+    },
+    "dct": "http://purl.org/dc/terms/",
+    "examples": {
+      "@id": "cr:examples",
+      "@type": "@json"
+    },
+    "extract": "cr:extract",
+    "field": "cr:field",
+    "fileProperty": "cr:fileProperty",
+    "fileObject": "cr:fileObject",
+    "fileSet": "cr:fileSet",
+    "format": "cr:format",
+    "includes": "cr:includes",
+    "isLiveDataset": "cr:isLiveDataset",
+    "jsonPath": "cr:jsonPath",
+    "key": "cr:key",
+    "md5": "cr:md5",
+    "parentField": "cr:parentField",
+    "path": "cr:path",
+    "recordSet": "cr:recordSet",
+    "references": "cr:references",
+    "regex": "cr:regex",
+    "repeated": "cr:repeated",
+    "replace": "cr:replace",
+    "sc": "https://schema.org/",
+    "separator": "cr:separator",
+    "source": "cr:source",
+    "subField": "cr:subField",
+    "transform": "cr:transform"
+  },
+  "@type": "sc:Dataset",
+  "name": "InterpBench",
+  "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.",
+  "conformsTo": "http://mlcommons.org/croissant/1.0",
+  "license": "https://creativecommons.org/licenses/by/4.0/",
+  "url": "https://huggingface.co/cybershiptrooper/InterpBench",
+  "version": "1.0.0",
+  "distribution": [
+    {
+      "@type": "cr:FileObject",
+      "@id": "hf-repository",
+      "name": "hf-repository",
+      "description": "The Hugging Face git repository.",
+      "contentUrl": "https://huggingface.co/cybershiptrooper/InterpBench",
+      "encodingFormat": "git+https",
+      "sha256": "main"
+    },
+    {
+      "@type": "cr:FileObject",
+      "@id": "benchmark-cases-parquet",
+      "name": "benchmark-cases-parquet",
+      "description": "Parquet file describing all the cases in the benchmark.",
+      "containedIn": {
+        "@id": "hf-repository"
+      },
+      "encodingFormat": "application/x-parquet"
+    },
+    {
+      "@type": "cr:FileSet",
+      "@id": "training-args",
+      "name": "training-args",
+      "description": "Training arguments.",
+      "containedIn": {
+        "@id": "hf-repository"
+      },
+      "encodingFormat": "application/json",
+      "includes": "*/meta_[0-9]*.json"
+    },
+    {
+      "@type": "cr:FileSet",
+      "@id": "circuits",
+      "name": "circuits",
+      "description": "Ground truth circuits (Pickle).",
+      "containedIn": {
+        "@id": "hf-repository"
+      },
+      "encodingFormat": "application/octet-stream",
+      "includes": "*/edges.pkl"
+    },
+    {
+      "@type": "cr:FileSet",
+      "@id": "weights",
+      "name": "weights",
+      "description": "Serialized PyTorch state dictionaries (Pickle).",
+      "containedIn": {
+        "@id": "hf-repository"
+      },
+      "encodingFormat": "application/octet-stream",
+      "includes": "*/ll_model_[0-9]*.pkl"
+    },
+    {
+      "@type": "cr:FileSet",
+      "@id": "cfgs",
+      "name": "cfgs",
+      "description": "Architecture configs (Pickle).",
+      "containedIn": {
+        "@id": "hf-repository"
+      },
+      "encodingFormat": "application/octet-stream",
+      "includes": "*/ll_model_cfg_[0-9]*.pkl"
+    }
+  ],
+  "recordSet": [
+    {
+      "@type": "cr:RecordSet",
+      "@id": "cases",
+      "name": "cases",
+      "field": [
+        {
+          "@type": "cr:Field",
+          "@id": "case_id",
+          "name": "case_id",
+          "description": "Column 'case_id' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "case_id"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "url",
+          "name": "url",
+          "description": "Column 'url' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "url"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "task_description",
+          "name": "task_description",
+          "description": "Column 'task_description' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "task_description"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "max_seq_len",
+          "name": "max_seq_len",
+          "description": "Column 'max_seq_len' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Integer",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "max_seq_len"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "min_seq_len",
+          "name": "min_seq_len",
+          "description": "Column 'min_seq_len' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Integer",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "min_seq_len"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.atol",
+          "name": "training_args.atol",
+          "description": "Column 'training_args.atol' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.atol"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.lr",
+          "name": "training_args.lr",
+          "description": "Column 'training_args.lr' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.lr"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.use_single_loss",
+          "name": "training_args.use_single_loss",
+          "description": "Column 'training_args.use_single_loss' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.use_single_loss"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.iit_weight",
+          "name": "training_args.iit_weight",
+          "description": "Column 'training_args.iit_weight' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.iit_weight"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.behavior_weight",
+          "name": "training_args.behavior_weight",
+          "description": "Column 'training_args.behavior_weight' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.behavior_weight"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.strict_weight",
+          "name": "training_args.strict_weight",
+          "description": "Column 'training_args.strict_weight' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.strict_weight"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.epochs",
+          "name": "training_args.epochs",
+          "description": "Column 'training_args.epochs' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.epochs"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.act_fn",
+          "name": "training_args.act_fn",
+          "description": "Column 'training_args.act_fn' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.act_fn"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.clip_grad_norm",
+          "name": "training_args.clip_grad_norm",
+          "description": "Column 'training_args.clip_grad_norm' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.clip_grad_norm"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "training_args.lr_scheduler",
+          "name": "training_args.lr_scheduler",
+          "description": "Column 'training_args.lr_scheduler' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "training_args.lr_scheduler"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.n_layers",
+          "name": "transformer_cfg.n_layers",
+          "description": "Column 'transformer_cfg.n_layers' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.n_layers"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.d_model",
+          "name": "transformer_cfg.d_model",
+          "description": "Column 'transformer_cfg.d_model' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.d_model"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.n_ctx",
+          "name": "transformer_cfg.n_ctx",
+          "description": "Column 'transformer_cfg.n_ctx' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.n_ctx"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.d_head",
+          "name": "transformer_cfg.d_head",
+          "description": "Column 'transformer_cfg.d_head' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.d_head"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.model_name",
+          "name": "transformer_cfg.model_name",
+          "description": "Column 'transformer_cfg.model_name' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.model_name"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.n_heads",
+          "name": "transformer_cfg.n_heads",
+          "description": "Column 'transformer_cfg.n_heads' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.n_heads"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.d_mlp",
+          "name": "transformer_cfg.d_mlp",
+          "description": "Column 'transformer_cfg.d_mlp' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.d_mlp"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.act_fn",
+          "name": "transformer_cfg.act_fn",
+          "description": "Column 'transformer_cfg.act_fn' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.act_fn"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.d_vocab",
+          "name": "transformer_cfg.d_vocab",
+          "description": "Column 'transformer_cfg.d_vocab' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.d_vocab"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.eps",
+          "name": "transformer_cfg.eps",
+          "description": "Column 'transformer_cfg.eps' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.eps"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.use_attn_result",
+          "name": "transformer_cfg.use_attn_result",
+          "description": "Column 'transformer_cfg.use_attn_result' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.use_attn_result"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.use_attn_scale",
+          "name": "transformer_cfg.use_attn_scale",
+          "description": "Column 'transformer_cfg.use_attn_scale' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.use_attn_scale"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.use_split_qkv_input",
+          "name": "transformer_cfg.use_split_qkv_input",
+          "description": "Column 'transformer_cfg.use_split_qkv_input' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.use_split_qkv_input"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.use_hook_mlp_in",
+          "name": "transformer_cfg.use_hook_mlp_in",
+          "description": "Column 'transformer_cfg.use_hook_mlp_in' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.use_hook_mlp_in"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.use_attn_in",
+          "name": "transformer_cfg.use_attn_in",
+          "description": "Column 'transformer_cfg.use_attn_in' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.use_attn_in"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.use_local_attn",
+          "name": "transformer_cfg.use_local_attn",
+          "description": "Column 'transformer_cfg.use_local_attn' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.use_local_attn"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.original_architecture",
+          "name": "transformer_cfg.original_architecture",
+          "description": "Column 'transformer_cfg.original_architecture' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.original_architecture"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.from_checkpoint",
+          "name": "transformer_cfg.from_checkpoint",
+          "description": "Column 'transformer_cfg.from_checkpoint' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.from_checkpoint"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.checkpoint_index",
+          "name": "transformer_cfg.checkpoint_index",
+          "description": "Column 'transformer_cfg.checkpoint_index' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.checkpoint_index"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.checkpoint_label_type",
+          "name": "transformer_cfg.checkpoint_label_type",
+          "description": "Column 'transformer_cfg.checkpoint_label_type' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.checkpoint_label_type"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.checkpoint_value",
+          "name": "transformer_cfg.checkpoint_value",
+          "description": "Column 'transformer_cfg.checkpoint_value' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.checkpoint_value"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.tokenizer_name",
+          "name": "transformer_cfg.tokenizer_name",
+          "description": "Column 'transformer_cfg.tokenizer_name' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.tokenizer_name"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.window_size",
+          "name": "transformer_cfg.window_size",
+          "description": "Column 'transformer_cfg.window_size' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.window_size"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.attn_types",
+          "name": "transformer_cfg.attn_types",
+          "description": "Column 'transformer_cfg.attn_types' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.attn_types"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.init_mode",
+          "name": "transformer_cfg.init_mode",
+          "description": "Column 'transformer_cfg.init_mode' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.init_mode"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.normalization_type",
+          "name": "transformer_cfg.normalization_type",
+          "description": "Column 'transformer_cfg.normalization_type' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.normalization_type"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.device",
+          "name": "transformer_cfg.device",
+          "description": "Column 'transformer_cfg.device' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.device"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.n_devices",
+          "name": "transformer_cfg.n_devices",
+          "description": "Column 'transformer_cfg.n_devices' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.n_devices"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.attention_dir",
+          "name": "transformer_cfg.attention_dir",
+          "description": "Column 'transformer_cfg.attention_dir' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.attention_dir"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.attn_only",
+          "name": "transformer_cfg.attn_only",
+          "description": "Column 'transformer_cfg.attn_only' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.attn_only"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.seed",
+          "name": "transformer_cfg.seed",
+          "description": "Column 'transformer_cfg.seed' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.seed"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.initializer_range",
+          "name": "transformer_cfg.initializer_range",
+          "description": "Column 'transformer_cfg.initializer_range' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.initializer_range"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.init_weights",
+          "name": "transformer_cfg.init_weights",
+          "description": "Column 'transformer_cfg.init_weights' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.init_weights"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.scale_attn_by_inverse_layer_idx",
+          "name": "transformer_cfg.scale_attn_by_inverse_layer_idx",
+          "description": "Column 'transformer_cfg.scale_attn_by_inverse_layer_idx' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.scale_attn_by_inverse_layer_idx"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.positional_embedding_type",
+          "name": "transformer_cfg.positional_embedding_type",
+          "description": "Column 'transformer_cfg.positional_embedding_type' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.positional_embedding_type"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.final_rms",
+          "name": "transformer_cfg.final_rms",
+          "description": "Column 'transformer_cfg.final_rms' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.final_rms"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.d_vocab_out",
+          "name": "transformer_cfg.d_vocab_out",
+          "description": "Column 'transformer_cfg.d_vocab_out' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.d_vocab_out"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.parallel_attn_mlp",
+          "name": "transformer_cfg.parallel_attn_mlp",
+          "description": "Column 'transformer_cfg.parallel_attn_mlp' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.parallel_attn_mlp"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.rotary_dim",
+          "name": "transformer_cfg.rotary_dim",
+          "description": "Column 'transformer_cfg.rotary_dim' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.rotary_dim"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.n_params",
+          "name": "transformer_cfg.n_params",
+          "description": "Column 'transformer_cfg.n_params' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.n_params"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.use_hook_tokens",
+          "name": "transformer_cfg.use_hook_tokens",
+          "description": "Column 'transformer_cfg.use_hook_tokens' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.use_hook_tokens"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.gated_mlp",
+          "name": "transformer_cfg.gated_mlp",
+          "description": "Column 'transformer_cfg.gated_mlp' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.gated_mlp"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.default_prepend_bos",
+          "name": "transformer_cfg.default_prepend_bos",
+          "description": "Column 'transformer_cfg.default_prepend_bos' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.default_prepend_bos"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.dtype",
+          "name": "transformer_cfg.dtype",
+          "description": "Column 'transformer_cfg.dtype' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.dtype"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.tokenizer_prepends_bos",
+          "name": "transformer_cfg.tokenizer_prepends_bos",
+          "description": "Column 'transformer_cfg.tokenizer_prepends_bos' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.tokenizer_prepends_bos"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.n_key_value_heads",
+          "name": "transformer_cfg.n_key_value_heads",
+          "description": "Column 'transformer_cfg.n_key_value_heads' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.n_key_value_heads"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.post_embedding_ln",
+          "name": "transformer_cfg.post_embedding_ln",
+          "description": "Column 'transformer_cfg.post_embedding_ln' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.post_embedding_ln"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.rotary_base",
+          "name": "transformer_cfg.rotary_base",
+          "description": "Column 'transformer_cfg.rotary_base' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Float",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.rotary_base"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.trust_remote_code",
+          "name": "transformer_cfg.trust_remote_code",
+          "description": "Column 'transformer_cfg.trust_remote_code' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.trust_remote_code"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "transformer_cfg.rotary_adjacent_pairs",
+          "name": "transformer_cfg.rotary_adjacent_pairs",
+          "description": "Column 'transformer_cfg.rotary_adjacent_pairs' from the parquet file describing all the cases in the benchmark.",
+          "dataType": "sc:Boolean",
+          "source": {
+            "fileSet": {
+              "@id": "benchmark-cases-parquet"
+            },
+            "extract": {
+              "column": "transformer_cfg.rotary_adjacent_pairs"
+            }
+          }
+        }
+      ]
+    }
+  ]
+}