Upload ZettHypernet

config.json

@@ -1,15 +1,16 @@
 {
-  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B",
   "architectures": [
     "ZettHypernet"
   ],
+  "attention_bias": false,
   "attention_dropout": 0.0,
   "auto_map": {
     "AutoConfig": "configuration_hypernet.ZettHypernetConfig",
     "AutoModel": "modeling_hypernet.ZettHypernet"
   },
-  "bos_token_id": 1,
-  "eos_token_id": 2,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "hn_add_inter_token_attention": false,
@@ -24,7 +25,7 @@
   "hn_language_adapter_bottleneck_dim": 0,
   "hn_model_name_or_path": "roberta-base",
   "hn_model_type": "roberta",
-  "hn_n_extra_tokens": 522,
+  "hn_n_extra_tokens": 0,
   "hn_n_inter_token_blocks": 16,
   "hn_n_layers": 3,
   "hn_num_attention_heads": 32,
@@ -34,19 +35,20 @@
   "hn_surface_maxlen": 7,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 32768,
+  "max_position_embeddings": 8192,
   "n_embd": 4096,
   "n_langs": 7,
-  "name": "v7:mistral7b_en+code:lw=0.5_long",
+  "name": "v7:llama3-8b_en+code:lw=0.5_long",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "original_vocab_size": 32000,
-  "pad_token_id": 2,
+  "original_vocab_size": 128256,
+  "pad_token_id": 128001,
+  "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
-  "rope_theta": 10000.0,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
   "separate_out_embeddings": true,
-  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
   "transformers_version": "4.42.3",

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58ff19794dc856869f1c6a52df63ad0573d1081a2861929e7c48ae1634481af5
-size 2710971844
+oid sha256:3cf90db11d32842a09617888ceefd36919408ed84628b82b6e9ff25dea1fdc0d
+size 4791035356