{ "metadata": { "ParamSize": 518, "ParamBytes": 1738526720.0, "BitsPerParam": 5.010408347335939 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 64552960, "records": [ { "name": "gpt_neox.embed_in.q_weight", "shape": [ 50432, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 64552960, "byteOffset": 0 } ], "md5sum": "e65e8796e5e8cce8ac7ea3dfbf8a288a" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 32604160, "records": [ { "name": "gpt_neox.embed_in.q_scale", "shape": [ 50432, 80 ], "dtype": "float32", "format": "raw", "nbytes": 16138240, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16138240 }, { "name": "gpt_neox.layers.0.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16148480 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16158720 }, { "name": "gpt_neox.layers.0.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16168960 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16179200 }, { "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26009600 }, { "name": "gpt_neox.layers.0.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28467200 }, { "name": "gpt_neox.layers.0.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28497920 }, { "name": "gpt_neox.layers.0.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 31774720 }, { "name": "gpt_neox.layers.0.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32593920 } ], "md5sum": "506e9427406c74b0bf5b4b24fb00cbae" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.1.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.1.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.1.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "6185ffeb1b33da59d73ae2967c6bb442" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.1.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.1.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.1.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.1.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "8f749f91ddbab931dd212fa1b4df0d51" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.2.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.2.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.2.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.2.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.2.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.2.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.2.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "6e3e620ee925777df20f56f5b88e2b35" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.3.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.3.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.3.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "6799407e1a1eb4c5f3417c9c5709f642" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.3.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.3.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.3.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.3.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "d6fe5397b8cc5e0ad3a66b7c7a3536bf" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.4.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.4.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.4.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.4.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.4.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.4.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.4.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "b47ab02c5b6de722a6ea74fab12f2137" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.5.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.5.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.5.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "9100970503defeb117f58f11f03ea8de" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.5.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.5.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.5.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.5.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "dab7777ac759ba43db020d9c61ca24e9" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.6.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.6.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.6.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.6.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.6.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.6.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.6.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "5e9b5beac432bd7587c2f9abc49b4b3b" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.7.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.7.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.7.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "a71f63374fe69d389a9cf465ee611aca" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.7.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.7.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.7.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.7.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "e85556f704c2f3bb85c69e8ce71c2b2e" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.8.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.8.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.8.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.8.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.8.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.8.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.8.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "f22b0bd4f27faf2427f206899591181d" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.9.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.9.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.9.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "2438429194fbbac9c078460ca03c507c" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.9.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.9.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.9.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.9.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "ca6811a361c3c1067ca595cc5d6ce65f" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.10.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.10.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.10.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.10.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.10.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.10.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.10.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "6b5b31948d57cbf3553792e07dab0ad2" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.11.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.11.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.11.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "820ad1a5ec9ec66e960a27f157f80648" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.11.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.11.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.11.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.11.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "00acc9917321b899ff99ae3750804355" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.12.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.12.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.12.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.12.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.12.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.12.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.12.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.12.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "af475bbe569923eb2a2356049bf8ca83" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.13.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.13.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.13.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "c368499847e95159e151f2fb82ee34ca" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.13.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.13.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.13.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.13.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.13.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.13.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "d9655a3d9c6e7ea22f16eba46828c3ba" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.14.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.14.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.14.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.14.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.14.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.14.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.14.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.14.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "c1c1c1dca1a0fd1d23ac2743c94df013" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.15.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.15.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.15.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "5a574a2d682d4459daec407348aa7a43" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.15.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.15.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.15.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.15.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.15.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.15.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "754acaa5c6b0dc8316a437e14ea2f430" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.16.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.16.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.16.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.16.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.16.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.16.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.16.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.16.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.16.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "b26084a5f4defe2db8644ac3c8dc1d44" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.17.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.17.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.17.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "7525b918517a80e5d548f614d2fcd2e0" }, { "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.17.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.17.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.17.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.17.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.17.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.17.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "1ae1c39414fc4a973c739c12433a6443" }, { "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.18.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.18.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.18.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.18.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.18.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.18.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.18.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.18.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.18.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "dd741f555678b27d5707a01d9be859a2" }, { "dataPath": "params_shard_29.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.19.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.19.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.19.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "88fbad4e8f625461904603962dd4e3df" }, { "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.19.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.19.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.19.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.19.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.19.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.19.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "347678378af964ef64328217a72063bf" }, { "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.20.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.20.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.20.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.20.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.20.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.20.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.20.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.20.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.20.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "e3d095771bb04ff0ec6a2ebed2d72652" }, { "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.21.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.21.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.21.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "a08359c8c11cda410d439aec53a7c687" }, { "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.21.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.21.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.21.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.21.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.21.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.21.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "4db4063ca67dd7b6d3437df6d0598228" }, { "dataPath": "params_shard_34.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.22.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.22.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.22.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.22.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.22.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.22.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.22.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.22.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.22.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "eba53144a64724705d075065a345a4f9" }, { "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.23.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.23.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.23.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "fd7850af14a6a8d87cf20d105acfa0a3" }, { "dataPath": "params_shard_36.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.23.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.23.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.23.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.23.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.23.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.23.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "2508be19561d35b5e2c2ff8a53e9a269" }, { "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.24.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.24.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.24.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.24.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.24.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.24.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.24.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.24.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.24.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.24.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "aa9d9236ec75307d77a3b57db9bc72ba" }, { "dataPath": "params_shard_38.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.25.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.25.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.25.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.25.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "b250bb0f6165f4317a256902e5093657" }, { "dataPath": "params_shard_39.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.25.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.25.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.25.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.25.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.25.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.25.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "fbe1041cc8d2b2e1829bb9d6dba62028" }, { "dataPath": "params_shard_40.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.26.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.26.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.26.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.26.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.26.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.26.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.26.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.26.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.26.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.26.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "8f928520e3d689e7726d18548206366d" }, { "dataPath": "params_shard_41.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.27.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.27.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.27.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.27.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "409b12f1645ef4fdd9a61bd0da8e83fe" }, { "dataPath": "params_shard_42.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.27.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.27.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.27.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.27.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.27.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.27.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "33c3d76b713c519d7fe90bfd80e93def" }, { "dataPath": "params_shard_43.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.28.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.28.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.28.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.28.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.28.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.28.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.28.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.28.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.28.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.28.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "87cf7e48f7d6cc520be0fae00115ff57" }, { "dataPath": "params_shard_44.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.29.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.29.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.29.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.29.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "1667fb8df1a66ee054283caea6113da9" }, { "dataPath": "params_shard_45.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.29.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.29.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.29.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.29.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.29.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.29.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "f448efc3ae9e949d9343581e78fa64ef" }, { "dataPath": "params_shard_46.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.30.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.layers.30.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "gpt_neox.layers.30.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.30.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.30.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 16435200 }, { "name": "gpt_neox.layers.30.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 26265600 }, { "name": "gpt_neox.layers.30.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 28723200 }, { "name": "gpt_neox.layers.30.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 28753920 }, { "name": "gpt_neox.layers.30.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 32030720 }, { "name": "gpt_neox.layers.30.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "37b201b978536e7abf8694075a364488" }, { "dataPath": "params_shard_47.bin", "format": "raw-shard", "nbytes": 32860160, "records": [ { "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 16384000 }, { "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32808960 }, { "name": "gpt_neox.layers.31.input_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32819200 }, { "name": "gpt_neox.layers.31.input_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32829440 }, { "name": "gpt_neox.layers.31.post_attention_layernorm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32839680 }, { "name": "gpt_neox.layers.31.post_attention_layernorm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 32849920 } ], "md5sum": "77bf1ed46cfaf71cf88bbf05609cb3fc" }, { "dataPath": "params_shard_48.bin", "format": "raw-shard", "nbytes": 32849920, "records": [ { "name": "gpt_neox.layers.31.attention.query_key_value.q_weight", "shape": [ 7680, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 9830400, "byteOffset": 0 }, { "name": "gpt_neox.layers.31.attention.query_key_value.q_scale", "shape": [ 7680, 80 ], "dtype": "float32", "format": "raw", "nbytes": 2457600, "byteOffset": 9830400 }, { "name": "gpt_neox.layers.31.attention.query_key_value.bias", "shape": [ 7680 ], "dtype": "float32", "format": "raw", "nbytes": 30720, "byteOffset": 12288000 }, { "name": "gpt_neox.layers.31.attention.dense.q_weight", "shape": [ 2560, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 3276800, "byteOffset": 12318720 }, { "name": "gpt_neox.layers.31.attention.dense.q_scale", "shape": [ 2560, 80 ], "dtype": "float32", "format": "raw", "nbytes": 819200, "byteOffset": 15595520 }, { "name": "gpt_neox.layers.31.attention.dense.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16414720 }, { "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_weight", "shape": [ 10240, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 16424960 }, { "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_scale", "shape": [ 10240, 80 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 29532160 }, { "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.bias", "shape": [ 10240 ], "dtype": "float32", "format": "raw", "nbytes": 40960, "byteOffset": 32808960 } ], "md5sum": "3b34c10fdd5704db70771615e5e5ab6d" }, { "dataPath": "params_shard_49.bin", "format": "raw-shard", "nbytes": 64552960, "records": [ { "name": "embed_out.q_weight", "shape": [ 50432, 320 ], "dtype": "uint32", "format": "raw", "nbytes": 64552960, "byteOffset": 0 } ], "md5sum": "25ea3e71fa498e95a5f1849d36ab9f50" }, { "dataPath": "params_shard_50.bin", "format": "raw-shard", "nbytes": 32552960, "records": [ { "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_weight", "shape": [ 2560, 1280 ], "dtype": "uint32", "format": "raw", "nbytes": 13107200, "byteOffset": 0 }, { "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_scale", "shape": [ 2560, 320 ], "dtype": "float32", "format": "raw", "nbytes": 3276800, "byteOffset": 13107200 }, { "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16384000 }, { "name": "gpt_neox.final_layer_norm.weight", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16394240 }, { "name": "gpt_neox.final_layer_norm.bias", "shape": [ 2560 ], "dtype": "float32", "format": "raw", "nbytes": 10240, "byteOffset": 16404480 }, { "name": "embed_out.q_scale", "shape": [ 50432, 80 ], "dtype": "float32", "format": "raw", "nbytes": 16138240, "byteOffset": 16414720 } ], "md5sum": "3d49453ac0ed0334cf213967b00a0e7b" } ] }