{ "metadata": { "ParamSize": 195, "ParamBytes": 3513912320.0, "BitsPerParam": 16.0 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 262144000, "records": [ { "name": "transformer.embd.weight", "shape": [ 51200, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144000, "byteOffset": 0 } ], "md5sum": "92e1da58eb645448e09ca03417db18f3" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.0.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "fe6498a99c51e2973b01337d7ef56239" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.0.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "0bdc7b67ee83655337c030646e83d88e" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.0.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "5d10e5a3d6384a7fc31ec1f207b379ec" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.1.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "7bd23caad9e0fd9f4527ca5c25cf3d8f" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.1.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "4d3b3ee8deab0e85807b3a83c1eed0e1" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.1.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "48483266ee7a4c7ff9dfcb118ebf3d96" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.2.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "3a49f142bf7930c0ee5a94e20f010f13" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 26342400, "records": [ { "name": "transformer.h.0.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 0 }, { "name": "transformer.h.0.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 15360 }, { "name": "transformer.h.0.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13122560 }, { "name": "transformer.h.0.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13127680 }, { "name": "transformer.h.0.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13148160 }, { "name": "transformer.h.0.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13153280 }, { "name": "transformer.h.0.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13158400 }, { "name": "transformer.h.1.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13163520 }, { "name": "transformer.h.1.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13178880 }, { "name": "transformer.h.1.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26286080 }, { "name": "transformer.h.1.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26291200 }, { "name": "transformer.h.1.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26311680 }, { "name": "transformer.h.1.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26316800 }, { "name": "transformer.h.1.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26321920 }, { "name": "transformer.h.2.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26327040 } ], "md5sum": "21044736528bac2a17532ebbb4e6a9f2" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.2.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "3cea15f82e21560e1cce5d8063947824" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.2.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "8365db0f576a14c95219fc9dda08e260" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.3.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "26a07301f01ffa3e7a3a6149f65e405e" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.3.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "c1bb8512e839bf599a9ac98892009925" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.3.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "b96e8aafaac424695560941d5e50ad30" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.4.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "928dfd41b96d24a6ee5830f614e90a45" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.2.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.2.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.2.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.2.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.2.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.2.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.3.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.3.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.3.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.3.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.3.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.3.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.3.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.4.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "d4609d9670a54cdb87497edd6f171677" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.4.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "1f28448279bf1ba57250ee22add97da3" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.4.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "fd1c56b1710979e1d82ec440fdd94df0" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.5.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "17167b52036123547f13af01f8b693f4" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.5.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "3fa5a9f5f7bc1f159a4ebda20fb47362" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.5.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "64d5b6885efe51ccd3d7103ea0761603" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.6.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "8448e6e8d38c3886c23714baeacd9c37" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.4.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.4.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.4.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.4.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.4.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.4.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.5.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.5.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.5.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.5.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.5.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.5.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.5.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.6.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "0285d3baf1a0f6c92c51a34c88920bee" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.6.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "5d6e54c3dc08202399c9d31999f533d5" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.6.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "977cac1bc6c99b131e2ce1a4af98704a" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.7.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "36abbb0380d0d56a8713550c5e6734ff" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.7.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "c980565fe25899f39a70dd6b47eb58e6" }, { "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.7.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "30caa3ec65e21a25c14b8c8a49a80553" }, { "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.8.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "e6c2f54635dd7446a39479f3211d1e1a" }, { "dataPath": "params_shard_29.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.6.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.6.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.6.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.6.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.6.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.6.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.7.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.7.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.7.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.7.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.7.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.7.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.7.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.8.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "038978dc579f07e23435b6f645739932" }, { "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.8.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "fcbe8f8702945e3109b6471d11bb78f0" }, { "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.8.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "86725f22c1a86fab7db93b993e01afcd" }, { "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.9.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "fe7c12ea21e509686c2f2d7af962e1ea" }, { "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.9.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "ee3aeb307aafd0321a7939c6d203d315" }, { "dataPath": "params_shard_34.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.9.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "88e600b09a6931ffd3f1d62c7be0a352" }, { "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.10.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "c9bb5138c9fe774d49f0a7f4b8694c0a" }, { "dataPath": "params_shard_36.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.8.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.8.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.8.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.8.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.8.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.8.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.9.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.9.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.9.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.9.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.9.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.9.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.9.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.10.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "18c5ace06fb6a7e938c936f390c2e6b3" }, { "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.10.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "b05d02fc4c3eacdbb0d6e2d7785c6074" }, { "dataPath": "params_shard_38.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.10.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "2582c421aeb0e7e63bfa8453795202bd" }, { "dataPath": "params_shard_39.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.11.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "5fe526eac6f8a4e5a888b6f1e8799d2b" }, { "dataPath": "params_shard_40.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.11.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "eff5b70f517b4d1a0af2ad13f3262d2e" }, { "dataPath": "params_shard_41.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.11.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "a5859e66fa955c09de776975f24f6c81" }, { "dataPath": "params_shard_42.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.12.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "1e5ba87da59ac72ebfcd38871cd5fed2" }, { "dataPath": "params_shard_43.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.10.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.10.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.10.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.10.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.10.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.10.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.11.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.11.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.11.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.11.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.11.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.11.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.11.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.12.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "e811817d5018c810acb7a657795b4653" }, { "dataPath": "params_shard_44.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.12.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "5f4b1f6dd378cb1d86c262fd9cb537d6" }, { "dataPath": "params_shard_45.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.12.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "da2eb09044be04caeea3ba1198af0a02" }, { "dataPath": "params_shard_46.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.13.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "5ac09f0e8575f33a470514120718358c" }, { "dataPath": "params_shard_47.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.13.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "3fb82f49524663c96f127b37073c8860" }, { "dataPath": "params_shard_48.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.13.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "87130dccf8e9a4f58b9843d6c1e79fb5" }, { "dataPath": "params_shard_49.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.14.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "07962eb90630eaec197d525039e2e9df" }, { "dataPath": "params_shard_50.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.12.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.12.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.12.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.12.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.12.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.12.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.13.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.13.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.13.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.13.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.13.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.13.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.13.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.14.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "da694b4857bc0db25ed65654f8642d01" }, { "dataPath": "params_shard_51.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.14.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "abff0df61e86f9521dbec4e50ac77038" }, { "dataPath": "params_shard_52.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.14.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "25c3a2dff2a3ca9f8ee1ea8dddc8c3f1" }, { "dataPath": "params_shard_53.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.15.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "5eedc47ccdcff35f60877503bf6e66b9" }, { "dataPath": "params_shard_54.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.15.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "8dcced55b9eaec4b718b9a56f78e4d49" }, { "dataPath": "params_shard_55.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.15.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "5a75fd46e30ccf641daa0636b5dd0264" }, { "dataPath": "params_shard_56.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.16.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "b899b0dbcb031aeacc451306c8f1ae25" }, { "dataPath": "params_shard_57.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.14.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.14.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.14.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.14.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.14.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.14.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.15.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.15.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.15.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.15.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.15.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.15.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.15.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.16.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "709f450e75e4f94ca8aabaa481784b8b" }, { "dataPath": "params_shard_58.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.16.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "c0b683977d7a2edc44653b064306150d" }, { "dataPath": "params_shard_59.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.16.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "f125a8a4b3545863244f18623cc512da" }, { "dataPath": "params_shard_60.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.17.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "8404f1afc77cc40a7178a0df1c7c9f39" }, { "dataPath": "params_shard_61.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.17.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "bd8ac225ee2c277df3c93b493d2d8933" }, { "dataPath": "params_shard_62.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.17.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "037c2f1c81ba45bedc504c8856457dde" }, { "dataPath": "params_shard_63.bin", "format": "raw-shard", "nbytes": 39321600, "records": [ { "name": "transformer.h.18.mixer.Wqkv.weight", "shape": [ 7680, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 39321600, "byteOffset": 0 } ], "md5sum": "5c4516fcf851bf971eb062470b910b66" }, { "dataPath": "params_shard_64.bin", "format": "raw-shard", "nbytes": 26327040, "records": [ { "name": "transformer.h.16.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.16.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.16.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.16.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.16.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.16.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "transformer.h.17.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 13148160 }, { "name": "transformer.h.17.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13163520 }, { "name": "transformer.h.17.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26270720 }, { "name": "transformer.h.17.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 26275840 }, { "name": "transformer.h.17.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26296320 }, { "name": "transformer.h.17.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26301440 }, { "name": "transformer.h.17.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 26306560 }, { "name": "transformer.h.18.mixer.Wqkv.bias", "shape": [ 7680 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 15360, "byteOffset": 26311680 } ], "md5sum": "8dbc04f10dc464ba333178181f53cc86" }, { "dataPath": "params_shard_65.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.18.mlp.fc1.weight", "shape": [ 10240, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "cf04a8922daa9405df3bec9fa636137e" }, { "dataPath": "params_shard_66.bin", "format": "raw-shard", "nbytes": 52428800, "records": [ { "name": "transformer.h.18.mlp.fc2.weight", "shape": [ 2560, 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 52428800, "byteOffset": 0 } ], "md5sum": "ec8a0336b5359650f2d7475cf92aa9db" }, { "dataPath": "params_shard_67.bin", "format": "raw-shard", "nbytes": 262144000, "records": [ { "name": "lm_head.linear.weight", "shape": [ 51200, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144000, "byteOffset": 0 } ], "md5sum": "d975f6e26151b33f352a07426b0e0b04" }, { "dataPath": "params_shard_68.bin", "format": "raw-shard", "nbytes": 13260800, "records": [ { "name": "transformer.h.18.mixer.out_proj.weight", "shape": [ 2560, 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "transformer.h.18.mixer.out_proj.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13107200 }, { "name": "transformer.h.18.mlp.fc1.bias", "shape": [ 10240 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 20480, "byteOffset": 13112320 }, { "name": "transformer.h.18.mlp.fc2.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13132800 }, { "name": "transformer.h.18.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13137920 }, { "name": "transformer.h.18.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13143040 }, { "name": "lm_head.ln.weight", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13148160 }, { "name": "lm_head.ln.bias", "shape": [ 2560 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 5120, "byteOffset": 13153280 }, { "name": "lm_head.linear.bias", "shape": [ 51200 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 102400, "byteOffset": 13158400 } ], "md5sum": "807bde2f4922b71f7b70df2573b7e20f" } ] }