642 lines
10 KiB
JSON
642 lines
10 KiB
JSON
{
|
|
"__metadata__": null,
|
|
"mtp.fc_embedding.weight": {
|
|
"data_offsets": [
|
|
620910240,
|
|
634017440
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
2560,
|
|
2560
|
|
]
|
|
},
|
|
"mtp.fc_hidden.weight": {
|
|
"data_offsets": [
|
|
1179738016,
|
|
1192845216
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
2560,
|
|
2560
|
|
]
|
|
},
|
|
"mtp.hyper_connection_mixer.hc_norm.weight": {
|
|
"data_offsets": [
|
|
640571040,
|
|
640591520
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
10240
|
|
]
|
|
},
|
|
"mtp.hyper_connection_mixer.input_mix_weight_down.weight": {
|
|
"data_offsets": [
|
|
746319776,
|
|
752873376
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
320,
|
|
10240
|
|
]
|
|
},
|
|
"mtp.hyper_connection_mixer.input_mix_weight_up.weight": {
|
|
"data_offsets": [
|
|
614304160,
|
|
620857760
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
10240,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.attn_hyper_connection.block_inject_weight.weight": {
|
|
"data_offsets": [
|
|
80973824,
|
|
81055744
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
4,
|
|
10240
|
|
]
|
|
},
|
|
"mtp.layers.0.attn_hyper_connection.hc_norm.weight": {
|
|
"data_offsets": [
|
|
614283680,
|
|
614304160
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
10240
|
|
]
|
|
},
|
|
"mtp.layers.0.attn_hyper_connection.input_mix_weight_down.weight": {
|
|
"data_offsets": [
|
|
753754016,
|
|
760307616
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
320,
|
|
10240
|
|
]
|
|
},
|
|
"mtp.layers.0.attn_hyper_connection.input_mix_weight_up.weight": {
|
|
"data_offsets": [
|
|
634017440,
|
|
640571040
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
10240,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.gate.biases": {
|
|
"data_offsets": [
|
|
194699680,
|
|
194740640
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.gate.scales": {
|
|
"data_offsets": [
|
|
194740640,
|
|
194781600
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.gate.weight": {
|
|
"data_offsets": [
|
|
193388960,
|
|
194699680
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
512,
|
|
640
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.down_proj.biases": {
|
|
"data_offsets": [
|
|
140908960,
|
|
140960160
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
2560,
|
|
10
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.down_proj.scales": {
|
|
"data_offsets": [
|
|
614232480,
|
|
614283680
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
2560,
|
|
10
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.down_proj.weight": {
|
|
"data_offsets": [
|
|
640642720,
|
|
641461920
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
2560,
|
|
80
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.gate_proj.biases": {
|
|
"data_offsets": [
|
|
640591520,
|
|
640642720
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
640,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.gate_proj.scales": {
|
|
"data_offsets": [
|
|
19134464,
|
|
19185664
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
640,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.gate_proj.weight": {
|
|
"data_offsets": [
|
|
752934816,
|
|
753754016
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
640,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.up_proj.biases": {
|
|
"data_offsets": [
|
|
140038560,
|
|
140089760
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
640,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.up_proj.scales": {
|
|
"data_offsets": [
|
|
620857760,
|
|
620908960
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
640,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert.up_proj.weight": {
|
|
"data_offsets": [
|
|
140089760,
|
|
140908960
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
640,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert_gate.biases": {
|
|
"data_offsets": [
|
|
140038400,
|
|
140038480
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
1,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert_gate.scales": {
|
|
"data_offsets": [
|
|
140038480,
|
|
140038560
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
1,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.shared_expert_gate.weight": {
|
|
"data_offsets": [
|
|
620908960,
|
|
620910240
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
1,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.down_proj.biases": {
|
|
"data_offsets": [
|
|
140960160,
|
|
193388960
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
2560,
|
|
20
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.down_proj.scales": {
|
|
"data_offsets": [
|
|
641461920,
|
|
693890720
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
2560,
|
|
20
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.down_proj.weight": {
|
|
"data_offsets": [
|
|
1253138336,
|
|
1672568736
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
512,
|
|
2560,
|
|
80
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.gate_proj.biases": {
|
|
"data_offsets": [
|
|
693890976,
|
|
746319776
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
640,
|
|
80
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.gate_proj.scales": {
|
|
"data_offsets": [
|
|
20271104,
|
|
72699904
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
640,
|
|
80
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.gate_proj.weight": {
|
|
"data_offsets": [
|
|
760307616,
|
|
1179738016
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
512,
|
|
640,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.up_proj.biases": {
|
|
"data_offsets": [
|
|
87609600,
|
|
140038400
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
640,
|
|
80
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.up_proj.scales": {
|
|
"data_offsets": [
|
|
1192845216,
|
|
1245274016
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
640,
|
|
80
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp.switch_mlp.up_proj.weight": {
|
|
"data_offsets": [
|
|
194802080,
|
|
614232480
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
512,
|
|
640,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp_hyper_connection.block_inject_weight.weight": {
|
|
"data_offsets": [
|
|
74338304,
|
|
74420224
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
4,
|
|
10240
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp_hyper_connection.hc_norm.weight": {
|
|
"data_offsets": [
|
|
752914336,
|
|
752934816
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
10240
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp_hyper_connection.input_mix_weight_down.weight": {
|
|
"data_offsets": [
|
|
81055744,
|
|
87609344
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
320,
|
|
10240
|
|
]
|
|
},
|
|
"mtp.layers.0.mlp_hyper_connection.input_mix_weight_up.weight": {
|
|
"data_offsets": [
|
|
74420224,
|
|
80973824
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
10240,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.indexer.index_qk_proj.biases": {
|
|
"data_offsets": [
|
|
19185664,
|
|
19236864
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
640,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.indexer.index_qk_proj.scales": {
|
|
"data_offsets": [
|
|
19236864,
|
|
19288064
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
640,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.indexer.index_qk_proj.weight": {
|
|
"data_offsets": [
|
|
72699904,
|
|
74338304
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
640,
|
|
640
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.indexer.k_layernorm.weight": {
|
|
"data_offsets": [
|
|
693890720,
|
|
693890976
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
128
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.indexer.q_layernorm.weight": {
|
|
"data_offsets": [
|
|
87609344,
|
|
87609600
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
128
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.k_norm.weight": {
|
|
"data_offsets": [
|
|
19133952,
|
|
19134464
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
256
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.k_proj.biases": {
|
|
"data_offsets": [
|
|
752873376,
|
|
752914336
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.k_proj.scales": {
|
|
"data_offsets": [
|
|
18396672,
|
|
18437632
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.k_proj.weight": {
|
|
"data_offsets": [
|
|
18437632,
|
|
19092992
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
512,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.o_proj.biases": {
|
|
"data_offsets": [
|
|
17413632,
|
|
17905152
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
2560,
|
|
96
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.o_proj.scales": {
|
|
"data_offsets": [
|
|
17905152,
|
|
18396672
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
2560,
|
|
96
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.o_proj.weight": {
|
|
"data_offsets": [
|
|
1245274016,
|
|
1253138336
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
2560,
|
|
768
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.q_norm.weight": {
|
|
"data_offsets": [
|
|
17413120,
|
|
17413632
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
256
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.q_proj.biases": {
|
|
"data_offsets": [
|
|
701440,
|
|
1684480
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
12288,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.q_proj.scales": {
|
|
"data_offsets": [
|
|
19288064,
|
|
20271104
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
12288,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.q_proj.weight": {
|
|
"data_offsets": [
|
|
1684480,
|
|
17413120
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
12288,
|
|
320
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.v_proj.biases": {
|
|
"data_offsets": [
|
|
19092992,
|
|
19133952
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.v_proj.scales": {
|
|
"data_offsets": [
|
|
5120,
|
|
46080
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
512,
|
|
40
|
|
]
|
|
},
|
|
"mtp.layers.0.self_attn.v_proj.weight": {
|
|
"data_offsets": [
|
|
46080,
|
|
701440
|
|
],
|
|
"dtype": "U32",
|
|
"shape": [
|
|
512,
|
|
320
|
|
]
|
|
},
|
|
"mtp.pre_fc_norm_embedding.weight": {
|
|
"data_offsets": [
|
|
0,
|
|
5120
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
2560
|
|
]
|
|
},
|
|
"mtp.pre_fc_norm_hidden.weight": {
|
|
"data_offsets": [
|
|
194781600,
|
|
194802080
|
|
],
|
|
"dtype": "BF16",
|
|
"shape": [
|
|
10240
|
|
]
|
|
}
|
|
}
|