file name dtype shape quant_bits group_size quant_mode data_start data_end model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3801091950 3827306350 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2018512750 2044727150 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736750 4666167150 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 498077550 524291950 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 524291950 550506350 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3381661550 3801091950 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2490371950 2516586350 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1022365550 1048579950 model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2044727150 2464157550 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1520439150 1546653550 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2464157550 2490371950 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1599082350 2018512750 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1572867950 1599082350 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1494224750 1520439150 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2962231150 3381661550 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2936016750 2962231150 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1048579950 1074794350 model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1074794350 1494224750 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1546653550 1572867950 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 576720750 602935150 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3827306350 4246736750 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 471863150 498077550 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 550506350 576720750 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2516586350 2936016750 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 419434350 445648750 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 445648750 471863150 model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3950 419434350 model-00001-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 602935150 1022365550 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4272951312 4299165712 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3224375312 3250589712 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3303018512 3329232912 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3355447312 3381661712 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1389367312 1808797712 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1835012112 1861226512 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2359300112 2385514512 model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3853520912 4272951312 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3250589712 3276804112 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3329232912 3355447312 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2385514512 2804944912 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3276804112 3303018512 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2333085712 2359300112 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1861226512 2280656912 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1808797712 1835012112 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2306871312 2333085712 model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3434090512 3853520912 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3381661712 3407876112 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3407876112 3434090512 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 524292112 943722512 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2280656912 2306871312 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 498077712 524292112 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 969936912 1389367312 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 943722512 969936912 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 471863312 498077712 model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2804944912 3224375312 model-00002-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 445648912 471863312 model-00002-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4112 26218512 model-00002-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 26218512 445648912 model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1992298368 2018512768 model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2883587968 2909802368 model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 4246736768 4666167168 model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1415581568 1441795968 model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1939869568 1966083968 model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 498077568 917507968 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2857373568 2883587968 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194307968 4220522368 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1520439168 1939869568 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1441795968 1468010368 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1468010368 1494224768 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3355447168 3774877568 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1966083968 1992298368 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1494224768 1520439168 model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2018512768 2437943168 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 917507968 943722368 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1389367168 1415581568 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 943722368 1363152768 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1363152768 1389367168 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4220522368 4246736768 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3774877568 4194307968 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 26218368 52432768 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 471863168 498077568 model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2936016768 3355447168 model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2909802368 2936016768 model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3968 26218368 model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 52432768 471863168 model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2437943168 2857373568 model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3722448902 3748663302 model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4246736902 4272951302 model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2778730502 2804944902 model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4220522502 4246736902 model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2804944902 3224375302 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2306871302 2333085702 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4168093702 4194308102 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3748663302 4168093702 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3224375302 3250589702 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3250589702 3276804102 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3276804102 3696234502 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2280656902 2306871302 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 943722502 969936902 model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2333085702 2752516102 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4272951302 4299165702 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1835012102 1861226502 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1861226502 2280656902 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 891293702 917508102 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 917508102 943722502 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 996151302 1415581702 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 865079302 891293702 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 969936902 996151302 model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1415581702 1835012102 model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2752516102 2778730502 model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194308102 4220522502 model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 445648902 865079302 model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3696234502 3722448902 model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4102 26218502 model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 26218502 445648902 model-00005-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3748663169 3774877569 model-00005-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2831159169 2857373569 model-00005-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3774877569 4194307969 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4639952769 4666167169 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4613738369 4639952769 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2359299969 2778730369 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2857373569 2883587969 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3303018369 3329232769 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 917507969 1336938369 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2778730369 2804944769 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1441795969 1468010369 model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2883587969 3303018369 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1389367169 1415581569 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1468010369 1494224769 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1913655169 2333085569 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1415581569 1441795969 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2804944769 2831159169 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3329232769 3748663169 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1336938369 1363152769 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1363152769 1389367169 model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 4194307969 4613738369 model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 891293569 917507969 model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2333085569 2359299969 model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1494224769 1913655169 model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 419434369 445648769 model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 445648769 471863169 model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 471863169 891293569 model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3969 419434369 model-00006-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4246736908 4272951308 model-00006-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4272951308 4299165708 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3774877708 3801092108 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3303018508 3329232908 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3827306508 4246736908 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1363152908 1389367308 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2673872908 2700087308 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2254442508 2673872908 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2857373708 2883588108 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2700087308 2726301708 model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1835012108 2254442508 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1808797708 1835012108 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 445648908 471863308 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1389367308 1808797708 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2804944908 2831159308 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2831159308 2857373708 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2883588108 3303018508 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 917508108 943722508 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2752516108 2778730508 model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 943722508 1363152908 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 471863308 498077708 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3748663308 3774877708 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 498077708 917508108 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730508 2804944908 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2726301708 2752516108 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3329232908 3748663308 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3801092108 3827306508 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4108 26218508 model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 26218508 445648908 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3774877568 3801091968 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4220522368 4246736768 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736768 4666167168 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3722448768 3748663168 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3696234368 3722448768 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3801091968 4220522368 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3250589568 3276803968 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2752515968 2778730368 model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1363152768 1782583168 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2726301568 2752515968 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2804944768 2831159168 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2831159168 3250589568 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2700087168 2726301568 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2254442368 2280656768 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2280656768 2700087168 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1782583168 1808797568 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 471863168 498077568 model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1808797568 2228227968 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2228227968 2254442368 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 917507968 943722368 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 943722368 1363152768 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3748663168 3774877568 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2778730368 2804944768 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 498077568 917507968 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3968 26218368 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 26218368 52432768 model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 52432768 471863168 model-00007-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3276803968 3696234368 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3696234497 3722448897 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2359300097 2385514497 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 498077697 524292097 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4246736897 4272951297 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2385514497 2804944897 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 550506497 576720897 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3722448897 3748663297 model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3748663297 4168093697 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3224375297 3250589697 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3670020097 3696234497 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1939869697 2359300097 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4220522497 4246736897 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1913655297 1939869697 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 602935297 1022365697 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4272951297 4299165697 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1441796097 1468010497 model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3250589697 3670020097 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1468010497 1494224897 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194308097 4220522497 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1494224897 1913655297 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 524292097 550506497 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 576720897 602935297 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1022365697 1441796097 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 26218497 52432897 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 52432897 78647297 model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 78647297 498077697 model-00008-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4097 26218497 model-00008-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4168093697 4194308097 model-00008-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2804944897 3224375297 model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4613738362 4639952762 model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1468010362 1494224762 model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3303018362 3722448762 model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3250589562 3276803962 model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4587523962 4613738362 model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3722448762 4141879162 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2385514362 2411728762 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4561309562 4587523962 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2411728762 2831159162 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3276803962 3303018362 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4639952762 4666167162 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1939869562 2359299962 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1494224762 1520439162 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 524291962 550506362 model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1520439162 1939869562 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2359299962 2385514362 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 996151162 1022365562 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1022365562 1441795962 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1441795962 1468010362 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 498077562 524291962 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 550506362 969936762 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 969936762 996151162 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 52432762 78647162 model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 78647162 498077562 model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3962 26218362 model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 26218362 52432762 model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2831159162 3250589562 model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 4141879162 4561309562 model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2437943308 2464157708 model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1861226508 1887440908 model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3407876108 3434090508 model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3381661708 3407876108 model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2962231308 3381661708 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2516586508 2542800908 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2464157708 2490372108 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2542800908 2962231308 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2385514508 2411728908 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1913655308 1939869708 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3853520908 4272951308 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2411728908 2437943308 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1887440908 1913655308 model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1966084108 2385514508 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2490372108 2516586508 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1939869708 1966084108 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3434090508 3853520908 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 943722508 969936908 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1415581708 1441796108 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1441796108 1861226508 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4272951308 4299165708 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 969936908 996151308 model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 524292108 943722508 model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 445648908 471863308 model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 471863308 498077708 model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 996151308 1415581708 model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 498077708 524292108 model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4108 26218508 model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 26218508 445648908 model-00011-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 498077559 524291959 model-00011-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4220522359 4246736759 model-00011-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 4246736759 4666167159 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4168093559 4194307959 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3224375159 3250589559 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1835011959 2254442359 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4194307959 4220522359 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3670019959 3696234359 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1363152759 1782583159 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2726301559 2752515959 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2752515959 2778730359 model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3250589559 3670019959 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1808797559 1835011959 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2700087159 2726301559 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3748663159 4168093559 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3696234359 3722448759 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3722448759 3748663159 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 943722359 1363152759 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 471863159 498077559 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3198160759 3224375159 model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 524291959 943722359 model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2673872759 2700087159 model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1782583159 1808797559 model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2254442359 2673872759 model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3959 26218359 model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 26218359 52432759 model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 52432759 471863159 model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2778730359 3198160759 model-00012-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4246736906 4272951306 model-00012-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4272951306 4299165706 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1861226506 1887440906 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3696234506 3722448906 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3801092106 4220522506 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730506 2804944906 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3748663306 3774877706 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2831159306 3250589706 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2333085706 2359300106 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3722448906 3748663306 model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1363152906 1782583306 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2306871306 2333085706 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1782583306 1808797706 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2359300106 2778730506 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 917508106 943722506 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1835012106 1861226506 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1887440906 2306871306 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3670020106 3696234506 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1808797706 1835012106 model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 943722506 1363152906 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 471863306 498077706 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2804944906 2831159306 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 498077706 917508106 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 445648906 471863306 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3774877706 3801092106 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3250589706 3670020106 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4220522506 4246736906 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4106 26218506 model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 26218506 445648906 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3801091974 3827306374 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3774877574 3801091974 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736774 4666167174 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3093303174 3119517574 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3748663174 3774877574 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1782583174 2202013574 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3224375174 3250589574 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3722448774 3748663174 model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3827306374 4246736774 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3119517574 3145731974 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3198160774 3224375174 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3250589574 3670019974 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3171946374 3198160774 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3696234374 3722448774 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2673872774 3093303174 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3145731974 3171946374 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2202013574 2228227974 model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2254442374 2673872774 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3670019974 3696234374 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1756368774 1782583174 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1336938374 1756368774 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 865079174 891293574 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 891293574 917507974 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 917507974 1336938374 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2228227974 2254442374 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 419434374 445648774 model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 445648774 865079174 model-00013-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3974 419434374 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1415581704 1441796104 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1441796104 1468010504 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4168093704 4194308104 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4272951304 4299165704 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3250589704 3670020104 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1913655304 1939869704 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2804944904 2831159304 model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3696234504 4115664904 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4220522504 4246736904 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194308104 4220522504 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2831159304 3250589704 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730504 2804944904 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1468010504 1494224904 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2359300104 2778730504 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1389367304 1415581704 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4141879304 4168093704 model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1939869704 2359300104 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3670020104 3696234504 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 917508104 943722504 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 943722504 1363152904 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4115664904 4141879304 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 865079304 891293704 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1494224904 1913655304 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4246736904 4272951304 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 419434504 445648904 model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 445648904 865079304 model-00014-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1363152904 1389367304 model-00014-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 891293704 917508104 model-00014-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4104 419434504 model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730346 2804944746 model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4220522346 4246736746 model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2359299946 2778730346 model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2831159146 2857373546 model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2857373546 2883587946 model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3774877546 4194307946 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2333085546 2359299946 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2804944746 2831159146 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736746 4666167146 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2280656746 2306871146 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2306871146 2333085546 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3355447146 3774877546 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2883587946 2909802346 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2909802346 2936016746 model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1861226346 2280656746 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1336938346 1363152746 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1389367146 1415581546 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1415581546 1835011946 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4194307946 4220522346 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 891293546 917507946 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 917507946 1336938346 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 419434346 445648746 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1363152746 1389367146 model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 471863146 891293546 model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1835011946 1861226346 model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 445648746 471863146 model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3946 419434346 model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2936016746 3355447146 model-00016-of-00017.safetensors language_model.lm_head.biases BF16 248320x40 4 64 affine 1613106348 1632971948 model-00016-of-00017.safetensors language_model.lm_head.scales BF16 248320x40 4 64 affine 858405156 878270756 model-00016-of-00017.safetensors language_model.lm_head.weight U32 248320x320 4 64 affine 3127426076 3445275676 model-00016-of-00017.safetensors language_model.model.embed_tokens.biases BF16 248320x40 4 64 affine 2582830796 2602696396 model-00016-of-00017.safetensors language_model.model.embed_tokens.scales BF16 248320x40 4 64 affine 2454382332 2474247932 model-00016-of-00017.safetensors language_model.model.embed_tokens.weight U32 248320x320 4 64 affine 2626387660 2944237260 model-00016-of-00017.safetensors language_model.model.hyper_connection_mixer.hc_norm.weight BF16 10240 2035595164 2035615644 model-00016-of-00017.safetensors language_model.model.hyper_connection_mixer.input_mix_weight_down.weight BF16 320x10240 3490183452 3496737052 model-00016-of-00017.safetensors language_model.model.hyper_connection_mixer.input_mix_weight_up.weight BF16 10240x320 2944278220 2950831820 model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2339375180 2339457100 model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.hc_norm.weight BF16 10240 1232367948 1232388428 model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2523639660 2530193260 model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 4256296284 4262849884 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.A_log BF16 48 3796949756 3796949852 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.conv1d.weight BF16 10240x4x1 3752913324 3752995244 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.dt_bias BF16 48 443286612 443286708 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2300871420 2300875260 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2626301900 2626305740 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 4282602844 4282664284 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2427187372 2427191212 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1601261132 1601264972 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1116399332 1116460772 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 3572520044 3573339244 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3452320796 3453139996 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1019912580 1033019780 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3806995804 3807487324 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3775840764 3776332284 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3788204796 3796069116 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.norm.weight BF16 128 3786480380 3786480636 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3527216764 3527708284 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3620229836 3620721356 model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3462530332 3470394652 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.gate.biases BF16 512x40 8 64 affine 1757777772 1757818732 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.gate.scales BF16 512x40 8 64 affine 3526131324 3526172284 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.gate.weight U32 512x640 8 64 affine 1164862540 1166173260 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2359992396 2360043596 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2950831820 2950883020 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2103063724 2103882924 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3573380204 3573431404 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3554930284 3554981484 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2482564092 2483383292 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3084938860 3084990060 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2261224540 2261275740 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3001640636 3002459836 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1540541020 1540541100 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2986535276 2986535356 model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1581191516 1581192796 model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2626168780 2626250700 model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.hc_norm.weight BF16 10240 3752995244 3753015724 model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1719344492 1725898092 model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 824597204 831150804 model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3554797164 3554879084 model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.hc_norm.weight BF16 10240 1405324908 1405345388 model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2514679564 2521233164 model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 4262942044 4269495644 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.A_log BF16 48 3522758428 3522758524 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.conv1d.weight BF16 10240x4x1 2060499532 2060581452 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.dt_bias BF16 48 2252240124 2252240220 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 677817204 677821044 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 602640484 602644324 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3725482396 3725543836 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1813155004 1813158844 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1224938492 1224942332 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3638335564 3638397004 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1813158844 1813978044 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3726414236 3727233436 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1890223420 1903330620 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3668936604 3669428124 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3018970812 3019462332 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1697335324 1705199644 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.norm.weight BF16 128 3685207964 3685208220 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3094277820 3094769340 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3636880204 3637371724 model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3646982044 3654846364 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.gate.biases BF16 512x40 8 64 affine 2123694764 2123735724 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.gate.scales BF16 512x40 8 64 affine 2513768204 2513809164 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.gate.weight U32 512x640 8 64 affine 3019462332 3020773052 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 640473604 640524804 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3554879084 3554930284 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3588658284 3589477484 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3096407740 3096458940 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3638417484 3638468684 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3796130556 3796949756 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3796949852 3797001052 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3646879644 3646930844 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3751479468 3752298668 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3645026124 3645026204 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1778575212 1778575292 model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3610664396 3610665676 model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2044137036 2044218956 model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.hc_norm.weight BF16 10240 1928071996 1928092476 model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1928645436 1935199036 model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3483585052 3490138652 model-00016-of-00017.safetensors language_model.model.layers.1.ple.conv_weight BF16 10240x4x1 1817797660 1817879580 model-00016-of-00017.safetensors language_model.model.layers.1.ple.key_proj.weight BF16 10240x2560 755216516 807645316 model-00016-of-00017.safetensors language_model.model.layers.1.ple.norm_conv.weight BF16 10240 3637443404 3637463884 model-00016-of-00017.safetensors language_model.model.layers.1.ple.norm_key.weight BF16 10240 3637371724 3637392204 model-00016-of-00017.safetensors language_model.model.layers.1.ple.norm_query.weight BF16 10240 1232388428 1232408908 model-00016-of-00017.safetensors language_model.model.layers.1.ple.ple_embedding.layer_multipliers I64 3 1160070020 1160070044 model-00016-of-00017.safetensors language_model.model.layers.1.ple.ple_embedding.ngram_heads_offsets I64 16 3620721356 3620721484 model-00016-of-00017.safetensors language_model.model.layers.1.ple.ple_embedding.ngram_heads_vocab_sizes I64 16 1446977132 1446977260 model-00016-of-00017.safetensors language_model.model.layers.1.ple.value_proj.weight BF16 2560x2560 973500548 986607748 model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2298382940 2298464860 model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.hc_norm.weight BF16 10240 1373177356 1373197836 model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3516204828 3522758428 model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2156763564 2163317164 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.A_log BF16 48 3602779244 3602779340 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.conv1d.weight BF16 10240x4x1 2036305564 2036387484 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.dt_bias BF16 48 1883669724 1883669820 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1263845868 1263849708 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 3496818972 3496822812 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3016523196 3016584636 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2077162396 2077166236 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3588654444 3588658284 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2214565724 2214627164 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 927303716 928122916 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3556296044 3557115244 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3575451244 3588558444 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 727544100 728035620 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3572028524 3572520044 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2498338812 2506203132 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.norm.weight BF16 128 1601260876 1601261132 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1707603484 1708095004 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3554985324 3555476844 model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3557606764 3565471084 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.gate.biases BF16 512x40 8 64 affine 2085854108 2085895068 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.gate.scales BF16 512x40 8 64 affine 2328807500 2328848460 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.gate.weight U32 512x640 8 64 affine 3067464972 3068775692 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3524687484 3524738684 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2267931740 2267982940 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1804293564 1805112764 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2314205260 2314256460 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2238713004 2238764204 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3513575708 3514394908 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3638284364 3638335564 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3453795356 3453846556 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3610727116 3611546316 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3453846556 3453846636 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2067206732 2067206812 model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2603751788 2603753068 model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3483503132 3483585052 model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.hc_norm.weight BF16 10240 1601148156 1601168636 model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3052592572 3059146172 model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3453846636 3460400236 model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3523680124 3523762044 model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.hc_norm.weight BF16 10240 3453631516 3453651996 model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 669184276 675737876 model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3085809260 3092362860 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.gate.biases BF16 512x40 8 64 affine 3031498172 3031539132 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.gate.scales BF16 512x40 8 64 affine 2522738540 2522779500 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.gate.weight U32 512x640 8 64 affine 3524738684 3526049404 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3725543836 3725595036 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3453651996 3453703196 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2411130956 2411950156 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2483465212 2483516412 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2043819164 2043870364 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1357211228 1358030428 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3453703196 3453754396 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1750216812 1750268012 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3125787676 3126606876 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 4300667164 4300667244 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2279574972 2279575052 model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2052452172 2052453452 model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2522103660 2522185580 model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.hc_norm.weight BF16 10240 3638397004 3638417484 model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3077346924 3083900524 model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3111000604 3117554204 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1633555724 1633606924 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 3092444860 3092496060 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 3092639420 3094277820 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.k_layernorm.weight BF16 128 3084887404 3084887660 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.q_layernorm.weight BF16 128 3806176092 3806176348 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_norm.weight BF16 256 4291931484 4291931996 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_proj.biases BF16 512x40 4 64 affine 3453754396 3453795356 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_proj.scales BF16 512x40 4 64 affine 1718482956 1718523916 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_proj.weight U32 512x320 4 64 affine 1778575292 1779230652 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.o_proj.biases BF16 2560x96 4 64 affine 912390436 912881956 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.o_proj.scales BF16 2560x96 4 64 affine 3453139996 3453631516 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.o_proj.weight U32 2560x768 4 64 affine 2411950156 2419814476 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_norm.weight BF16 256 1224917500 1224918012 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_proj.biases BF16 12288x40 4 64 affine 3645896604 3646879644 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1497980236 1498963276 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_proj.weight U32 12288x320 4 64 affine 1126633620 1142362260 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.v_proj.biases BF16 512x40 4 64 affine 3052039612 3052080572 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.v_proj.scales BF16 512x40 4 64 affine 1117283812 1117324772 model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.v_proj.weight U32 512x320 4 64 affine 3069482252 3070137612 model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3536443084 3536525004 model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.hc_norm.weight BF16 10240 3038092732 3038113212 model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3031539132 3038092732 model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3118373404 3124927004 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.A_log BF16 48 3462530236 3462530332 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.conv1d.weight BF16 10240x4x1 1865977292 1866059212 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.dt_bias BF16 48 4300667068 4300667164 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3030982332 3030986172 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1883317644 1883321484 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 928122916 928184356 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3645022284 3645026124 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 508816404 508820244 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1928092476 1928153916 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1296138396 1296957596 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1084405028 1085224228 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 408606132 421713332 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3022626492 3023118012 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3068837132 3069328652 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3023118012 3030982332 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.norm.weight BF16 128 1098433828 1098434084 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3797001052 3797492572 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3022134972 3022626492 model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3621561164 3629425484 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.gate.biases BF16 512x40 8 64 affine 3018929852 3018970812 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.gate.scales BF16 512x40 8 64 affine 1601025276 1601066236 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.gate.weight U32 512x640 8 64 affine 3020773052 3022083772 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 4300779884 4300831084 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3536391884 3536443084 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3018110396 3018929596 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3016471996 3016523196 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 703831924 703883124 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2052576332 2053395532 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1708765036 1708816236 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3001589436 3001640636 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3002459836 3003279036 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2978588956 2978589036 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2252199084 2252199164 model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3000707516 3000708796 model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1581196636 1581278556 model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.hc_norm.weight BF16 10240 2950883020 2950903500 model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2612242124 2618795724 model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3060849932 3067403532 model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3522779004 3522860924 model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.hc_norm.weight BF16 10240 2043942044 2043962524 model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2604869324 2611422924 model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1336075772 1342629372 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.A_log BF16 48 2603804268 2603804364 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.conv1d.weight BF16 10240x4x1 2602696988 2602778908 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.dt_bias BF16 48 675758356 675758452 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2507210588 2507214428 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2538159980 2538163820 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 2566999756 2567061196 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3588650604 3588654444 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1486922316 1486926156 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1682179852 1682241292 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 483632308 484451508 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2513809164 2514628364 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2339512140 2352619340 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2522185580 2522677100 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 675809652 676301172 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 443286708 451151028 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.norm.weight BF16 128 849588100 849588356 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2328899660 2329391180 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2053446732 2053938252 model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1866601932 1874466252 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.gate.biases BF16 512x40 8 64 affine 928184356 928225316 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.gate.scales BF16 512x40 8 64 affine 2507169532 2507210492 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.gate.weight U32 512x640 8 64 affine 2275673532 2276984252 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2507077372 2507128572 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1733018732 1733069932 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1117857252 1118676452 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3554694764 3554745964 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2507026172 2507077372 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3629507404 3630326604 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1848016156 1848067356 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2496664572 2496715772 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2506206972 2507026172 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1242874188 1242874268 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3059146172 3059146252 model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2481702652 2481703932 model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2626305740 2626387660 model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.hc_norm.weight BF16 10240 2296671324 2296691804 model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2474329852 2480883452 model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3638468684 3645022284 model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2474247932 2474329852 model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.hc_norm.weight BF16 10240 3060788492 3060808972 model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2447828732 2454382332 model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2441275132 2447828732 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.A_log BF16 48 1874466252 1874466348 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.conv1d.weight BF16 10240x4x1 3000625596 3000707516 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.dt_bias BF16 48 3111000508 3111000604 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1495590476 1495594316 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2053938252 2053942092 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3077183084 3077244524 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2506203132 2506206972 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2427191212 2427195052 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3526663804 3526725244 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1963318076 1964137276 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 676403572 677222772 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1373453836 1386561036 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2603845324 2604336844 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 839016484 839508004 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1396641388 1404505708 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.norm.weight BF16 128 1789433788 1789434044 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2395566156 2396057676 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 4298485948 4298977468 model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2402611276 2410475596 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.gate.biases BF16 512x40 8 64 affine 1447028460 1447069420 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.gate.scales BF16 512x40 8 64 affine 3554653804 3554694764 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.gate.weight U32 512x640 8 64 affine 2394173516 2395484236 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2336763980 2336815180 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2339457100 2339508300 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2352619340 2353438540 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2036250524 2036301724 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2328848460 2328899660 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2433799852 2434619052 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2314256460 2314307660 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2279032252 2279083452 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3461711036 3462530236 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2314201340 2314201420 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1077024468 1077024548 model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2497374972 2497376252 model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2314043900 2314125820 model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.hc_norm.weight BF16 10240 2300916220 2300936700 model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2434619052 2441172652 model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2269078972 2275632572 model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2483383292 2483465212 model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.hc_norm.weight BF16 10240 3620721484 3620741964 model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1211676844 1218230444 model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2320861260 2327414860 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.gate.biases BF16 512x40 8 64 affine 2214524764 2214565724 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.gate.scales BF16 512x40 8 64 affine 2044096076 2044137036 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.gate.weight U32 512x640 8 64 affine 846925604 848236324 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2951012636 2951063836 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2367907916 2367959116 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1847196956 1848016156 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1978882876 1978934076 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2299929180 2299980380 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2299980380 2300799580 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 703886964 703938164 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3645845404 3645896604 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2296744284 2297563484 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2295800844 2295800924 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2950960076 2950960156 model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2295799564 2295800844 model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3526049404 3526131324 model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.hc_norm.weight BF16 10240 2060581452 2060601932 model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3445275676 3451829276 model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2490070012 2496623612 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1812977084 1813028284 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 2603753068 2603804268 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 2294161164 2295799564 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.k_layernorm.weight BF16 128 2102192556 2102192812 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.q_layernorm.weight BF16 128 2268904540 2268904796 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_norm.weight BF16 256 2281049612 2281050124 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_proj.biases BF16 512x40 4 64 affine 2300875260 2300916220 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_proj.scales BF16 512x40 4 64 affine 2298516060 2298557020 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_proj.weight U32 512x320 4 64 affine 2279575052 2280230412 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.o_proj.biases BF16 2560x96 4 64 affine 2279083452 2279574972 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.o_proj.scales BF16 2560x96 4 64 affine 2987026876 2987518396 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.o_proj.weight U32 2560x768 4 64 affine 3602779340 3610643660 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_norm.weight BF16 256 1263952108 1263952620 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_proj.biases BF16 12288x40 4 64 affine 2276984252 2277967292 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_proj.scales BF16 12288x40 4 64 affine 2277967292 2278950332 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_proj.weight U32 12288x320 4 64 affine 1759999852 1775728492 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.v_proj.biases BF16 512x40 4 64 affine 2035554204 2035595164 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.v_proj.scales BF16 512x40 4 64 affine 2275632572 2275673532 model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.v_proj.weight U32 512x320 4 64 affine 857606436 858261796 model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2268904796 2268986716 model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.hc_norm.weight BF16 10240 2156743084 2156763564 model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 4269495644 4276049244 model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2618795724 2625349324 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.A_log BF16 48 1682353932 1682354028 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.conv1d.weight BF16 10240x4x1 2238764204 2238846124 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.dt_bias BF16 48 2507210492 2507210588 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2036387484 2036391324 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1060255956 1060259796 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3092383420 3092444860 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2261220700 2261224540 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1707599644 1707603484 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2267982940 2268044380 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1536306524 1537125724 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2611422924 2612242124 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 729858340 742965540 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 593842788 594334308 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3070137612 3070629132 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2243024044 2250888364 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.norm.weight BF16 128 3103033020 3103033276 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2240310444 2240801964 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2242532524 2243024044 model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3760060844 3767925164 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.gate.biases BF16 512x40 8 64 affine 4262901084 4262942044 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.gate.scales BF16 512x40 8 64 affine 1750123372 1750164332 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.gate.weight U32 512x640 8 64 affine 2238999724 2240310444 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3536606924 3536658124 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2238897324 2238948524 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 718736340 719555540 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1160070044 1160121244 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1540868956 1540920156 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3059150092 3059969292 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2101136476 2101187676 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2060601932 2060653132 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2480883452 2481702652 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2214627164 2214627244 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2300871260 2300871340 model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2238527404 2238528684 model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 625728004 625809924 model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.hc_norm.weight BF16 10240 1540848476 1540868956 model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2206660444 2213214044 model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3685208220 3691761820 model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2185533964 2185615884 model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.hc_norm.weight BF16 10240 1883403404 1883423884 model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2178929164 2185482764 model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 952252292 958805892 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.A_log BF16 48 2513768108 2513768204 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.conv1d.weight BF16 10240x4x1 1757644652 1757726572 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.dt_bias BF16 48 2170019756 2170019852 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2170012076 2170015916 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2170015916 2170019756 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3001527996 3001589436 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2163317164 2163321004 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2163454636 2163458476 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2298557020 2298618460 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2497437692 2498256892 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1305804956 1306624156 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1979896732 1993003932 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1911755836 1912247356 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1717991436 1718482956 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1839076636 1846940956 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.norm.weight BF16 128 3125787420 3125787676 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2140908204 2141399724 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1663050940 1663542460 model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2141420204 2149284524 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.gate.biases BF16 512x40 8 64 affine 2139556524 2139597484 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.gate.scales BF16 512x40 8 64 affine 1903330620 1903371580 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.gate.weight U32 512x640 8 64 affine 2139597484 2140908204 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3727233516 3727284716 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1429772908 1429824108 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3117554204 3118373404 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2103882924 2103934124 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 958887812 958939012 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1749130092 1749949292 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3646930844 3646982044 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2102193324 2102244524 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2102244524 2103063724 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2102191196 2102191276 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1218363820 1218363900 model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2102191276 2102192556 model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2102109276 2102191196 model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.hc_norm.weight BF16 10240 1935199036 1935219516 model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2094531676 2101085276 model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2330210380 2336763980 model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2044013980 2044095900 model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.hc_norm.weight BF16 10240 2141399724 2141420204 model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2103934124 2110487724 model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2087978076 2094531676 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.A_log BF16 48 2087424764 2087424860 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.conv1d.weight BF16 10240x4x1 678365044 678446964 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.dt_bias BF16 48 2086718108 2086718204 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3059146252 3059150092 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2085850268 2085854108 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1487008076 1487069516 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1787740348 1787744188 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2053942092 2053945932 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2522677100 2522738540 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2077166236 2077985436 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3555476844 3556296044 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2957617436 2970724636 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2076056476 2076547996 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2076588956 2077080476 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3527708364 3535572684 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.norm.weight BF16 128 602640228 602640484 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2075564956 2076056476 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2178437644 2178929164 model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1573220956 1581085276 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.gate.biases BF16 512x40 8 64 affine 2075441980 2075482940 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.gate.scales BF16 512x40 8 64 affine 1750031212 1750072172 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.gate.weight U32 512x640 8 64 affine 3797492572 3798803292 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1373310476 1373361676 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2075390780 2075441980 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3785558524 3786377724 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2117045164 2117096364 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1866059212 1866110412 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2074571580 2075390780 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2950908876 2950960076 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2067393340 2067444540 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1740938092 1741757292 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2067391388 2067391468 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2067391980 2067392060 model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2067392060 2067393340 model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2067309212 2067391132 model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.hc_norm.weight BF16 10240 2067288732 2067309212 model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 553066772 559620372 model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1956764476 1963318076 model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2067206812 2067288732 model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.hc_norm.weight BF16 10240 3610643660 3610664140 model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2060653132 2067206732 model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2053945932 2060499532 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.gate.biases BF16 512x40 8 64 affine 719637460 719678420 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.gate.scales BF16 512x40 8 64 affine 2052535372 2052576332 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.gate.weight U32 512x640 8 64 affine 742965540 744276260 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2626250700 2626301900 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2042944924 2042996124 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3552437964 3553257164 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3725431196 3725482396 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1787689148 1787740348 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3645026204 3645845404 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2101085276 2101136476 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2036199324 2036250524 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2051632972 2052452172 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3092362860 3092362940 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2044095900 2044095980 model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2025394588 2025395868 model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1358163724 1358245644 model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.hc_norm.weight BF16 10240 2043921564 2043942044 model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1751091052 1757644652 model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2036391324 2042944924 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 662538516 662589716 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 2035615644 2035666844 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 4299028668 4300667068 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.k_layernorm.weight BF16 128 2034734748 2034735004 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.q_layernorm.weight BF16 128 1112856036 1112856292 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_norm.weight BF16 256 509886484 509886996 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_proj.biases BF16 512x40 4 64 affine 2012154268 2012195228 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_proj.scales BF16 512x40 4 64 affine 2025353628 2025394588 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_proj.weight U32 512x320 4 64 affine 2033260188 2033915548 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.o_proj.biases BF16 2560x96 4 64 affine 2011054492 2011546012 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.o_proj.scales BF16 2560x96 4 64 affine 2011662668 2012154188 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.o_proj.weight U32 2560x768 4 64 affine 2000855452 2008719772 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_norm.weight BF16 256 2067391468 2067391980 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_proj.biases BF16 12288x40 4 64 affine 3654846364 3655829404 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_proj.scales BF16 12288x40 4 64 affine 2009252252 2010235292 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_proj.weight U32 12288x320 4 64 affine 2567061196 2582789836 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.v_proj.biases BF16 512x40 4 64 affine 2009211292 2009252252 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.v_proj.scales BF16 512x40 4 64 affine 3060808972 3060849932 model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.v_proj.weight U32 512x320 4 64 affine 624967604 625622964 model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2498256892 2498338812 model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.hc_norm.weight BF16 10240 1552720476 1552740956 model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1941855036 1948408636 model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1994226332 2000779932 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.A_log BF16 48 4298485596 4298485692 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.conv1d.weight BF16 10240x4x1 3553257324 3553339244 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.dt_bias BF16 48 649208404 649208500 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1927576636 1927580476 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1601000956 1601004796 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1994164892 1994226332 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2000779932 2000783772 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1993495452 1993499292 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1993611932 1993673372 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 617574244 618393444 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1705199644 1706018844 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1965775676 1978882876 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1928153916 1928645436 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1334996636 1335488156 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1948900156 1956764476 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.norm.weight BF16 128 1053056900 1053057156 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1680869132 1681360652 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1927580476 1928071996 model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2170573324 2178437644 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.gate.biases BF16 512x40 8 64 affine 1905157436 1905198396 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.gate.scales BF16 512x40 8 64 affine 1913066556 1913107516 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.gate.weight U32 512x640 8 64 affine 451151028 452461748 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1664526012 1664577212 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3069410572 3069461772 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1904338236 1905157436 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 4291880284 4291931484 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2300820060 2300871260 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1903519036 1904338236 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1903375420 1903426620 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1903467580 1903518780 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2132142764 2132961964 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2011549948 2011550028 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1883505804 1883505884 model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1158715780 1158717060 model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1883423884 1883505804 model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.hc_norm.weight BF16 10240 2000834972 2000855452 model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1857396076 1863949676 model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2281050124 2287603724 model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1883321484 1883403404 model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.hc_norm.weight BF16 10240 1463959020 1463979500 model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3729141996 3735695596 model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2261326940 2267880540 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.A_log BF16 48 1883256108 1883256204 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.conv1d.weight BF16 10240x4x1 1935219516 1935301436 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.dt_bias BF16 48 2011549852 2011549948 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3728267756 3728271596 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 3554981484 3554985324 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1979753276 1979814716 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1883252268 1883256108 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2036301724 2036305564 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2087916636 2087978076 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1882330668 1883149868 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 987430788 988249988 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1307463836 1320571036 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3807487324 3807978844 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1993003932 1993495452 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1848118556 1855982876 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.norm.weight BF16 128 2353438540 2353438796 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1865403756 1865895276 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1866110412 1866601932 model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1874466348 1882330668 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.gate.biases BF16 512x40 8 64 affine 2522779500 2522820460 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.gate.scales BF16 512x40 8 64 affine 3524646524 3524687484 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.gate.weight U32 512x640 8 64 affine 1864052076 1865362796 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1864000876 1864052076 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2296620124 2296671324 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3051220412 3052039612 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1863949676 1864000876 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2086718204 2086769404 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1912247356 1913066556 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1848067356 1848118556 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1857344876 1857396076 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1551082076 1551901276 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2603690268 2603690348 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2513768028 2513768108 model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 625624324 625625604 model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1847033116 1847115036 model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.hc_norm.weight BF16 10240 1832502556 1832523036 model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1825457436 1832011036 model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2544717516 2551271116 model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2051551052 2051632972 model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.hc_norm.weight BF16 10240 3030986172 3031006652 model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1817879580 1824433180 model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 840327204 846880804 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.A_log BF16 48 1813978044 1813978140 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.conv1d.weight BF16 10240x4x1 2163321004 2163402924 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.dt_bias BF16 48 3070629132 3070629228 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1804289724 1804293564 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1813028284 1813032124 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1813032124 1813093564 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1473564140 1473567980 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 704150740 704154580 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1538069340 1538130780 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1787744188 1788563388 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1978934076 1979753276 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1791080124 1804187324 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 677273972 677765492 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1335488156 1335979676 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2077985948 2085850268 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.norm.weight BF16 128 677765652 677765908 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1777039212 1777530732 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1779230908 1779722428 model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1805112764 1812977084 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.gate.biases BF16 512x40 8 64 affine 3752872364 3752913324 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.gate.scales BF16 512x40 8 64 affine 3602687084 3602728044 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.gate.weight U32 512x640 8 64 affine 1775728492 1777039212 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2441223932 2441275132 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2538108780 2538159980 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1759180652 1759999852 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1757726572 1757777772 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1706995484 1707046684 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2297563740 2298382940 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1750165612 1750216812 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 728906020 728957220 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1186662220 1187481420 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 648389124 648389204 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1654168844 1654168924 model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1750164332 1750165612 model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1749949292 1750031212 model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.hc_norm.weight BF16 10240 2102006876 2102027356 model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2396057676 2402611276 model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1741757292 1748310892 model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3003282876 3003364796 model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.hc_norm.weight BF16 10240 1732998252 1733018732 model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1725949292 1732502892 model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1358245644 1364799244 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.A_log BF16 48 1719344396 1719344492 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.conv1d.weight BF16 10240x4x1 500180148 500262068 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.dt_bias BF16 48 1979855676 1979855772 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1789437884 1789441724 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2011546012 2011549852 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 689024884 689086324 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3565471084 3565474924 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1613102508 1613106348 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2170019852 2170081292 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1611259308 1612078508 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1718523916 1719343116 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2012195228 2025302428 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1717499916 1717991436 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 501638164 502129684 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2978670956 2986535276 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.norm.weight BF16 128 1779230652 1779230908 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3451829276 3452320796 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1708273516 1708765036 model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1709635596 1717499916 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.gate.biases BF16 512x40 8 64 affine 649318676 649359636 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.gate.scales BF16 512x40 8 64 affine 2051427916 2051468876 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.gate.weight U32 512x640 8 64 affine 3514894108 3516204828 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1431504492 1431555692 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1708135964 1708187164 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2602830108 2603649308 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3022083772 3022134972 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3077244524 3077295724 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1706176284 1706995484 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3092496060 3092547260 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1706018844 1706070044 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3059969292 3060788492 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1688928108 1688928188 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1709635516 1709635596 model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2950907596 2950908876 model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3497724188 3497806108 model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.hc_norm.weight BF16 10240 1688907628 1688928108 model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2420633676 2427187276 model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1682354028 1688907628 model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2102027356 2102109276 model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.hc_norm.weight BF16 10240 2314129660 2314150140 model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2044218956 2050772556 model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1905202236 1911755836 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.gate.biases BF16 512x40 8 64 affine 1865362796 1865403756 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.gate.scales BF16 512x40 8 64 affine 2582789836 2582830796 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.gate.weight U32 512x640 8 64 affine 3460400316 3461711036 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3775789564 3775840764 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1682302732 1682353932 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1750271852 1751091052 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1680817932 1680869132 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2043962524 2044013724 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1681360652 1682179852 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1679947532 1679998732 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1061756292 1061807492 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1679998732 1680817932 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2206660364 2206660444 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1679886012 1679886092 model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2193480204 2193481484 model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2051468876 2051550796 model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.hc_norm.weight BF16 10240 1672001212 1672021692 model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3630326604 3636880204 model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1665447612 1672001212 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1665396412 1665447612 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 2240842924 2240894124 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 2240894124 2242532524 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.k_layernorm.weight BF16 128 2626168524 2626168780 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.q_layernorm.weight BF16 128 1903518780 1903519036 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_norm.weight BF16 256 2170081292 2170081804 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_proj.biases BF16 512x40 4 64 affine 1373361676 1373402636 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_proj.scales BF16 512x40 4 64 affine 2117100204 2117141164 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_proj.weight U32 512x320 4 64 affine 1210979756 1211635116 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.o_proj.biases BF16 2560x96 4 64 affine 501081268 501572788 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.o_proj.scales BF16 2560x96 4 64 affine 3753015724 3753507244 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.o_proj.weight U32 2560x768 4 64 affine 3612365516 3620229836 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_norm.weight BF16 256 1664525500 1664526012 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1662067804 1663050844 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1663542460 1664525500 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_proj.weight U32 12288x320 4 64 affine 460858548 476587188 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.v_proj.biases BF16 512x40 4 64 affine 1661320284 1661361244 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.v_proj.scales BF16 512x40 4 64 affine 2268044380 2268085340 model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.v_proj.weight U32 512x320 4 64 affine 1661412444 1662067804 model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2077080476 2077162396 model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.hc_norm.weight BF16 10240 1661299804 1661320284 model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3753507244 3760060844 model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1654746204 1661299804 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.A_log BF16 48 1538069244 1538069340 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.conv1d.weight BF16 10240x4x1 2238579884 2238661804 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.dt_bias BF16 48 1293793340 1293793436 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1654660444 1654664284 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1706172444 1706176284 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3796069116 3796130556 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3514890268 3514894108 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3003279036 3003282876 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3523762044 3523823484 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 3620741964 3621561164 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1612078508 1612897708 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1633688844 1646796044 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1707108124 1707599644 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1632971948 1633463468 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1919712316 1927576636 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.norm.weight BF16 128 2044013724 2044013980 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1498984012 1499475532 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1832011036 1832502556 model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1689419708 1697284028 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.gate.biases BF16 512x40 8 64 affine 1613061548 1613102508 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.gate.scales BF16 512x40 8 64 affine 1633463468 1633504428 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.gate.weight U32 512x640 8 64 affine 2213214044 2214524764 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2261275740 2261326940 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3110898108 3110949308 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2042999964 2043819164 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3016584636 3016635836 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2296691804 2296743004 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1601265068 1602084268 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2053395532 2053446732 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1601209676 1601260876 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3611546316 3612365516 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3553257244 3553257324 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1601168636 1601168716 model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1719343116 1719344396 model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1601066236 1601148156 model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.hc_norm.weight BF16 10240 1706070044 1706090524 model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2150189484 2156743084 model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2110491564 2117045164 model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3752790444 3752872364 model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.hc_norm.weight BF16 10240 1601004796 1601025276 model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1594447356 1601000956 model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1587893756 1594447356 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.A_log BF16 48 1697284028 1697284124 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.conv1d.weight BF16 10240x4x1 1708191084 1708273004 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.dt_bias BF16 48 3735695596 3735695692 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1203111596 1203115436 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1581192796 1581196636 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1581278556 1581339996 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1581167196 1581171036 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2496715772 2496719612 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1053057156 1053118596 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2101187676 2102006876 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1552740956 1553560156 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1553560156 1566667356 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1550586716 1551078236 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1654168924 1654660444 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2970724636 2978588956 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.norm.weight BF16 128 3786480124 3786480380 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1542230876 1542722396 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1824965916 1825457436 model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1672021692 1679886012 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.gate.biases BF16 512x40 8 64 affine 1540807516 1540848476 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.gate.scales BF16 512x40 8 64 affine 1098331428 1098372388 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.gate.weight U32 512x640 8 64 affine 1540920156 1542230876 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1540602716 1540653916 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1540756316 1540807516 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 754229636 755048836 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1538237020 1538288220 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1539650140 1539701340 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1539701340 1540520540 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2185482764 2185533964 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1538130780 1538181980 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2149370284 2150189484 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1538067804 1538067884 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1538067884 1538067964 model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1538067964 1538069244 model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1537125724 1537207644 model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.hc_norm.weight BF16 10240 1529711964 1529732444 model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1522666844 1529220444 model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3096458940 3103012540 model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1522584844 1522666764 model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.hc_norm.weight BF16 10240 2206588684 2206609164 model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1514556684 1521110284 model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1646796044 1653349644 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.A_log BF16 48 1514556588 1514556684 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.conv1d.weight BF16 10240x4x1 3496737052 3496818972 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.dt_bias BF16 48 1514556492 1514556588 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3514394908 3514398748 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1513897292 1513901132 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 704044500 704105940 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1513893452 1513897292 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 912881956 912885796 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1707046684 1707108124 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1499967052 1500786252 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 508902164 509721364 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1500786252 1513893452 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1499475532 1499967052 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2087425116 2087916636 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1439112812 1446977132 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.norm.weight BF16 128 1498983756 1498984012 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1496997196 1497488716 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1497488716 1497980236 model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3829001564 3836865884 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.gate.biases BF16 512x40 8 64 affine 2269038012 2269078972 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.gate.scales BF16 512x40 8 64 affine 1495549516 1495590476 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.gate.weight U32 512x640 8 64 affine 1495594316 1496905036 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1494576716 1494627916 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1494627916 1494679116 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1494730316 1495549516 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1883567324 1883618524 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2025302428 2025353628 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2338453580 2339272780 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3596082284 3596133484 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1487069516 1487120716 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1487120716 1487939916 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3460400236 3460400316 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3767925164 3767925244 model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1486839116 1486840396 model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1486675180 1486757100 model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.hc_norm.weight BF16 10240 3069461772 3069482252 model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1473567980 1480121580 model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1467010540 1473564140 model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1465617900 1465699820 model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.hc_norm.weight BF16 10240 1463887340 1463907820 model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1457309420 1463863020 model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3070629484 3077183084 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.gate.biases BF16 512x40 8 64 affine 1448543980 1448584940 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.gate.scales BF16 512x40 8 64 affine 1457268460 1457309420 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.gate.weight U32 512x640 8 64 affine 2327414860 2328725580 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1661361244 1661412444 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1446977260 1447028460 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1447724780 1448543980 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1439061612 1439112812 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2011611468 2011662668 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1748310892 1749130092 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1857293676 1857344876 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1779773628 1779824828 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1438242412 1439061612 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2300871340 2300871420 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2441172652 2441172732 model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1488021836 1488023116 model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1438160492 1438242412 model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.hc_norm.weight BF16 10240 2507128572 2507149052 model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1431606892 1438160492 model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1883669820 1890223420 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 2252240220 2252291420 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 1358112444 1358163644 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1429866092 1431504492 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.k_layernorm.weight BF16 128 1429865836 1429866092 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.q_layernorm.weight BF16 128 1429865580 1429865836 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_norm.weight BF16 256 1429865068 1429865580 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_proj.biases BF16 512x40 4 64 affine 1295605916 1295646876 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_proj.scales BF16 512x40 4 64 affine 1429824108 1429865068 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_proj.weight U32 512x320 4 64 affine 2410475596 2411130956 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.o_proj.biases BF16 2560x96 4 64 affine 1422236268 1422727788 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1429281388 1429772908 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.o_proj.weight U32 2560x768 4 64 affine 2252291420 2260155740 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_norm.weight BF16 256 2102192812 2102193324 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_proj.biases BF16 12288x40 4 64 affine 3727284716 3728267756 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1421253228 1422236268 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_proj.weight U32 12288x320 4 64 affine 3497806108 3513534748 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.v_proj.biases BF16 512x40 4 64 affine 1419471468 1419512428 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.v_proj.scales BF16 512x40 4 64 affine 2240801964 2240842924 model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.v_proj.weight U32 512x320 4 64 affine 1419727468 1420382828 model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2395484236 2395566156 model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.hc_norm.weight BF16 10240 1581171036 1581191516 model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1412917868 1419471468 model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1406364268 1412917868 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.A_log BF16 48 2269037916 2269038012 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.conv1d.weight BF16 10240x4x1 1406237548 1406319468 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.dt_bias BF16 48 2521284364 2521284460 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1406168428 1406172268 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1406172268 1406176108 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1406176108 1406237548 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3490138652 3490142492 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1406164588 1406168428 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1540694876 1540756316 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1404505708 1405324908 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1405345388 1406164588 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1271898940 1285006140 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 831152084 831643604 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1688928188 1689419708 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3103033276 3110897596 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.norm.weight BF16 128 1396641132 1396641388 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 677821044 678312564 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2067526460 2068017980 model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1388776812 1396641132 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.gate.biases BF16 512x40 8 64 affine 1387380236 1387421196 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.gate.scales BF16 512x40 8 64 affine 1387421292 1387462252 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.gate.weight U32 512x640 8 64 affine 1387462252 1388772972 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1373197836 1373249036 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1373402636 1373453836 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1386561036 1387380236 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1816927260 1816978460 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 649359636 649410836 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3523823484 3524642684 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1364882956 1364934156 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1364934156 1364985356 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1365804556 1366623756 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1540541100 1540541180 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1358163644 1358163724 model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1364799244 1364800524 model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1358030428 1358112348 model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.hc_norm.weight BF16 10240 1357149788 1357170268 model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1366623756 1373177356 model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3798803292 3805356892 model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1349203548 1349285468 model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.hc_norm.weight BF16 10240 1349183068 1349203548 model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3589477484 3596031084 model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1342629468 1349183068 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.A_log BF16 48 1663050844 1663050940 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.conv1d.weight BF16 10240x4x1 1633606924 1633688844 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.dt_bias BF16 48 1342629372 1342629468 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1750268012 1750271852 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1335979676 1335983516 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1883256204 1883317644 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1334497436 1334501276 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1334501276 1334505116 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1813093564 1813155004 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1306644636 1307463836 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1320571036 1321390236 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1321390236 1334497436 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1297449116 1297940636 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3031006652 3031498172 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1297940636 1305804956 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.norm.weight BF16 128 2067391132 2067391388 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1295646876 1296138396 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1296957596 1297449116 model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1449404140 1457268460 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.gate.biases BF16 512x40 8 64 affine 1295483036 1295523996 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.gate.scales BF16 512x40 8 64 affine 1601168716 1601209676 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.gate.weight U32 512x640 8 64 affine 3553343084 3554653804 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1293742140 1293793340 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1293793436 1293844636 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1293844636 1294663836 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 4262849884 4262901084 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3776332284 3776383484 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3000708796 3001527996 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1285007420 1285058620 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3110949308 3111000508 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1292922940 1293742140 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1709635436 1709635516 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1271898860 1271898940 model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1285006140 1285007420 model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2215118764 2215200684 model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.hc_norm.weight BF16 10240 1263952620 1263973100 model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1566667356 1573220956 model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2117141164 2123694764 model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1263870188 1263952108 model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.hc_norm.weight BF16 10240 1263849708 1263870188 model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1257292268 1263845868 model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1242874268 1249427868 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.gate.biases BF16 512x40 8 64 affine 3588609644 3588650604 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.gate.scales BF16 512x40 8 64 affine 1242833228 1242874188 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.gate.weight U32 512x640 8 64 affine 3784247804 3785558524 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1241092428 1241143628 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1241143628 1241194828 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1241194828 1242014028 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 343788804 343840004 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2000783772 2000834972 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1232408908 1233228108 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1224943948 1224995148 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 4300667244 4300718444 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1224995148 1225814348 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1224942412 1224942492 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1224942588 1224942668 model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1224942668 1224943948 model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3725349276 3725431196 model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.hc_norm.weight BF16 10240 1224918012 1224938492 model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1218363900 1224917500 model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2314307660 2320861260 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 678312564 678363764 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 1218312620 1218363820 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 3094769340 3096407740 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.k_layernorm.weight BF16 128 1218312364 1218312620 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.q_layernorm.weight BF16 128 1211676588 1211676844 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_norm.weight BF16 256 1211635116 1211635628 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_proj.biases BF16 512x40 4 64 affine 1824924956 1824965916 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_proj.scales BF16 512x40 4 64 affine 2481703932 2481744892 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_proj.weight U32 512x320 4 64 affine 2050772556 2051427916 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.o_proj.biases BF16 2560x96 4 64 affine 1201637036 1202128556 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1202620076 1203111596 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.o_proj.weight U32 2560x768 4 64 affine 1203115436 1210979756 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_norm.weight BF16 256 1364800524 1364801036 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1777592172 1778575212 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1200653996 1201637036 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_proj.weight U32 12288x320 4 64 affine 3536658124 3552386764 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.v_proj.biases BF16 512x40 4 64 affine 704105940 704146900 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.v_proj.scales BF16 512x40 4 64 affine 1903426620 1903467580 model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.v_proj.weight U32 512x320 4 64 affine 1061079076 1061734436 model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1486840396 1486922316 model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.hc_norm.weight BF16 10240 3052080572 3052101052 model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 912885796 919439396 model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1077843748 1084397348 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.A_log BF16 48 1187546700 1187546796 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.conv1d.weight BF16 10240x4x1 1419594348 1419676268 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.dt_bias BF16 48 1224942492 1224942588 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1187481420 1187485260 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2950903500 2950907340 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1187485260 1187546700 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1186658380 1186662220 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3751475628 3751479468 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3788139516 3788200956 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1163531340 1164350540 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1166173260 1166992460 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1166996300 1180103500 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2214627244 2215118764 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1161964524 1162456044 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1542722396 1550586716 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.norm.weight BF16 128 4298485692 4298485948 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1162548300 1163039820 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1163039820 1163531340 model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.out_proj.weight U32 2560x768 4 64 affine 4282664284 4290528604 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.gate.biases BF16 512x40 8 64 affine 1162507340 1162548300 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.gate.scales BF16 512x40 8 64 affine 1846992156 1847033116 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.gate.weight U32 512x640 8 64 affine 1465699820 1467010540 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1161913324 1161964524 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1162456044 1162507244 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1060259796 1061078996 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2132091564 2132142764 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1161862124 1161913324 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2329391180 2330210380 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1160940524 1160991724 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2267880540 2267931740 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1160991724 1161810924 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 434988212 434988292 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1160121244 1160121324 model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3573431404 3573432684 model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1159003780 1159085700 model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.hc_norm.weight BF16 10240 1158932100 1158952580 model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 632936964 639490564 model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1529752924 1536306524 model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1158850180 1158932100 model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.hc_norm.weight BF16 10240 1061734436 1061754916 model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 453813428 460367028 model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1152162180 1158715780 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.gate.biases BF16 512x40 8 64 affine 1033105540 1033146500 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.gate.scales BF16 512x40 8 64 affine 1150810500 1150851460 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.gate.weight U32 512x640 8 64 affine 1150851460 1152162180 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3786377724 3786428924 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 537682452 537733652 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1149991300 1150810500 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1336024476 1336075676 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1149940020 1149991220 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 365587556 366406756 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1539107420 1539158620 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1149069620 1149120820 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1149120820 1149940020 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1149048980 1149049060 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1149049060 1149049140 model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2951011356 2951012636 model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3496823068 3496904988 model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.hc_norm.weight BF16 10240 2252219644 2252240124 model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1142413460 1148967060 model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1069866452 1076420052 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1126582420 1126633620 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 1142362260 1142413460 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1789441724 1791080124 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.k_layernorm.weight BF16 128 1126582084 1126582340 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.q_layernorm.weight BF16 128 1125721668 1125721924 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_norm.weight BF16 256 988290948 988291460 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_proj.biases BF16 512x40 4 64 affine 3092598460 3092639420 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_proj.scales BF16 512x40 4 64 affine 1117816292 1117857252 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_proj.weight U32 512x320 4 64 affine 3017455036 3018110396 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.o_proj.biases BF16 2560x96 4 64 affine 1104991620 1105483140 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1117324772 1117816292 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.o_proj.weight U32 2560x768 4 64 affine 3821137244 3829001564 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_norm.weight BF16 256 3110897596 3110898108 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1113614052 1114597092 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1114597092 1115580132 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_proj.weight U32 12288x320 4 64 affine 2551271116 2566999756 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.v_proj.biases BF16 512x40 4 64 affine 552349972 552390932 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.v_proj.scales BF16 512x40 4 64 affine 1496905036 1496945996 model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.v_proj.weight U32 512x320 4 64 affine 1112958692 1113614052 model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1112876772 1112958692 model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.hc_norm.weight BF16 10240 1112856292 1112876772 model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 626383364 632936964 model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1106302436 1112856036 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.A_log BF16 48 1106302340 1106302436 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.conv1d.weight BF16 10240x4x1 2260155740 2260237660 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.dt_bias BF16 48 1104991524 1104991620 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1098434084 1098437924 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2042996124 2042999964 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1883190828 1883252268 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1084397348 1084401188 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1084401188 1084405028 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1098372388 1098433828 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1077024548 1077843748 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2625349324 2626168524 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2300936700 2314043900 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1076532948 1077024468 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1006313860 1006805380 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2185615884 2193480204 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.norm.weight BF16 128 1076471252 1076471508 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1856802076 1857293596 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3526725244 3527216764 model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1285058620 1292922940 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.gate.biases BF16 512x40 8 64 affine 2496623612 2496664572 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.gate.scales BF16 512x40 8 64 affine 3573339244 3573380204 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.gate.weight U32 512x640 8 64 affine 1814797340 1816108060 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1076420052 1076471252 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1431555692 1431606892 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1813978140 1814797340 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2238948524 2238999724 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1061858772 1061909972 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1068514772 1069333972 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2206609164 2206660364 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1061807492 1061858692 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3522860924 3523680124 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1059723396 1059723476 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1061078996 1061079076 model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1061754916 1061756196 model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3069328652 3069410572 model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.hc_norm.weight BF16 10240 3522758524 3522779004 model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1053118596 1059672196 model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1046503300 1053056900 model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1033019780 1033101700 model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.hc_norm.weight BF16 10240 1046431620 1046452100 model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1039826820 1046380420 model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2287607564 2294161164 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.A_log BF16 48 1061756196 1061756292 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.conv1d.weight BF16 10240x4x1 386733236 386815156 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.dt_bias BF16 48 1633555628 1633555724 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1789434044 1789437884 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2149366444 2149370284 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1033207940 1033269380 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1033101700 1033105540 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3788200956 3788204796 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1033146500 1033207940 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2336815180 2337634380 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2481744892 2482564092 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1006805380 1019912580 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1005822340 1006313860 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1539158620 1539650140 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1264014060 1271878380 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.norm.weight BF16 128 1824924700 1824924956 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1059723476 1060214996 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 997466500 997958020 model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.out_proj.weight U32 2560x768 4 64 affine 997958020 1005822340 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.gate.biases BF16 512x40 8 64 affine 988249988 988290948 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.gate.scales BF16 512x40 8 64 affine 3513534748 3513575708 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.gate.weight U32 512x640 8 64 affine 988291460 989602180 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1993499292 1993550492 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3084887660 3084938860 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3496904988 3497724188 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 973449348 973500548 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3092547260 3092598460 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 986611588 987430788 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1804238524 1804289724 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 973398148 973449348 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3535572684 3536391884 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 973377412 973377492 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 973397972 973398052 model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1159085700 1159086980 model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1865895372 1865977292 model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.hc_norm.weight BF16 10240 973356932 973377412 model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2483516412 2490070012 model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 966803332 973356932 model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2978589036 2978670956 model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.hc_norm.weight BF16 10240 973377492 973397972 model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 958939012 965492612 model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1180104780 1186658380 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.A_log BF16 48 1162507244 1162507340 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.conv1d.weight BF16 10240x4x1 958805892 958887812 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.dt_bias BF16 48 952252196 952252292 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 986607748 986611588 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 952248356 952252196 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 4300718444 4300779884 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 846880804 846884644 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1039822980 1039826820 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 536258836 536320276 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 928716836 929536036 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 929587236 930406436 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 930457636 943564836 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1164350540 1164842060 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 928225316 928716836 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2025395868 2033260188 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.norm.weight BF16 128 2297563484 2297563740 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1202128556 1202620076 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1529220444 1529711964 model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.out_proj.weight U32 2560x768 4 64 affine 919439396 927303716 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.gate.biases BF16 512x40 8 64 affine 858364196 858405156 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.gate.scales BF16 512x40 8 64 affine 878270756 878311716 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.gate.weight U32 512x640 8 64 affine 878311716 879622436 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 858261796 858312996 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 858312996 858364196 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1160121324 1160940524 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1725898092 1725949292 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2163402924 2163454124 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2521284460 2522103660 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1496945996 1496997196 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1788563388 1788614588 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 856787236 857606436 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 849690756 849690836 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 849690836 849690916 model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3574794604 3574795884 model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2075483036 2075564956 model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.hc_norm.weight BF16 10240 849670276 849690756 model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2427195052 2433748652 model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1118676548 1125230148 model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 849588356 849670276 model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.hc_norm.weight BF16 10240 2339354700 2339375180 model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 719679700 726233300 model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2538163916 2544717516 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.gate.biases BF16 512x40 8 64 affine 846884644 846925604 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.gate.scales BF16 512x40 8 64 affine 848236420 848277380 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.gate.weight U32 512x640 8 64 affine 848277380 849588100 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3588558444 3588609644 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3685156764 3685207964 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 839508004 840327204 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3602728044 3602779244 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1633504428 1633555628 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 831643604 832462804 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1883618524 1883669724 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1158717060 1158768260 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1364985356 1365804556 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 824105604 824105684 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3527708284 3527708364 model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 831150804 831152084 model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1218230444 1218312364 model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.hc_norm.weight BF16 10240 1498963276 1498983756 model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 817500804 824054404 model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1061909972 1068463572 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 816087684 816138884 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 817449604 817500804 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1463979500 1465617900 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.k_layernorm.weight BF16 128 2950907340 2950907596 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.q_layernorm.weight BF16 128 816001668 816001924 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_norm.weight BF16 256 2602696396 2602696908 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_proj.biases BF16 512x40 4 64 affine 1487980876 1488021836 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_proj.scales BF16 512x40 4 64 affine 1406323308 1406364268 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_proj.weight U32 512x320 4 64 affine 3574795884 3575451244 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.o_proj.biases BF16 2560x96 4 64 affine 807645828 808137348 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1993673372 1994164892 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.o_proj.weight U32 2560x768 4 64 affine 808137348 816001668 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_norm.weight BF16 256 807645316 807645828 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_proj.biases BF16 12288x40 4 64 affine 2260237660 2261220700 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_proj.scales BF16 12288x40 4 64 affine 3083904364 3084887404 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_proj.weight U32 12288x320 4 64 affine 2221754284 2237482924 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.v_proj.biases BF16 512x40 4 64 affine 2604336844 2604377804 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.v_proj.scales BF16 512x40 4 64 affine 2603649308 2603690268 model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.v_proj.weight U32 512x320 4 64 affine 1447069420 1447724780 model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 755052676 755134596 model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.hc_norm.weight BF16 10240 754209156 754229636 model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2507214428 2513768028 model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 4291931996 4298485596 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.A_log BF16 48 3735746892 3735746988 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.conv1d.weight BF16 10240x4x1 3536525004 3536606924 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.dt_bias BF16 48 1601264972 1601265068 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2314125820 2314129660 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1905198396 1905202236 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3068775692 3068837132 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 442463572 442467412 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1708187244 1708191084 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 508840724 508902164 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1855982876 1856802076 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 728998180 729817380 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3807978844 3821086044 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 753174916 753666436 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 753666436 754157956 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 603156324 611020644 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.norm.weight BF16 128 2087424860 2087425116 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 752191876 752683396 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 752683396 753174916 model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.out_proj.weight U32 2560x768 4 64 affine 989602180 997466500 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.gate.biases BF16 512x40 8 64 affine 728957220 728998180 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.gate.scales BF16 512x40 8 64 affine 729817380 729858340 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.gate.weight U32 512x640 8 64 affine 1609948588 1611259308 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1538185820 1538237020 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1846940956 1846992156 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3573924204 3574743404 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 509938196 509989396 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 728854820 728906020 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2268085340 2268904540 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1046380420 1046431620 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 754157956 754209156 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 728035620 728854820 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 726233300 726233380 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1708187164 1708187244 model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 719678420 719679700 model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 719555540 719637460 model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.hc_norm.weight BF16 10240 2252199164 2252219644 model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 712182740 718736340 model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 704154580 710708180 model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 703962580 704044500 model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.hc_norm.weight BF16 10240 703942100 703962580 model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 905836836 912390436 model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1581340156 1587893756 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.A_log BF16 48 2044095980 2044096076 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.conv1d.weight BF16 10240x4x1 1486926156 1487008076 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.dt_bias BF16 48 703942004 703942100 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1740934252 1740938092 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 703938164 703942004 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 2011550028 2011611468 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 704146900 704150740 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 755048836 755052676 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1076471508 1076532948 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 689086324 689905524 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 689905524 690724724 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 690724724 703831924 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1732506732 1732998252 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 680669044 681160564 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 681160564 689024884 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.norm.weight BF16 128 2051550796 2051551052 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1522093324 1522584844 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2123735724 2124227244 model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2360043596 2367907916 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.gate.biases BF16 512x40 8 64 affine 1335983516 1336024476 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.gate.scales BF16 512x40 8 64 affine 679317364 679358324 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.gate.weight U32 512x640 8 64 affine 679358324 680669044 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2441172732 2441223932 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 856244516 856295716 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 678498164 679317364 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1463907820 1463959020 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3637392204 3637443404 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1708816236 1709635436 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 678446964 678498164 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 929536036 929587236 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1664577212 1665396412 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 677765492 677765572 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 677765572 677765652 model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 678363764 678365044 model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 676321652 676403572 model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.hc_norm.weight BF16 10240 675737876 675758356 model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1033269380 1039822980 model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 662630676 669184276 model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 755134596 755216516 model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.hc_norm.weight BF16 10240 655964436 655984916 model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2215200684 2221754284 model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 649410836 655964436 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.A_log BF16 48 649267380 649267476 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.conv1d.weight BF16 10240x4x1 1486757196 1486839116 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.dt_bias BF16 48 2427187276 2427187372 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 649212340 649216180 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1903371580 1903375420 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1679886092 1679947532 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 649208500 649212340 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2086714268 2086718108 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3067403532 3067464972 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 648389204 649208404 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2522820460 2523639660 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3038113212 3051220412 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 639490564 639982084 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 639982084 640473604 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 640524804 648389124 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.norm.weight BF16 128 3806176348 3806176604 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1824433180 1824924700 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 625891844 626383364 model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1349285468 1357149788 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.gate.biases BF16 512x40 8 64 affine 625809924 625850884 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.gate.scales BF16 512x40 8 64 affine 625850884 625891844 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.gate.weight U32 512x640 8 64 affine 2298618460 2299929180 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2538057580 2538108780 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1494679116 1494730316 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2010235292 2011054492 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 625676804 625728004 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2238661804 2238713004 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3126606876 3127426076 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1068463572 1068514772 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 625625604 625676804 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1420434028 1421253228 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 624967524 624967604 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 625622964 625623044 model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 625623044 625624324 model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1612979628 1613061548 model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.hc_norm.weight BF16 10240 624947044 624967524 model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 618393444 624947044 model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 611020644 617574244 model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2278950332 2279032252 model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.hc_norm.weight BF16 10240 602644324 602664804 model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 596086628 602640228 model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2163458476 2170012076 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.gate.biases BF16 512x40 8 64 affine 662589716 662630676 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.gate.scales BF16 512x40 8 64 affine 596045668 596086628 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.gate.weight U32 512x640 8 64 affine 1757869932 1759180652 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 675758452 675809652 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 595994468 596045668 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 595154788 595973988 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1059672196 1059723396 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3596031084 3596082284 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 594335588 595154788 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 677222772 677273972 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1161810924 1161862124 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2033915548 2034734748 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1149991220 1149991300 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 593842708 593842788 model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 594334308 594335588 model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1158768260 1158850180 model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.hc_norm.weight BF16 10240 3786480636 3786501116 model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1781135548 1787689148 model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 587289108 593842708 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 585548308 585599508 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 585599508 585650708 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 585650708 587289108 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.k_layernorm.weight BF16 128 585548052 585548308 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.q_layernorm.weight BF16 128 3125787164 3125787420 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_norm.weight BF16 256 2077985436 2077985948 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_proj.biases BF16 512x40 4 64 affine 1979855772 1979896732 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_proj.scales BF16 512x40 4 64 affine 585507092 585548052 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_proj.weight U32 512x320 4 64 affine 2086769404 2087424764 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.o_proj.biases BF16 2560x96 4 64 affine 577151252 577642772 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.o_proj.scales BF16 2560x96 4 64 affine 2986535356 2987026876 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.o_proj.weight U32 2560x768 4 64 affine 577642772 585507092 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_norm.weight BF16 256 1708273004 1708273516 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_proj.biases BF16 12288x40 4 64 affine 559620372 560603412 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1159086980 1160070020 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_proj.weight U32 12288x320 4 64 affine 560603412 576332052 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.v_proj.biases BF16 512x40 4 64 affine 2076547996 2076588956 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.v_proj.scales BF16 512x40 4 64 affine 1357170268 1357211228 model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.v_proj.weight U32 512x320 4 64 affine 552411412 553066772 model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1419512428 1419594348 model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.hc_norm.weight BF16 10240 552390932 552411412 model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 832462804 839016404 model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 545796372 552349972 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.A_log BF16 48 973398052 973398148 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.conv1d.weight BF16 10240x4x1 537798932 537880852 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.dt_bias BF16 48 744276260 744276356 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 703883124 703886964 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 537733652 537737492 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 537737492 537798932 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2287603724 2287607564 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3083900524 3083904364 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1883505884 1883567324 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1105483140 1106302340 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1115580132 1116399332 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3655829404 3668936604 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3573432684 3573924204 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2035707804 2036199324 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 357063940 364928260 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.norm.weight BF16 128 537682196 537682452 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3557115244 3557606764 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 537190676 537682196 model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1249427948 1257292268 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.gate.biases BF16 512x40 8 64 affine 1540653916 1540694876 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.gate.scales BF16 512x40 8 64 affine 1125721924 1125762884 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.gate.weight U32 512x640 8 64 affine 816138884 817449604 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1158952580 1159003780 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 536320276 536371476 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1125762884 1126582084 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 677766004 677817204 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 536207636 536258836 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1816978460 1817797660 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1046452100 1046503300 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 509886996 509938196 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3725595036 3726414236 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1522666764 1522666844 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1224942332 1224942412 model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 509885204 509886484 model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 509803284 509885204 model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.hc_norm.weight BF16 10240 1306624156 1306644636 model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1913107516 1919661116 model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1422727788 1429281388 model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 509721364 509803284 model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.hc_norm.weight BF16 10240 508820244 508840724 model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 502211604 508765204 model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 655984916 662538516 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.A_log BF16 48 1358112348 1358112444 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.conv1d.weight BF16 10240x4x1 502129684 502211604 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.dt_bias BF16 48 501638068 501638164 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 501634228 501638068 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1538181980 1538185820 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3610665676 3610727116 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2339508300 2339512140 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3553339244 3553343084 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 501572788 501634228 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 500262068 501081268 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3016635836 3017455036 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1085224228 1098331428 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 856295716 856787236 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2170081804 2170573324 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 492315828 500180148 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.norm.weight BF16 128 3752790188 3752790444 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 460367028 460858548 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 476587188 477078708 model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.out_proj.weight U32 2560x768 4 64 affine 484451508 492315828 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.gate.biases BF16 512x40 8 64 affine 452461748 452502708 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.gate.scales BF16 512x40 8 64 affine 1537207644 1537248604 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.gate.weight U32 512x640 8 64 affine 452502708 453813428 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 442412372 442463572 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3786428924 3786480124 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 442467412 443286612 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1750072172 1750123372 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2514628364 2514679564 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 441593172 442412372 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 441541972 441593172 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 752140676 752191876 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 576332052 577151252 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 441541892 441541972 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1581340076 1581340156 model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3483501852 3483503132 model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 816005764 816087684 model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.hc_norm.weight BF16 10240 3092362940 3092383420 model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1480121580 1486675180 model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 434988292 441541892 model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 434906292 434988212 model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.hc_norm.weight BF16 10240 434885812 434906292 model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 428332212 434885812 model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 421778612 428332212 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.A_log BF16 48 677765908 677766004 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.conv1d.weight BF16 10240x4x1 2067444540 2067526460 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.dt_bias BF16 48 1118676452 1118676548 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1388772972 1388776812 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 421713332 421717172 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 421717172 421778612 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1406319468 1406323308 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1116460772 1116464612 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2497376252 2497437692 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 394679476 395498676 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3124967964 3125787164 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 395498676 408605876 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3526172284 3526663804 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 711691220 712182740 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 386815156 394679476 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.norm.weight BF16 128 3070629228 3070629484 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1125230148 1125721668 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3752298668 3752790188 model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.out_proj.weight U32 2560x768 4 64 affine 944384036 952248356 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.gate.biases BF16 512x40 8 64 affine 386692276 386733236 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.gate.scales BF16 512x40 8 64 affine 2035666844 2035707804 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.gate.weight U32 512x640 8 64 affine 1779824828 1781135548 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1757818732 1757869932 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2521233164 2521284364 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2085895068 2086714268 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3554745964 3554797164 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3729090796 3729141996 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3724529820 3725349020 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 386641076 386692276 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3821086044 3821137244 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3637465164 3638284364 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2012154188 2012154268 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 386640996 386641076 model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3637463884 3637465164 model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1148967060 1149048980 model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.hc_norm.weight BF16 10240 1529732444 1529752924 model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 380087396 386640996 model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 373533796 380087396 model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 372960356 373042276 model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.hc_norm.weight BF16 10240 676301172 676321652 model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2951063836 2957617436 model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 366406756 372960356 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.A_log BF16 48 848236324 848236420 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.conv1d.weight BF16 10240x4x1 2149284524 2149366444 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.dt_bias BF16 48 365587460 365587556 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 357060100 357063940 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 364928260 364932100 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1373249036 1373310476 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 509989396 509993236 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2110487724 2110491564 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 356998404 357059844 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1537248604 1538067804 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1816108060 1816927260 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 343840004 356947204 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 373042276 373533796 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 602664804 603156324 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1233228108 1241092428 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.norm.weight BF16 128 343788548 343788804 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 334941188 335432708 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 335432708 335924228 model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.out_proj.weight U32 2560x768 4 64 affine 335924228 343788548 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.gate.biases BF16 512x40 8 64 affine 333548548 333589508 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.gate.scales BF16 512x40 8 64 affine 333589508 333630468 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.gate.weight U32 512x640 8 64 affine 333630468 334941188 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 332626948 332678148 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 332678148 332729348 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 332729348 333548548 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 332524548 332575748 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 332575748 332626948 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1551901276 1552720476 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3552386764 3552437964 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 332473348 332524548 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3806176604 3806995804 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1249427868 1249427948 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1857293596 1857293676 model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 332420868 332422148 model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 332338948 332420868 model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.hc_norm.weight BF16 10240 332318468 332338948 model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2353438796 2359992396 model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2068017980 2074571580 model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 332236548 332318468 model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.hc_norm.weight BF16 10240 2238506924 2238527404 model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 325682948 332236548 model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 319129348 325682948 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.gate.biases BF16 512x40 8 64 affine 3490142492 3490183452 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.gate.scales BF16 512x40 8 64 affine 317777668 317818628 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.gate.weight U32 512x640 8 64 affine 317818628 319129348 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2602778908 2602830108 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 316907268 316958468 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 316958468 317777668 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 4298977468 4299028668 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 316856068 316907268 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2295800924 2296620124 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1697284124 1697335324 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 315985668 316036868 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 316036868 316856068 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 315984228 315984308 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 315984308 315984388 model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 315984388 315985668 model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1295523996 1295605916 model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.hc_norm.weight BF16 10240 1164842060 1164862540 model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 309430628 315984228 model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 302877028 309430628 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 302733668 302784868 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 302784868 302836068 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1964137276 1965775676 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.k_layernorm.weight BF16 128 357059844 357060100 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.q_layernorm.weight BF16 128 302733412 302733668 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_norm.weight BF16 256 296179300 296179812 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_proj.biases BF16 512x40 4 64 affine 2139515564 2139556524 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_proj.scales BF16 512x40 4 64 affine 296138340 296179300 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_proj.weight U32 512x320 4 64 affine 2496719612 2497374972 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.o_proj.biases BF16 2560x96 4 64 affine 287782500 288274020 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1334505116 1334996636 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.o_proj.weight U32 2560x768 4 64 affine 288274020 296138340 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_norm.weight BF16 256 287730788 287731300 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1521110284 1522093324 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_proj.scales BF16 12288x40 4 64 affine 286747748 287730788 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_proj.weight U32 12288x320 4 64 affine 3735746988 3751475628 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.v_proj.biases BF16 512x40 4 64 affine 2238465964 2238506924 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.v_proj.scales BF16 512x40 4 64 affine 1211635628 1211676588 model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.v_proj.weight U32 512x320 4 64 affine 1513901132 1514556492 model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2052453452 2052535372 model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.hc_norm.weight BF16 10240 286727268 286747748 model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1935301436 1941855036 model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 477078708 483632308 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.A_log BF16 48 286727172 286727268 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.conv1d.weight BF16 10240x4x1 286645252 286727172 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.dt_bias BF16 48 286645156 286645252 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 286637476 286641316 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 286641316 286645156 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1777530732 1777592172 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1166992460 1166996300 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 286572196 286576036 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 286576036 286637476 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1242014028 1242833228 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 272645796 273464996 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 273464996 286572196 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 270843556 271335076 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 272154276 272645796 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3767925244 3775789564 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.norm.weight BF16 128 3496822812 3496823068 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 262487716 262979236 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1069374932 1069866452 model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.out_proj.weight U32 2560x768 4 64 affine 262979236 270843556 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.gate.biases BF16 512x40 8 64 affine 262405796 262446756 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.gate.scales BF16 512x40 8 64 affine 262446756 262487716 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.gate.weight U32 512x640 8 64 affine 726233380 727544100 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 262354596 262405796 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1420382828 1420434028 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1294663836 1295483036 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2298464860 2298516060 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1919661116 1919712316 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1788614588 1789433788 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2314150140 2314201340 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 261484196 261535396 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 261535396 262354596 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1126582340 1126582420 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 261482836 261482916 model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 261482916 261484196 model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1612897708 1612979628 model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.hc_norm.weight BF16 10240 1463866860 1463887340 model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 254929236 261482836 model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 248375636 254929236 model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3629425484 3629507404 model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.hc_norm.weight BF16 10240 3103012540 3103033020 model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 241822036 248375636 model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 235268436 241822036 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.A_log BF16 48 1387421196 1387421292 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.conv1d.weight BF16 10240x4x1 235186516 235268436 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.dt_bias BF16 48 235186420 235186516 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 235178740 235182580 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 235182580 235186420 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1540541276 1540602716 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 235174900 235178740 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 816001924 816005764 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1993550492 1993611932 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 3084990060 3085809260 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3805356892 3806176092 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2987518396 3000625596 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 234683380 235174900 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 824105684 824597204 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2530193260 2538057580 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.norm.weight BF16 128 234683124 234683380 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3514398748 3514890268 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1948408636 1948900156 model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3776383484 3784247804 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.gate.biases BF16 512x40 8 64 affine 2603804364 2603845324 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.gate.scales BF16 512x40 8 64 affine 1263973100 1264014060 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.gate.weight U32 512x640 8 64 affine 965492612 966803332 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 233812724 233863924 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1804187324 1804238524 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 233863924 234683124 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 232942324 232993524 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 287731300 287782500 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 232993524 233812724 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 649267476 649318676 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 232891124 232942324 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2419814476 2420633676 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3553257164 3553257244 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1581339996 1581340076 model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 232889844 232891124 model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1364801036 1364882956 model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.hc_norm.weight BF16 10240 1149049140 1149069620 model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 226336244 232889844 model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 219782644 226336244 model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1847115036 1847196956 model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.hc_norm.weight BF16 10240 219762164 219782644 model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 213208564 219762164 model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3691761820 3698315420 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.A_log BF16 48 1865895276 1865895372 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.conv1d.weight BF16 10240x4x1 213126644 213208564 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.dt_bias BF16 48 213126548 213126644 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 213118868 213122708 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 213122708 213126548 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1682241292 1682302732 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3524642684 3524646524 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 213053588 213057428 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 213057428 213118868 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1653349644 1654168844 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 192573588 193392788 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 193392788 206499988 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 183726228 184217748 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 184217748 184709268 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 184709268 192573588 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.norm.weight BF16 128 183725972 183726228 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 182742932 183234452 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 183234452 183725972 model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1733069932 1740934252 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.gate.biases BF16 512x40 8 64 affine 1060214996 1060255956 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.gate.scales BF16 512x40 8 64 affine 182701972 182742932 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.gate.weight U32 512x640 8 64 affine 2250888364 2252199084 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 182650772 182701972 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 356947204 356998404 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1538288220 1539107420 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 649216180 649267380 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2950960156 2951011356 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 271335076 272154276 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 508765204 508816404 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 181780372 181831572 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 181831572 182650772 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 181779012 181779092 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 839016404 839016484 model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 181779092 181780372 model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 181697092 181779012 model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.hc_norm.weight BF16 10240 181676612 181697092 model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 175123012 181676612 model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 168569412 175123012 model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 160623172 160705092 model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.hc_norm.weight BF16 10240 1540520540 1540541020 model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1098437924 1104991524 model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 154069572 160623172 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.gate.biases BF16 512x40 8 64 affine 152717892 152758852 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.gate.scales BF16 512x40 8 64 affine 302836068 302877028 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.gate.weight U32 512x640 8 64 affine 152758852 154069572 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 151796292 151847492 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 151847492 151898692 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 151898692 152717892 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 150925892 150977092 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 545745172 545796372 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 150977092 151796292 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 150004292 150055492 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 150055492 150106692 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 150106692 150925892 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 150002852 150002932 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 150002932 150003012 model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 150003012 150004292 model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 149920932 150002852 model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.hc_norm.weight BF16 10240 149900452 149920932 model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 143346852 149900452 model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 130239652 136793252 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 130137252 130188452 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 130188452 130239652 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 3786501116 3788139516 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.k_layernorm.weight BF16 128 3725349020 3725349276 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.q_layernorm.weight BF16 128 3018929596 3018929852 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_norm.weight BF16 256 130094500 130095012 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_proj.biases BF16 512x40 4 64 affine 2944237260 2944278220 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_proj.scales BF16 512x40 4 64 affine 1069333972 1069374932 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_proj.weight U32 512x320 4 64 affine 129439140 130094500 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.o_proj.biases BF16 2560x96 4 64 affine 128456100 128947620 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.o_proj.scales BF16 2560x96 4 64 affine 128947620 129439140 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.o_proj.weight U32 2560x768 4 64 affine 160705092 168569412 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_norm.weight BF16 256 2163454124 2163454636 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_proj.biases BF16 12288x40 4 64 affine 127473060 128456100 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_proj.scales BF16 12288x40 4 64 affine 2237482924 2238465964 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_proj.weight U32 12288x320 4 64 affine 3669428124 3685156764 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.v_proj.biases BF16 512x40 4 64 affine 127432100 127473060 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.v_proj.scales BF16 512x40 4 64 affine 1487939916 1487980876 model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.v_proj.weight U32 512x320 4 64 affine 126776740 127432100 model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 126694820 126776740 model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.hc_norm.weight BF16 10240 126674340 126694820 model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3565474924 3572028524 model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 136793252 143346852 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.A_log BF16 48 126674244 126674340 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.conv1d.weight BF16 10240x4x1 126592324 126674244 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.dt_bias BF16 48 1540541180 1540541276 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 126522948 126526788 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 126527044 126530884 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 126530884 126592324 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1463863020 1463866860 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2314201420 2314205260 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 126461508 126522948 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 125642308 126461508 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1448584940 1449404140 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3470394652 3483501852 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2008719772 2009211292 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 117286468 117777988 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 117777988 125642308 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.norm.weight BF16 128 117286212 117286468 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 108438852 108930372 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 108930372 109421892 model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.out_proj.weight U32 2560x768 4 64 affine 109421892 117286212 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.gate.biases BF16 512x40 8 64 affine 3124927004 3124967964 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.gate.scales BF16 512x40 8 64 affine 108397892 108438852 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.gate.weight U32 512x640 8 64 affine 4290569564 4291880284 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 107527492 107578692 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3077295724 3077346924 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 107578692 108397892 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1779722428 1779773628 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 106657092 106708292 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 106708292 107527492 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 105786692 105837892 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2268986716 2269037916 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 105837892 106657092 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 105786612 105786692 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3727233436 3727233516 model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 130135972 130137252 model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2328725580 2328807500 model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.hc_norm.weight BF16 10240 2507149052 2507169532 model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 99212532 105766132 model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1225814348 1232367948 model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2339272780 2339354700 model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.hc_norm.weight BF16 10240 105766132 105786612 model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 4276049244 4282602844 model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1488023116 1494576716 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.A_log BF16 48 99212436 99212532 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.conv1d.weight BF16 10240x4x1 1654664284 1654746204 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.dt_bias BF16 48 99212340 99212436 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 99143220 99147060 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 99147060 99150900 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 99150900 99212340 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 99077940 99081780 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2117096364 2117100204 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 99081780 99143220 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 97439540 98258740 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 98258740 99077940 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2193481484 2206588684 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 96456500 96948020 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 96948020 97439540 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 744276356 752140676 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.norm.weight BF16 128 96456244 96456500 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 87608884 88100404 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 88100404 88591924 model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.out_proj.weight U32 2560x768 4 64 affine 88591924 96456244 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.gate.biases BF16 512x40 8 64 affine 1979814716 1979855676 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.gate.scales BF16 512x40 8 64 affine 4290528604 4290569564 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.gate.weight U32 512x640 8 64 affine 86298164 87608884 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2433748652 2433799852 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 85427764 85478964 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 85478964 86298164 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2238528684 2238579884 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2238846124 2238897324 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 84608564 85427764 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 84557364 84608564 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1419676268 1419727468 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 943564836 944384036 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 84557204 84557284 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 84557284 84557364 model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2296743004 2296744284 model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 84475284 84557204 model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.hc_norm.weight BF16 10240 595973988 595994468 model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3596133484 3602687084 model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 77921684 84475284 model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 77839764 77921684 model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.hc_norm.weight BF16 10240 77819284 77839764 model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1832523036 1839076636 model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 71265684 77819284 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.gate.biases BF16 512x40 8 64 affine 1883149868 1883190828 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.gate.scales BF16 512x40 8 64 affine 69914004 69954964 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.gate.weight U32 512x640 8 64 affine 69954964 71265684 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 332422148 332473348 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3574743404 3574794604 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 69094804 69914004 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 69043604 69094804 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 930406436 930457636 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2337634380 2338453580 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 68992404 69043604 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3735695692 3735746892 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 536371476 537190676 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2602696908 2602696988 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1061858692 1061858772 model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 68991124 68992404 model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1581085276 1581167196 model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.hc_norm.weight BF16 10240 68970644 68991124 model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 62417044 68970644 model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 296179812 302733412 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 824054404 824105604 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 60727444 60778644 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 60778644 62417044 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.k_layernorm.weight BF16 128 408605876 408606132 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.q_layernorm.weight BF16 128 60727188 60727444 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_norm.weight BF16 256 60726676 60727188 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_proj.biases BF16 512x40 4 64 affine 59989396 60030356 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_proj.scales BF16 512x40 4 64 affine 60030356 60071316 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_proj.weight U32 512x320 4 64 affine 60071316 60726676 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.o_proj.biases BF16 2560x96 4 64 affine 2604377804 2604869324 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.o_proj.scales BF16 2560x96 4 64 affine 59497876 59989396 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.o_proj.weight U32 2560x768 4 64 affine 537880852 545745172 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_norm.weight BF16 256 59497364 59497876 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_proj.biases BF16 12288x40 4 64 affine 710708180 711691220 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_proj.scales BF16 12288x40 4 64 affine 42785684 43768724 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_proj.weight U32 12288x320 4 64 affine 43768724 59497364 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.v_proj.biases BF16 512x40 4 64 affine 42703764 42744724 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.v_proj.scales BF16 512x40 4 64 affine 42744724 42785684 model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.v_proj.weight U32 512x320 4 64 affine 364932100 365587460 model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 42621844 42703764 model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.hc_norm.weight BF16 10240 1271878380 1271898860 model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 36068244 42621844 model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 29514644 36068244 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.A_log BF16 48 1486757100 1486757196 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.conv1d.weight BF16 10240x4x1 1706090524 1706172444 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.dt_bias BF16 48 2075482940 2075483036 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 29510804 29514644 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1551078236 1551082076 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 2603690348 2603751788 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1732502892 1732506732 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 29445524 29449364 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 29449364 29510804 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2280230412 2281049612 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 28626324 29445524 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1187546796 1200653996 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 19778964 20270484 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 20270484 20762004 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 20762004 28626324 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.norm.weight BF16 128 126526788 126527044 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3052101052 3052592572 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 19287444 19778964 model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2124227244 2132091564 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.gate.biases BF16 512x40 8 64 affine 1708095004 1708135964 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.gate.scales BF16 512x40 8 64 affine 130095012 130135972 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.gate.weight U32 512x640 8 64 affine 17976724 19287444 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 17874324 17925524 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 17925524 17976724 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3728271596 3729090796 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2043870364 2043921564 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 17003924 17055124 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 17055124 17874324 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 16901524 16952724 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 16952724 17003924 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2034735004 2035554204 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 16901364 16901444 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 16901444 16901524 model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1180103500 1180104780 model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 16819444 16901364 model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.hc_norm.weight BF16 10240 2300799580 2300820060 model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 849690916 856244516 model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 206499988 213053588 model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 16737524 16819444 model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.hc_norm.weight BF16 10240 16717044 16737524 model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2132961964 2139515564 model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 10163444 16717044 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.A_log BF16 48 2538163820 2538163916 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.conv1d.weight BF16 10240x4x1 10081524 10163444 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.dt_bias BF16 48 1336075676 1336075772 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 10012404 10016244 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 10016244 10020084 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 10020084 10081524 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 9943284 9947124 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 9947124 9950964 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 9950964 10012404 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 9124084 9943284 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1116464612 1117283812 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3003364796 3016471996 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 8141044 8632564 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 8632564 9124084 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1602084268 1609948588 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.norm.weight BF16 128 3610664140 3610664396 model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.out_proj.weight U32 2560x768 4 64 affine 276724 8141044 model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 509993236 536207636 model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 879622436 905836836 model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2367959116 2394173516 model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3698315420 3724529820 model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3836865884 4256296284 model-00017-of-00017.safetensors language_model.model.layers.9.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 16552042 17043562 model-00017-of-00017.safetensors language_model.model.layers.9.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 17862762 18354282 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.gate.biases BF16 512x40 8 64 affine 7704682 7745642 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.gate.scales BF16 512x40 8 64 affine 8564842 8605802 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.gate.weight U32 512x640 8 64 affine 8605802 9916522 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 209002 260202 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 260202 311402 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 311402 1130602 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 106602 157802 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 157802 209002 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 17043562 17862762 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 4202 55402 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 55402 106602 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 7745642 8564842 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2842 2922 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2762 2842 model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2922 4202 model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 16470122 16552042 model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.hc_norm.weight BF16 10240 7684202 7704682 model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 9916522 16470122 model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1130602 7684202 ngram-table.safetensors ngram.biases BF16 320001536x5 4 32 affine 28800138648 32000154008 ngram-table.safetensors ngram.scales BF16 320001536x5 4 32 affine 25600123288 28800138648 ngram-table.safetensors ngram.weight U32 320001536x20 4 32 affine 408 25600123288 mtp.safetensors mtp.fc_embedding.weight BF16 2560x2560 620917036 634024236 mtp.safetensors mtp.fc_hidden.weight BF16 2560x2560 1179744812 1192852012 mtp.safetensors mtp.hyper_connection_mixer.hc_norm.weight BF16 10240 640577836 640598316 mtp.safetensors mtp.hyper_connection_mixer.input_mix_weight_down.weight BF16 320x10240 746326572 752880172 mtp.safetensors mtp.hyper_connection_mixer.input_mix_weight_up.weight BF16 10240x320 614310956 620864556 mtp.safetensors mtp.layers.0.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 80980620 81062540 mtp.safetensors mtp.layers.0.attn_hyper_connection.hc_norm.weight BF16 10240 614290476 614310956 mtp.safetensors mtp.layers.0.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 753760812 760314412 mtp.safetensors mtp.layers.0.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 634024236 640577836 mtp.safetensors mtp.layers.0.mlp.gate.biases BF16 512x40 8 64 affine 194706476 194747436 mtp.safetensors mtp.layers.0.mlp.gate.scales BF16 512x40 8 64 affine 194747436 194788396 mtp.safetensors mtp.layers.0.mlp.gate.weight U32 512x640 8 64 affine 193395756 194706476 mtp.safetensors mtp.layers.0.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 140915756 140966956 mtp.safetensors mtp.layers.0.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 614239276 614290476 mtp.safetensors mtp.layers.0.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 640649516 641468716 mtp.safetensors mtp.layers.0.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 640598316 640649516 mtp.safetensors mtp.layers.0.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 19141260 19192460 mtp.safetensors mtp.layers.0.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 752941612 753760812 mtp.safetensors mtp.layers.0.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 140045356 140096556 mtp.safetensors mtp.layers.0.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 620864556 620915756 mtp.safetensors mtp.layers.0.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 140096556 140915756 mtp.safetensors mtp.layers.0.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 140045196 140045276 mtp.safetensors mtp.layers.0.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 140045276 140045356 mtp.safetensors mtp.layers.0.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 620915756 620917036 mtp.safetensors mtp.layers.0.mlp.switch_mlp.down_proj.biases BF16 512x2560x20 4 32 affine 140966956 193395756 mtp.safetensors mtp.layers.0.mlp.switch_mlp.down_proj.scales BF16 512x2560x20 4 32 affine 641468716 693897516 mtp.safetensors mtp.layers.0.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 32 affine 1253145132 1672575532 mtp.safetensors mtp.layers.0.mlp.switch_mlp.gate_proj.biases BF16 512x640x80 4 32 affine 693897772 746326572 mtp.safetensors mtp.layers.0.mlp.switch_mlp.gate_proj.scales BF16 512x640x80 4 32 affine 20277900 72706700 mtp.safetensors mtp.layers.0.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 32 affine 760314412 1179744812 mtp.safetensors mtp.layers.0.mlp.switch_mlp.up_proj.biases BF16 512x640x80 4 32 affine 87616396 140045196 mtp.safetensors mtp.layers.0.mlp.switch_mlp.up_proj.scales BF16 512x640x80 4 32 affine 1192852012 1245280812 mtp.safetensors mtp.layers.0.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 32 affine 194808876 614239276 mtp.safetensors mtp.layers.0.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 74345100 74427020 mtp.safetensors mtp.layers.0.mlp_hyper_connection.hc_norm.weight BF16 10240 752921132 752941612 mtp.safetensors mtp.layers.0.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 81062540 87616140 mtp.safetensors mtp.layers.0.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 74427020 80980620 mtp.safetensors mtp.layers.0.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 19192460 19243660 mtp.safetensors mtp.layers.0.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 19243660 19294860 mtp.safetensors mtp.layers.0.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 72706700 74345100 mtp.safetensors mtp.layers.0.self_attn.indexer.k_layernorm.weight BF16 128 693897516 693897772 mtp.safetensors mtp.layers.0.self_attn.indexer.q_layernorm.weight BF16 128 87616140 87616396 mtp.safetensors mtp.layers.0.self_attn.k_norm.weight BF16 256 19140748 19141260 mtp.safetensors mtp.layers.0.self_attn.k_proj.biases BF16 512x40 4 64 affine 752880172 752921132 mtp.safetensors mtp.layers.0.self_attn.k_proj.scales BF16 512x40 4 64 affine 18403468 18444428 mtp.safetensors mtp.layers.0.self_attn.k_proj.weight U32 512x320 4 64 affine 18444428 19099788 mtp.safetensors mtp.layers.0.self_attn.o_proj.biases BF16 2560x96 4 64 affine 17420428 17911948 mtp.safetensors mtp.layers.0.self_attn.o_proj.scales BF16 2560x96 4 64 affine 17911948 18403468 mtp.safetensors mtp.layers.0.self_attn.o_proj.weight U32 2560x768 4 64 affine 1245280812 1253145132 mtp.safetensors mtp.layers.0.self_attn.q_norm.weight BF16 256 17419916 17420428 mtp.safetensors mtp.layers.0.self_attn.q_proj.biases BF16 12288x40 4 64 affine 708236 1691276 mtp.safetensors mtp.layers.0.self_attn.q_proj.scales BF16 12288x40 4 64 affine 19294860 20277900 mtp.safetensors mtp.layers.0.self_attn.q_proj.weight U32 12288x320 4 64 affine 1691276 17419916 mtp.safetensors mtp.layers.0.self_attn.v_proj.biases BF16 512x40 4 64 affine 19099788 19140748 mtp.safetensors mtp.layers.0.self_attn.v_proj.scales BF16 512x40 4 64 affine 11916 52876 mtp.safetensors mtp.layers.0.self_attn.v_proj.weight U32 512x320 4 64 affine 52876 708236 mtp.safetensors mtp.pre_fc_norm_embedding.weight BF16 2560 6796 11916 mtp.safetensors mtp.pre_fc_norm_hidden.weight BF16 10240 194788396 194808876