From 4c83ac03604ad4f3385ca0c8b7b28e1e4c031822 Mon Sep 17 00:00:00 2001 From: Georg Bauer Date: Thu, 3 Sep 2026 19:15:53 +0200 Subject: [PATCH] Prepare Qwen3.8 artifact set --- Cargo.toml | 5 + .../qwen38-flash-next-bare-speed-tensors.tsv | 2528 +++++++++++++++++ .../models/qwen38-flash-next-bare-speed.json | 115 + docs/QWEN38_ARTIFACTS.md | 81 + tools/qwen38-artifacts.rs | 1250 ++++++++ 5 files changed, 3979 insertions(+) create mode 100644 assets/models/qwen38-flash-next-bare-speed-tensors.tsv create mode 100644 assets/models/qwen38-flash-next-bare-speed.json create mode 100644 docs/QWEN38_ARTIFACTS.md create mode 100644 tools/qwen38-artifacts.rs diff --git a/Cargo.toml b/Cargo.toml index 811a757..66c665e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "ds4-server" version = "0.1.0" edition = "2024" rust-version = "1.97" +default-run = "ds4-server" description = "A native macOS coding-agent GUI for DwarfStar" license = "MIT" publish = false @@ -49,3 +50,7 @@ resources = ["metal", "assets/dev-brain"] [package.metadata.packager.macos] minimum-system-version = "15.0" + +[[bin]] +name = "qwen38-artifacts" +path = "tools/qwen38-artifacts.rs" diff --git a/assets/models/qwen38-flash-next-bare-speed-tensors.tsv b/assets/models/qwen38-flash-next-bare-speed-tensors.tsv new file mode 100644 index 0000000..db342d9 --- /dev/null +++ b/assets/models/qwen38-flash-next-bare-speed-tensors.tsv @@ -0,0 +1,2528 @@ +file name dtype shape quant_bits group_size quant_mode data_start data_end +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3801091950 3827306350 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2018512750 2044727150 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736750 4666167150 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 498077550 524291950 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 524291950 550506350 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3381661550 3801091950 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2490371950 2516586350 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1022365550 1048579950 +model-00001-of-00017.safetensors language_model.model.layers.0.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2044727150 2464157550 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1520439150 1546653550 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2464157550 2490371950 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1599082350 2018512750 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1572867950 1599082350 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1494224750 1520439150 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2962231150 3381661550 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2936016750 2962231150 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1048579950 1074794350 +model-00001-of-00017.safetensors language_model.model.layers.1.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1074794350 1494224750 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1546653550 1572867950 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 576720750 602935150 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3827306350 4246736750 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 471863150 498077550 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 550506350 576720750 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2516586350 2936016750 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 419434350 445648750 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 445648750 471863150 +model-00001-of-00017.safetensors language_model.model.layers.10.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3950 419434350 +model-00001-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 602935150 1022365550 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4272951312 4299165712 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3224375312 3250589712 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3303018512 3329232912 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3355447312 3381661712 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1389367312 1808797712 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1835012112 1861226512 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2359300112 2385514512 +model-00002-of-00017.safetensors language_model.model.layers.11.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3853520912 4272951312 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3250589712 3276804112 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3329232912 3355447312 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2385514512 2804944912 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3276804112 3303018512 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2333085712 2359300112 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1861226512 2280656912 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1808797712 1835012112 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2306871312 2333085712 +model-00002-of-00017.safetensors language_model.model.layers.12.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3434090512 3853520912 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3381661712 3407876112 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3407876112 3434090512 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 524292112 943722512 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2280656912 2306871312 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 498077712 524292112 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 969936912 1389367312 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 943722512 969936912 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 471863312 498077712 +model-00002-of-00017.safetensors language_model.model.layers.13.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2804944912 3224375312 +model-00002-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 445648912 471863312 +model-00002-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4112 26218512 +model-00002-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 26218512 445648912 +model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1992298368 2018512768 +model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2883587968 2909802368 +model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 4246736768 4666167168 +model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1415581568 1441795968 +model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1939869568 1966083968 +model-00003-of-00017.safetensors language_model.model.layers.14.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 498077568 917507968 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2857373568 2883587968 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194307968 4220522368 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1520439168 1939869568 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1441795968 1468010368 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1468010368 1494224768 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3355447168 3774877568 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1966083968 1992298368 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1494224768 1520439168 +model-00003-of-00017.safetensors language_model.model.layers.15.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2018512768 2437943168 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 917507968 943722368 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1389367168 1415581568 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 943722368 1363152768 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1363152768 1389367168 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4220522368 4246736768 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3774877568 4194307968 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 26218368 52432768 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 471863168 498077568 +model-00003-of-00017.safetensors language_model.model.layers.16.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2936016768 3355447168 +model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2909802368 2936016768 +model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3968 26218368 +model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 52432768 471863168 +model-00003-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2437943168 2857373568 +model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3722448902 3748663302 +model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4246736902 4272951302 +model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2778730502 2804944902 +model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4220522502 4246736902 +model-00004-of-00017.safetensors language_model.model.layers.17.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2804944902 3224375302 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2306871302 2333085702 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4168093702 4194308102 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3748663302 4168093702 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3224375302 3250589702 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3250589702 3276804102 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3276804102 3696234502 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2280656902 2306871302 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 943722502 969936902 +model-00004-of-00017.safetensors language_model.model.layers.18.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2333085702 2752516102 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4272951302 4299165702 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1835012102 1861226502 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1861226502 2280656902 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 891293702 917508102 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 917508102 943722502 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 996151302 1415581702 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 865079302 891293702 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 969936902 996151302 +model-00004-of-00017.safetensors language_model.model.layers.19.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1415581702 1835012102 +model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2752516102 2778730502 +model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194308102 4220522502 +model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 445648902 865079302 +model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3696234502 3722448902 +model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4102 26218502 +model-00004-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 26218502 445648902 +model-00005-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3748663169 3774877569 +model-00005-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2831159169 2857373569 +model-00005-of-00017.safetensors language_model.model.layers.2.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3774877569 4194307969 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4639952769 4666167169 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4613738369 4639952769 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2359299969 2778730369 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2857373569 2883587969 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3303018369 3329232769 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 917507969 1336938369 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2778730369 2804944769 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1441795969 1468010369 +model-00005-of-00017.safetensors language_model.model.layers.20.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2883587969 3303018369 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1389367169 1415581569 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1468010369 1494224769 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1913655169 2333085569 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1415581569 1441795969 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2804944769 2831159169 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3329232769 3748663169 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1336938369 1363152769 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1363152769 1389367169 +model-00005-of-00017.safetensors language_model.model.layers.21.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 4194307969 4613738369 +model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 891293569 917507969 +model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2333085569 2359299969 +model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1494224769 1913655169 +model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 419434369 445648769 +model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 445648769 471863169 +model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 471863169 891293569 +model-00005-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3969 419434369 +model-00006-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4246736908 4272951308 +model-00006-of-00017.safetensors language_model.model.layers.22.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4272951308 4299165708 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3774877708 3801092108 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3303018508 3329232908 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3827306508 4246736908 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1363152908 1389367308 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2673872908 2700087308 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2254442508 2673872908 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2857373708 2883588108 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2700087308 2726301708 +model-00006-of-00017.safetensors language_model.model.layers.23.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1835012108 2254442508 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1808797708 1835012108 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 445648908 471863308 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1389367308 1808797708 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2804944908 2831159308 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2831159308 2857373708 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2883588108 3303018508 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 917508108 943722508 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2752516108 2778730508 +model-00006-of-00017.safetensors language_model.model.layers.24.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 943722508 1363152908 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 471863308 498077708 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3748663308 3774877708 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 498077708 917508108 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730508 2804944908 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2726301708 2752516108 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3329232908 3748663308 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3801092108 3827306508 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4108 26218508 +model-00006-of-00017.safetensors language_model.model.layers.25.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 26218508 445648908 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3774877568 3801091968 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4220522368 4246736768 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736768 4666167168 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3722448768 3748663168 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3696234368 3722448768 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3801091968 4220522368 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3250589568 3276803968 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2752515968 2778730368 +model-00007-of-00017.safetensors language_model.model.layers.26.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1363152768 1782583168 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2726301568 2752515968 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2804944768 2831159168 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2831159168 3250589568 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2700087168 2726301568 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2254442368 2280656768 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2280656768 2700087168 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1782583168 1808797568 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 471863168 498077568 +model-00007-of-00017.safetensors language_model.model.layers.27.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1808797568 2228227968 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2228227968 2254442368 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 917507968 943722368 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 943722368 1363152768 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3748663168 3774877568 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2778730368 2804944768 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 498077568 917507968 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3968 26218368 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 26218368 52432768 +model-00007-of-00017.safetensors language_model.model.layers.28.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 52432768 471863168 +model-00007-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3276803968 3696234368 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3696234497 3722448897 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2359300097 2385514497 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 498077697 524292097 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4246736897 4272951297 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2385514497 2804944897 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 550506497 576720897 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3722448897 3748663297 +model-00008-of-00017.safetensors language_model.model.layers.29.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3748663297 4168093697 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3224375297 3250589697 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3670020097 3696234497 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1939869697 2359300097 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4220522497 4246736897 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1913655297 1939869697 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 602935297 1022365697 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4272951297 4299165697 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1441796097 1468010497 +model-00008-of-00017.safetensors language_model.model.layers.3.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3250589697 3670020097 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1468010497 1494224897 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194308097 4220522497 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1494224897 1913655297 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 524292097 550506497 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 576720897 602935297 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1022365697 1441796097 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 26218497 52432897 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 52432897 78647297 +model-00008-of-00017.safetensors language_model.model.layers.30.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 78647297 498077697 +model-00008-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4097 26218497 +model-00008-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4168093697 4194308097 +model-00008-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2804944897 3224375297 +model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4613738362 4639952762 +model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1468010362 1494224762 +model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3303018362 3722448762 +model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3250589562 3276803962 +model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4587523962 4613738362 +model-00009-of-00017.safetensors language_model.model.layers.31.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3722448762 4141879162 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2385514362 2411728762 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4561309562 4587523962 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2411728762 2831159162 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3276803962 3303018362 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4639952762 4666167162 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1939869562 2359299962 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1494224762 1520439162 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 524291962 550506362 +model-00009-of-00017.safetensors language_model.model.layers.32.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1520439162 1939869562 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2359299962 2385514362 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 996151162 1022365562 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1022365562 1441795962 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 1441795962 1468010362 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 498077562 524291962 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 550506362 969936762 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 969936762 996151162 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 52432762 78647162 +model-00009-of-00017.safetensors language_model.model.layers.33.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 78647162 498077562 +model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3962 26218362 +model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 26218362 52432762 +model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2831159162 3250589562 +model-00009-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 4141879162 4561309562 +model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2437943308 2464157708 +model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1861226508 1887440908 +model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3407876108 3434090508 +model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3381661708 3407876108 +model-00010-of-00017.safetensors language_model.model.layers.34.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2962231308 3381661708 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2516586508 2542800908 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2464157708 2490372108 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2542800908 2962231308 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2385514508 2411728908 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1913655308 1939869708 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3853520908 4272951308 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2411728908 2437943308 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1887440908 1913655308 +model-00010-of-00017.safetensors language_model.model.layers.35.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1966084108 2385514508 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2490372108 2516586508 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1939869708 1966084108 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3434090508 3853520908 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 943722508 969936908 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1415581708 1441796108 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1441796108 1861226508 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4272951308 4299165708 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 969936908 996151308 +model-00010-of-00017.safetensors language_model.model.layers.36.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 524292108 943722508 +model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 445648908 471863308 +model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 471863308 498077708 +model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 996151308 1415581708 +model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 498077708 524292108 +model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4108 26218508 +model-00010-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 26218508 445648908 +model-00011-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 498077559 524291959 +model-00011-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4220522359 4246736759 +model-00011-of-00017.safetensors language_model.model.layers.37.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 4246736759 4666167159 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4168093559 4194307959 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3224375159 3250589559 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1835011959 2254442359 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4194307959 4220522359 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3670019959 3696234359 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1363152759 1782583159 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2726301559 2752515959 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2752515959 2778730359 +model-00011-of-00017.safetensors language_model.model.layers.38.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3250589559 3670019959 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1808797559 1835011959 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2700087159 2726301559 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3748663159 4168093559 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3696234359 3722448759 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3722448759 3748663159 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 943722359 1363152759 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 471863159 498077559 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3198160759 3224375159 +model-00011-of-00017.safetensors language_model.model.layers.39.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 524291959 943722359 +model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2673872759 2700087159 +model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1782583159 1808797559 +model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2254442359 2673872759 +model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3959 26218359 +model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 26218359 52432759 +model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 52432759 471863159 +model-00011-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2778730359 3198160759 +model-00012-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4246736906 4272951306 +model-00012-of-00017.safetensors language_model.model.layers.4.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4272951306 4299165706 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1861226506 1887440906 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3696234506 3722448906 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3801092106 4220522506 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730506 2804944906 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3748663306 3774877706 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2831159306 3250589706 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2333085706 2359300106 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3722448906 3748663306 +model-00012-of-00017.safetensors language_model.model.layers.40.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1363152906 1782583306 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2306871306 2333085706 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1782583306 1808797706 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2359300106 2778730506 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 917508106 943722506 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1835012106 1861226506 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1887440906 2306871306 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3670020106 3696234506 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1808797706 1835012106 +model-00012-of-00017.safetensors language_model.model.layers.41.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 943722506 1363152906 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 471863306 498077706 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2804944906 2831159306 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 498077706 917508106 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 445648906 471863306 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3774877706 3801092106 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3250589706 3670020106 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4220522506 4246736906 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4106 26218506 +model-00012-of-00017.safetensors language_model.model.layers.42.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 26218506 445648906 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3801091974 3827306374 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3774877574 3801091974 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736774 4666167174 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3093303174 3119517574 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3748663174 3774877574 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1782583174 2202013574 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3224375174 3250589574 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3722448774 3748663174 +model-00013-of-00017.safetensors language_model.model.layers.43.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3827306374 4246736774 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3119517574 3145731974 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 3198160774 3224375174 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3250589574 3670019974 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 3171946374 3198160774 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 3696234374 3722448774 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2673872774 3093303174 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 3145731974 3171946374 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2202013574 2228227974 +model-00013-of-00017.safetensors language_model.model.layers.44.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 2254442374 2673872774 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3670019974 3696234374 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1756368774 1782583174 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1336938374 1756368774 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 865079174 891293574 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 891293574 917507974 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 917507974 1336938374 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2228227974 2254442374 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 419434374 445648774 +model-00013-of-00017.safetensors language_model.model.layers.45.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 445648774 865079174 +model-00013-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3974 419434374 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1415581704 1441796104 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1441796104 1468010504 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4168093704 4194308104 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4272951304 4299165704 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3250589704 3670020104 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1913655304 1939869704 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2804944904 2831159304 +model-00014-of-00017.safetensors language_model.model.layers.46.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3696234504 4115664904 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 4220522504 4246736904 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 4194308104 4220522504 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 2831159304 3250589704 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730504 2804944904 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 1468010504 1494224904 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2359300104 2778730504 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 1389367304 1415581704 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 4141879304 4168093704 +model-00014-of-00017.safetensors language_model.model.layers.47.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1939869704 2359300104 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 3670020104 3696234504 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 917508104 943722504 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 943722504 1363152904 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4115664904 4141879304 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 865079304 891293704 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 1494224904 1913655304 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 4246736904 4272951304 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 419434504 445648904 +model-00014-of-00017.safetensors language_model.model.layers.5.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 445648904 865079304 +model-00014-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1363152904 1389367304 +model-00014-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 891293704 917508104 +model-00014-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4104 419434504 +model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2778730346 2804944746 +model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 4220522346 4246736746 +model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2359299946 2778730346 +model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2831159146 2857373546 +model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2857373546 2883587946 +model-00015-of-00017.safetensors language_model.model.layers.6.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3774877546 4194307946 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 2333085546 2359299946 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 2804944746 2831159146 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 4246736746 4666167146 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 2280656746 2306871146 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 2306871146 2333085546 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 3355447146 3774877546 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2883587946 2909802346 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 2909802346 2936016746 +model-00015-of-00017.safetensors language_model.model.layers.7.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 1861226346 2280656746 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1336938346 1363152746 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 1389367146 1415581546 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 1415581546 1835011946 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 4194307946 4220522346 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 891293546 917507946 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 917507946 1336938346 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 419434346 445648746 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 1363152746 1389367146 +model-00015-of-00017.safetensors language_model.model.layers.8.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 471863146 891293546 +model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.down_proj.biases BF16 512x2560x10 4 64 affine 1835011946 1861226346 +model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.down_proj.scales BF16 512x2560x10 4 64 affine 445648746 471863146 +model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.down_proj.weight U32 512x2560x80 4 64 affine 3946 419434346 +model-00015-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight U32 512x640x320 4 64 affine 2936016746 3355447146 +model-00016-of-00017.safetensors language_model.lm_head.biases BF16 248320x40 4 64 affine 1613106348 1632971948 +model-00016-of-00017.safetensors language_model.lm_head.scales BF16 248320x40 4 64 affine 858405156 878270756 +model-00016-of-00017.safetensors language_model.lm_head.weight U32 248320x320 4 64 affine 3127426076 3445275676 +model-00016-of-00017.safetensors language_model.model.embed_tokens.biases BF16 248320x40 4 64 affine 2582830796 2602696396 +model-00016-of-00017.safetensors language_model.model.embed_tokens.scales BF16 248320x40 4 64 affine 2454382332 2474247932 +model-00016-of-00017.safetensors language_model.model.embed_tokens.weight U32 248320x320 4 64 affine 2626387660 2944237260 +model-00016-of-00017.safetensors language_model.model.hyper_connection_mixer.hc_norm.weight BF16 10240 2035595164 2035615644 +model-00016-of-00017.safetensors language_model.model.hyper_connection_mixer.input_mix_weight_down.weight BF16 320x10240 3490183452 3496737052 +model-00016-of-00017.safetensors language_model.model.hyper_connection_mixer.input_mix_weight_up.weight BF16 10240x320 2944278220 2950831820 +model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2339375180 2339457100 +model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.hc_norm.weight BF16 10240 1232367948 1232388428 +model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2523639660 2530193260 +model-00016-of-00017.safetensors language_model.model.layers.0.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 4256296284 4262849884 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.A_log BF16 48 3796949756 3796949852 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.conv1d.weight BF16 10240x4x1 3752913324 3752995244 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.dt_bias BF16 48 443286612 443286708 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2300871420 2300875260 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2626301900 2626305740 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 4282602844 4282664284 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2427187372 2427191212 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1601261132 1601264972 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1116399332 1116460772 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 3572520044 3573339244 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3452320796 3453139996 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1019912580 1033019780 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3806995804 3807487324 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3775840764 3776332284 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3788204796 3796069116 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.norm.weight BF16 128 3786480380 3786480636 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3527216764 3527708284 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3620229836 3620721356 +model-00016-of-00017.safetensors language_model.model.layers.0.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3462530332 3470394652 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.gate.biases BF16 512x40 8 64 affine 1757777772 1757818732 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.gate.scales BF16 512x40 8 64 affine 3526131324 3526172284 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.gate.weight U32 512x640 8 64 affine 1164862540 1166173260 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2359992396 2360043596 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2950831820 2950883020 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2103063724 2103882924 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3573380204 3573431404 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3554930284 3554981484 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2482564092 2483383292 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3084938860 3084990060 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2261224540 2261275740 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3001640636 3002459836 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1540541020 1540541100 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2986535276 2986535356 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1581191516 1581192796 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2626168780 2626250700 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.hc_norm.weight BF16 10240 3752995244 3753015724 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1719344492 1725898092 +model-00016-of-00017.safetensors language_model.model.layers.0.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 824597204 831150804 +model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3554797164 3554879084 +model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.hc_norm.weight BF16 10240 1405324908 1405345388 +model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2514679564 2521233164 +model-00016-of-00017.safetensors language_model.model.layers.1.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 4262942044 4269495644 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.A_log BF16 48 3522758428 3522758524 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.conv1d.weight BF16 10240x4x1 2060499532 2060581452 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.dt_bias BF16 48 2252240124 2252240220 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 677817204 677821044 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 602640484 602644324 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3725482396 3725543836 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1813155004 1813158844 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1224938492 1224942332 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3638335564 3638397004 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1813158844 1813978044 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3726414236 3727233436 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1890223420 1903330620 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3668936604 3669428124 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3018970812 3019462332 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1697335324 1705199644 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.norm.weight BF16 128 3685207964 3685208220 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3094277820 3094769340 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3636880204 3637371724 +model-00016-of-00017.safetensors language_model.model.layers.1.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3646982044 3654846364 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.gate.biases BF16 512x40 8 64 affine 2123694764 2123735724 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.gate.scales BF16 512x40 8 64 affine 2513768204 2513809164 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.gate.weight U32 512x640 8 64 affine 3019462332 3020773052 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 640473604 640524804 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3554879084 3554930284 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3588658284 3589477484 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3096407740 3096458940 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3638417484 3638468684 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3796130556 3796949756 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3796949852 3797001052 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3646879644 3646930844 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3751479468 3752298668 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3645026124 3645026204 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1778575212 1778575292 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3610664396 3610665676 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2044137036 2044218956 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.hc_norm.weight BF16 10240 1928071996 1928092476 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1928645436 1935199036 +model-00016-of-00017.safetensors language_model.model.layers.1.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3483585052 3490138652 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.conv_weight BF16 10240x4x1 1817797660 1817879580 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.key_proj.weight BF16 10240x2560 755216516 807645316 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.norm_conv.weight BF16 10240 3637443404 3637463884 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.norm_key.weight BF16 10240 3637371724 3637392204 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.norm_query.weight BF16 10240 1232388428 1232408908 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.ple_embedding.layer_multipliers I64 3 1160070020 1160070044 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.ple_embedding.ngram_heads_offsets I64 16 3620721356 3620721484 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.ple_embedding.ngram_heads_vocab_sizes I64 16 1446977132 1446977260 +model-00016-of-00017.safetensors language_model.model.layers.1.ple.value_proj.weight BF16 2560x2560 973500548 986607748 +model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2298382940 2298464860 +model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.hc_norm.weight BF16 10240 1373177356 1373197836 +model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3516204828 3522758428 +model-00016-of-00017.safetensors language_model.model.layers.10.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2156763564 2163317164 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.A_log BF16 48 3602779244 3602779340 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.conv1d.weight BF16 10240x4x1 2036305564 2036387484 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.dt_bias BF16 48 1883669724 1883669820 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1263845868 1263849708 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 3496818972 3496822812 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3016523196 3016584636 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2077162396 2077166236 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3588654444 3588658284 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2214565724 2214627164 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 927303716 928122916 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3556296044 3557115244 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3575451244 3588558444 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 727544100 728035620 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3572028524 3572520044 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2498338812 2506203132 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.norm.weight BF16 128 1601260876 1601261132 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1707603484 1708095004 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3554985324 3555476844 +model-00016-of-00017.safetensors language_model.model.layers.10.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3557606764 3565471084 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.gate.biases BF16 512x40 8 64 affine 2085854108 2085895068 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.gate.scales BF16 512x40 8 64 affine 2328807500 2328848460 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.gate.weight U32 512x640 8 64 affine 3067464972 3068775692 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3524687484 3524738684 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2267931740 2267982940 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1804293564 1805112764 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2314205260 2314256460 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2238713004 2238764204 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3513575708 3514394908 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3638284364 3638335564 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3453795356 3453846556 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3610727116 3611546316 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3453846556 3453846636 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2067206732 2067206812 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2603751788 2603753068 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3483503132 3483585052 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.hc_norm.weight BF16 10240 1601148156 1601168636 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3052592572 3059146172 +model-00016-of-00017.safetensors language_model.model.layers.10.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3453846636 3460400236 +model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3523680124 3523762044 +model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.hc_norm.weight BF16 10240 3453631516 3453651996 +model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 669184276 675737876 +model-00016-of-00017.safetensors language_model.model.layers.11.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3085809260 3092362860 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.gate.biases BF16 512x40 8 64 affine 3031498172 3031539132 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.gate.scales BF16 512x40 8 64 affine 2522738540 2522779500 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.gate.weight U32 512x640 8 64 affine 3524738684 3526049404 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3725543836 3725595036 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3453651996 3453703196 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2411130956 2411950156 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2483465212 2483516412 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2043819164 2043870364 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1357211228 1358030428 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3453703196 3453754396 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1750216812 1750268012 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3125787676 3126606876 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 4300667164 4300667244 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2279574972 2279575052 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2052452172 2052453452 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2522103660 2522185580 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.hc_norm.weight BF16 10240 3638397004 3638417484 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3077346924 3083900524 +model-00016-of-00017.safetensors language_model.model.layers.11.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3111000604 3117554204 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1633555724 1633606924 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 3092444860 3092496060 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 3092639420 3094277820 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.k_layernorm.weight BF16 128 3084887404 3084887660 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.indexer.q_layernorm.weight BF16 128 3806176092 3806176348 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_norm.weight BF16 256 4291931484 4291931996 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_proj.biases BF16 512x40 4 64 affine 3453754396 3453795356 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_proj.scales BF16 512x40 4 64 affine 1718482956 1718523916 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.k_proj.weight U32 512x320 4 64 affine 1778575292 1779230652 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.o_proj.biases BF16 2560x96 4 64 affine 912390436 912881956 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.o_proj.scales BF16 2560x96 4 64 affine 3453139996 3453631516 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.o_proj.weight U32 2560x768 4 64 affine 2411950156 2419814476 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_norm.weight BF16 256 1224917500 1224918012 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_proj.biases BF16 12288x40 4 64 affine 3645896604 3646879644 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1497980236 1498963276 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.q_proj.weight U32 12288x320 4 64 affine 1126633620 1142362260 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.v_proj.biases BF16 512x40 4 64 affine 3052039612 3052080572 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.v_proj.scales BF16 512x40 4 64 affine 1117283812 1117324772 +model-00016-of-00017.safetensors language_model.model.layers.11.self_attn.v_proj.weight U32 512x320 4 64 affine 3069482252 3070137612 +model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3536443084 3536525004 +model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.hc_norm.weight BF16 10240 3038092732 3038113212 +model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3031539132 3038092732 +model-00016-of-00017.safetensors language_model.model.layers.12.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3118373404 3124927004 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.A_log BF16 48 3462530236 3462530332 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.conv1d.weight BF16 10240x4x1 1865977292 1866059212 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.dt_bias BF16 48 4300667068 4300667164 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3030982332 3030986172 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1883317644 1883321484 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 928122916 928184356 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3645022284 3645026124 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 508816404 508820244 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1928092476 1928153916 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1296138396 1296957596 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1084405028 1085224228 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 408606132 421713332 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3022626492 3023118012 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3068837132 3069328652 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3023118012 3030982332 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.norm.weight BF16 128 1098433828 1098434084 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3797001052 3797492572 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3022134972 3022626492 +model-00016-of-00017.safetensors language_model.model.layers.12.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3621561164 3629425484 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.gate.biases BF16 512x40 8 64 affine 3018929852 3018970812 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.gate.scales BF16 512x40 8 64 affine 1601025276 1601066236 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.gate.weight U32 512x640 8 64 affine 3020773052 3022083772 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 4300779884 4300831084 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3536391884 3536443084 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3018110396 3018929596 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3016471996 3016523196 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 703831924 703883124 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2052576332 2053395532 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1708765036 1708816236 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3001589436 3001640636 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3002459836 3003279036 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2978588956 2978589036 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2252199084 2252199164 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3000707516 3000708796 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1581196636 1581278556 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.hc_norm.weight BF16 10240 2950883020 2950903500 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2612242124 2618795724 +model-00016-of-00017.safetensors language_model.model.layers.12.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3060849932 3067403532 +model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3522779004 3522860924 +model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.hc_norm.weight BF16 10240 2043942044 2043962524 +model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2604869324 2611422924 +model-00016-of-00017.safetensors language_model.model.layers.13.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1336075772 1342629372 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.A_log BF16 48 2603804268 2603804364 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.conv1d.weight BF16 10240x4x1 2602696988 2602778908 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.dt_bias BF16 48 675758356 675758452 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2507210588 2507214428 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2538159980 2538163820 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 2566999756 2567061196 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3588650604 3588654444 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1486922316 1486926156 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1682179852 1682241292 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 483632308 484451508 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2513809164 2514628364 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2339512140 2352619340 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2522185580 2522677100 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 675809652 676301172 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 443286708 451151028 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.norm.weight BF16 128 849588100 849588356 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2328899660 2329391180 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2053446732 2053938252 +model-00016-of-00017.safetensors language_model.model.layers.13.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1866601932 1874466252 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.gate.biases BF16 512x40 8 64 affine 928184356 928225316 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.gate.scales BF16 512x40 8 64 affine 2507169532 2507210492 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.gate.weight U32 512x640 8 64 affine 2275673532 2276984252 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2507077372 2507128572 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1733018732 1733069932 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1117857252 1118676452 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3554694764 3554745964 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2507026172 2507077372 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3629507404 3630326604 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1848016156 1848067356 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2496664572 2496715772 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2506206972 2507026172 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1242874188 1242874268 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3059146172 3059146252 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2481702652 2481703932 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2626305740 2626387660 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.hc_norm.weight BF16 10240 2296671324 2296691804 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2474329852 2480883452 +model-00016-of-00017.safetensors language_model.model.layers.13.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3638468684 3645022284 +model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2474247932 2474329852 +model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.hc_norm.weight BF16 10240 3060788492 3060808972 +model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2447828732 2454382332 +model-00016-of-00017.safetensors language_model.model.layers.14.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2441275132 2447828732 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.A_log BF16 48 1874466252 1874466348 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.conv1d.weight BF16 10240x4x1 3000625596 3000707516 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.dt_bias BF16 48 3111000508 3111000604 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1495590476 1495594316 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2053938252 2053942092 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3077183084 3077244524 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2506203132 2506206972 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2427191212 2427195052 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3526663804 3526725244 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1963318076 1964137276 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 676403572 677222772 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1373453836 1386561036 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2603845324 2604336844 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 839016484 839508004 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1396641388 1404505708 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.norm.weight BF16 128 1789433788 1789434044 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2395566156 2396057676 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 4298485948 4298977468 +model-00016-of-00017.safetensors language_model.model.layers.14.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2402611276 2410475596 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.gate.biases BF16 512x40 8 64 affine 1447028460 1447069420 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.gate.scales BF16 512x40 8 64 affine 3554653804 3554694764 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.gate.weight U32 512x640 8 64 affine 2394173516 2395484236 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2336763980 2336815180 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2339457100 2339508300 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2352619340 2353438540 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2036250524 2036301724 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2328848460 2328899660 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2433799852 2434619052 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2314256460 2314307660 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2279032252 2279083452 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3461711036 3462530236 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2314201340 2314201420 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1077024468 1077024548 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2497374972 2497376252 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2314043900 2314125820 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.hc_norm.weight BF16 10240 2300916220 2300936700 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2434619052 2441172652 +model-00016-of-00017.safetensors language_model.model.layers.14.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2269078972 2275632572 +model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2483383292 2483465212 +model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.hc_norm.weight BF16 10240 3620721484 3620741964 +model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1211676844 1218230444 +model-00016-of-00017.safetensors language_model.model.layers.15.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2320861260 2327414860 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.gate.biases BF16 512x40 8 64 affine 2214524764 2214565724 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.gate.scales BF16 512x40 8 64 affine 2044096076 2044137036 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.gate.weight U32 512x640 8 64 affine 846925604 848236324 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2951012636 2951063836 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2367907916 2367959116 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1847196956 1848016156 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1978882876 1978934076 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2299929180 2299980380 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2299980380 2300799580 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 703886964 703938164 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3645845404 3645896604 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2296744284 2297563484 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2295800844 2295800924 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2950960076 2950960156 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2295799564 2295800844 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3526049404 3526131324 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.hc_norm.weight BF16 10240 2060581452 2060601932 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3445275676 3451829276 +model-00016-of-00017.safetensors language_model.model.layers.15.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2490070012 2496623612 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1812977084 1813028284 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 2603753068 2603804268 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 2294161164 2295799564 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.k_layernorm.weight BF16 128 2102192556 2102192812 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.indexer.q_layernorm.weight BF16 128 2268904540 2268904796 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_norm.weight BF16 256 2281049612 2281050124 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_proj.biases BF16 512x40 4 64 affine 2300875260 2300916220 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_proj.scales BF16 512x40 4 64 affine 2298516060 2298557020 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.k_proj.weight U32 512x320 4 64 affine 2279575052 2280230412 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.o_proj.biases BF16 2560x96 4 64 affine 2279083452 2279574972 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.o_proj.scales BF16 2560x96 4 64 affine 2987026876 2987518396 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.o_proj.weight U32 2560x768 4 64 affine 3602779340 3610643660 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_norm.weight BF16 256 1263952108 1263952620 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_proj.biases BF16 12288x40 4 64 affine 2276984252 2277967292 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_proj.scales BF16 12288x40 4 64 affine 2277967292 2278950332 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.q_proj.weight U32 12288x320 4 64 affine 1759999852 1775728492 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.v_proj.biases BF16 512x40 4 64 affine 2035554204 2035595164 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.v_proj.scales BF16 512x40 4 64 affine 2275632572 2275673532 +model-00016-of-00017.safetensors language_model.model.layers.15.self_attn.v_proj.weight U32 512x320 4 64 affine 857606436 858261796 +model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2268904796 2268986716 +model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.hc_norm.weight BF16 10240 2156743084 2156763564 +model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 4269495644 4276049244 +model-00016-of-00017.safetensors language_model.model.layers.16.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2618795724 2625349324 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.A_log BF16 48 1682353932 1682354028 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.conv1d.weight BF16 10240x4x1 2238764204 2238846124 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.dt_bias BF16 48 2507210492 2507210588 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2036387484 2036391324 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1060255956 1060259796 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3092383420 3092444860 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2261220700 2261224540 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1707599644 1707603484 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2267982940 2268044380 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1536306524 1537125724 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2611422924 2612242124 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 729858340 742965540 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 593842788 594334308 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3070137612 3070629132 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2243024044 2250888364 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.norm.weight BF16 128 3103033020 3103033276 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2240310444 2240801964 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2242532524 2243024044 +model-00016-of-00017.safetensors language_model.model.layers.16.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3760060844 3767925164 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.gate.biases BF16 512x40 8 64 affine 4262901084 4262942044 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.gate.scales BF16 512x40 8 64 affine 1750123372 1750164332 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.gate.weight U32 512x640 8 64 affine 2238999724 2240310444 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3536606924 3536658124 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2238897324 2238948524 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 718736340 719555540 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1160070044 1160121244 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1540868956 1540920156 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3059150092 3059969292 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2101136476 2101187676 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2060601932 2060653132 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2480883452 2481702652 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2214627164 2214627244 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2300871260 2300871340 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2238527404 2238528684 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 625728004 625809924 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.hc_norm.weight BF16 10240 1540848476 1540868956 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2206660444 2213214044 +model-00016-of-00017.safetensors language_model.model.layers.16.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3685208220 3691761820 +model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2185533964 2185615884 +model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.hc_norm.weight BF16 10240 1883403404 1883423884 +model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2178929164 2185482764 +model-00016-of-00017.safetensors language_model.model.layers.17.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 952252292 958805892 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.A_log BF16 48 2513768108 2513768204 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.conv1d.weight BF16 10240x4x1 1757644652 1757726572 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.dt_bias BF16 48 2170019756 2170019852 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2170012076 2170015916 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2170015916 2170019756 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3001527996 3001589436 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2163317164 2163321004 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2163454636 2163458476 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2298557020 2298618460 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2497437692 2498256892 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1305804956 1306624156 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1979896732 1993003932 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1911755836 1912247356 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1717991436 1718482956 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1839076636 1846940956 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.norm.weight BF16 128 3125787420 3125787676 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2140908204 2141399724 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1663050940 1663542460 +model-00016-of-00017.safetensors language_model.model.layers.17.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2141420204 2149284524 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.gate.biases BF16 512x40 8 64 affine 2139556524 2139597484 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.gate.scales BF16 512x40 8 64 affine 1903330620 1903371580 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.gate.weight U32 512x640 8 64 affine 2139597484 2140908204 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3727233516 3727284716 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1429772908 1429824108 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3117554204 3118373404 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2103882924 2103934124 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 958887812 958939012 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1749130092 1749949292 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3646930844 3646982044 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2102193324 2102244524 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2102244524 2103063724 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2102191196 2102191276 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1218363820 1218363900 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2102191276 2102192556 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2102109276 2102191196 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.hc_norm.weight BF16 10240 1935199036 1935219516 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2094531676 2101085276 +model-00016-of-00017.safetensors language_model.model.layers.17.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2330210380 2336763980 +model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2044013980 2044095900 +model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.hc_norm.weight BF16 10240 2141399724 2141420204 +model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2103934124 2110487724 +model-00016-of-00017.safetensors language_model.model.layers.18.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2087978076 2094531676 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.A_log BF16 48 2087424764 2087424860 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.conv1d.weight BF16 10240x4x1 678365044 678446964 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.dt_bias BF16 48 2086718108 2086718204 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3059146252 3059150092 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2085850268 2085854108 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1487008076 1487069516 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1787740348 1787744188 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2053942092 2053945932 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2522677100 2522738540 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2077166236 2077985436 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3555476844 3556296044 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2957617436 2970724636 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2076056476 2076547996 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2076588956 2077080476 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3527708364 3535572684 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.norm.weight BF16 128 602640228 602640484 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 2075564956 2076056476 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2178437644 2178929164 +model-00016-of-00017.safetensors language_model.model.layers.18.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1573220956 1581085276 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.gate.biases BF16 512x40 8 64 affine 2075441980 2075482940 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.gate.scales BF16 512x40 8 64 affine 1750031212 1750072172 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.gate.weight U32 512x640 8 64 affine 3797492572 3798803292 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1373310476 1373361676 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2075390780 2075441980 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3785558524 3786377724 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2117045164 2117096364 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1866059212 1866110412 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2074571580 2075390780 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2950908876 2950960076 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2067393340 2067444540 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1740938092 1741757292 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2067391388 2067391468 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2067391980 2067392060 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2067392060 2067393340 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2067309212 2067391132 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.hc_norm.weight BF16 10240 2067288732 2067309212 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 553066772 559620372 +model-00016-of-00017.safetensors language_model.model.layers.18.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1956764476 1963318076 +model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2067206812 2067288732 +model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.hc_norm.weight BF16 10240 3610643660 3610664140 +model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2060653132 2067206732 +model-00016-of-00017.safetensors language_model.model.layers.19.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2053945932 2060499532 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.gate.biases BF16 512x40 8 64 affine 719637460 719678420 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.gate.scales BF16 512x40 8 64 affine 2052535372 2052576332 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.gate.weight U32 512x640 8 64 affine 742965540 744276260 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2626250700 2626301900 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2042944924 2042996124 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3552437964 3553257164 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3725431196 3725482396 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1787689148 1787740348 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3645026204 3645845404 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2101085276 2101136476 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2036199324 2036250524 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2051632972 2052452172 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3092362860 3092362940 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2044095900 2044095980 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2025394588 2025395868 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1358163724 1358245644 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.hc_norm.weight BF16 10240 2043921564 2043942044 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1751091052 1757644652 +model-00016-of-00017.safetensors language_model.model.layers.19.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2036391324 2042944924 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 662538516 662589716 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 2035615644 2035666844 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 4299028668 4300667068 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.k_layernorm.weight BF16 128 2034734748 2034735004 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.indexer.q_layernorm.weight BF16 128 1112856036 1112856292 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_norm.weight BF16 256 509886484 509886996 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_proj.biases BF16 512x40 4 64 affine 2012154268 2012195228 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_proj.scales BF16 512x40 4 64 affine 2025353628 2025394588 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.k_proj.weight U32 512x320 4 64 affine 2033260188 2033915548 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.o_proj.biases BF16 2560x96 4 64 affine 2011054492 2011546012 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.o_proj.scales BF16 2560x96 4 64 affine 2011662668 2012154188 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.o_proj.weight U32 2560x768 4 64 affine 2000855452 2008719772 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_norm.weight BF16 256 2067391468 2067391980 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_proj.biases BF16 12288x40 4 64 affine 3654846364 3655829404 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_proj.scales BF16 12288x40 4 64 affine 2009252252 2010235292 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.q_proj.weight U32 12288x320 4 64 affine 2567061196 2582789836 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.v_proj.biases BF16 512x40 4 64 affine 2009211292 2009252252 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.v_proj.scales BF16 512x40 4 64 affine 3060808972 3060849932 +model-00016-of-00017.safetensors language_model.model.layers.19.self_attn.v_proj.weight U32 512x320 4 64 affine 624967604 625622964 +model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2498256892 2498338812 +model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.hc_norm.weight BF16 10240 1552720476 1552740956 +model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1941855036 1948408636 +model-00016-of-00017.safetensors language_model.model.layers.2.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1994226332 2000779932 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.A_log BF16 48 4298485596 4298485692 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.conv1d.weight BF16 10240x4x1 3553257324 3553339244 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.dt_bias BF16 48 649208404 649208500 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1927576636 1927580476 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1601000956 1601004796 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1994164892 1994226332 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2000779932 2000783772 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1993495452 1993499292 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1993611932 1993673372 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 617574244 618393444 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1705199644 1706018844 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1965775676 1978882876 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1928153916 1928645436 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1334996636 1335488156 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1948900156 1956764476 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.norm.weight BF16 128 1053056900 1053057156 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1680869132 1681360652 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1927580476 1928071996 +model-00016-of-00017.safetensors language_model.model.layers.2.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2170573324 2178437644 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.gate.biases BF16 512x40 8 64 affine 1905157436 1905198396 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.gate.scales BF16 512x40 8 64 affine 1913066556 1913107516 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.gate.weight U32 512x640 8 64 affine 451151028 452461748 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1664526012 1664577212 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3069410572 3069461772 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1904338236 1905157436 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 4291880284 4291931484 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2300820060 2300871260 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1903519036 1904338236 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1903375420 1903426620 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1903467580 1903518780 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2132142764 2132961964 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2011549948 2011550028 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1883505804 1883505884 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1158715780 1158717060 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1883423884 1883505804 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.hc_norm.weight BF16 10240 2000834972 2000855452 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1857396076 1863949676 +model-00016-of-00017.safetensors language_model.model.layers.2.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2281050124 2287603724 +model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1883321484 1883403404 +model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.hc_norm.weight BF16 10240 1463959020 1463979500 +model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3729141996 3735695596 +model-00016-of-00017.safetensors language_model.model.layers.20.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2261326940 2267880540 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.A_log BF16 48 1883256108 1883256204 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.conv1d.weight BF16 10240x4x1 1935219516 1935301436 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.dt_bias BF16 48 2011549852 2011549948 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3728267756 3728271596 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 3554981484 3554985324 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1979753276 1979814716 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1883252268 1883256108 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2036301724 2036305564 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2087916636 2087978076 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1882330668 1883149868 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 987430788 988249988 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1307463836 1320571036 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3807487324 3807978844 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1993003932 1993495452 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1848118556 1855982876 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.norm.weight BF16 128 2353438540 2353438796 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1865403756 1865895276 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1866110412 1866601932 +model-00016-of-00017.safetensors language_model.model.layers.20.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1874466348 1882330668 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.gate.biases BF16 512x40 8 64 affine 2522779500 2522820460 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.gate.scales BF16 512x40 8 64 affine 3524646524 3524687484 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.gate.weight U32 512x640 8 64 affine 1864052076 1865362796 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1864000876 1864052076 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2296620124 2296671324 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3051220412 3052039612 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1863949676 1864000876 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2086718204 2086769404 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1912247356 1913066556 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1848067356 1848118556 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1857344876 1857396076 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1551082076 1551901276 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2603690268 2603690348 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2513768028 2513768108 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 625624324 625625604 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1847033116 1847115036 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.hc_norm.weight BF16 10240 1832502556 1832523036 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1825457436 1832011036 +model-00016-of-00017.safetensors language_model.model.layers.20.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2544717516 2551271116 +model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2051551052 2051632972 +model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.hc_norm.weight BF16 10240 3030986172 3031006652 +model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1817879580 1824433180 +model-00016-of-00017.safetensors language_model.model.layers.21.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 840327204 846880804 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.A_log BF16 48 1813978044 1813978140 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.conv1d.weight BF16 10240x4x1 2163321004 2163402924 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.dt_bias BF16 48 3070629132 3070629228 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1804289724 1804293564 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1813028284 1813032124 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1813032124 1813093564 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1473564140 1473567980 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 704150740 704154580 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1538069340 1538130780 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1787744188 1788563388 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1978934076 1979753276 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1791080124 1804187324 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 677273972 677765492 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1335488156 1335979676 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2077985948 2085850268 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.norm.weight BF16 128 677765652 677765908 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1777039212 1777530732 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1779230908 1779722428 +model-00016-of-00017.safetensors language_model.model.layers.21.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1805112764 1812977084 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.gate.biases BF16 512x40 8 64 affine 3752872364 3752913324 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.gate.scales BF16 512x40 8 64 affine 3602687084 3602728044 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.gate.weight U32 512x640 8 64 affine 1775728492 1777039212 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2441223932 2441275132 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2538108780 2538159980 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1759180652 1759999852 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1757726572 1757777772 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1706995484 1707046684 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2297563740 2298382940 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1750165612 1750216812 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 728906020 728957220 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1186662220 1187481420 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 648389124 648389204 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1654168844 1654168924 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1750164332 1750165612 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1749949292 1750031212 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.hc_norm.weight BF16 10240 2102006876 2102027356 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2396057676 2402611276 +model-00016-of-00017.safetensors language_model.model.layers.21.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1741757292 1748310892 +model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3003282876 3003364796 +model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.hc_norm.weight BF16 10240 1732998252 1733018732 +model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1725949292 1732502892 +model-00016-of-00017.safetensors language_model.model.layers.22.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1358245644 1364799244 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.A_log BF16 48 1719344396 1719344492 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.conv1d.weight BF16 10240x4x1 500180148 500262068 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.dt_bias BF16 48 1979855676 1979855772 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1789437884 1789441724 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2011546012 2011549852 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 689024884 689086324 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3565471084 3565474924 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1613102508 1613106348 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2170019852 2170081292 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1611259308 1612078508 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1718523916 1719343116 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2012195228 2025302428 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1717499916 1717991436 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 501638164 502129684 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2978670956 2986535276 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.norm.weight BF16 128 1779230652 1779230908 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3451829276 3452320796 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1708273516 1708765036 +model-00016-of-00017.safetensors language_model.model.layers.22.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1709635596 1717499916 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.gate.biases BF16 512x40 8 64 affine 649318676 649359636 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.gate.scales BF16 512x40 8 64 affine 2051427916 2051468876 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.gate.weight U32 512x640 8 64 affine 3514894108 3516204828 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1431504492 1431555692 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1708135964 1708187164 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2602830108 2603649308 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3022083772 3022134972 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3077244524 3077295724 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1706176284 1706995484 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3092496060 3092547260 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1706018844 1706070044 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3059969292 3060788492 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1688928108 1688928188 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1709635516 1709635596 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2950907596 2950908876 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3497724188 3497806108 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.hc_norm.weight BF16 10240 1688907628 1688928108 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2420633676 2427187276 +model-00016-of-00017.safetensors language_model.model.layers.22.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1682354028 1688907628 +model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2102027356 2102109276 +model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.hc_norm.weight BF16 10240 2314129660 2314150140 +model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2044218956 2050772556 +model-00016-of-00017.safetensors language_model.model.layers.23.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1905202236 1911755836 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.gate.biases BF16 512x40 8 64 affine 1865362796 1865403756 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.gate.scales BF16 512x40 8 64 affine 2582789836 2582830796 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.gate.weight U32 512x640 8 64 affine 3460400316 3461711036 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3775789564 3775840764 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1682302732 1682353932 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1750271852 1751091052 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1680817932 1680869132 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2043962524 2044013724 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1681360652 1682179852 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1679947532 1679998732 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1061756292 1061807492 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1679998732 1680817932 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2206660364 2206660444 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1679886012 1679886092 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2193480204 2193481484 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2051468876 2051550796 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.hc_norm.weight BF16 10240 1672001212 1672021692 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3630326604 3636880204 +model-00016-of-00017.safetensors language_model.model.layers.23.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1665447612 1672001212 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1665396412 1665447612 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 2240842924 2240894124 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 2240894124 2242532524 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.k_layernorm.weight BF16 128 2626168524 2626168780 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.indexer.q_layernorm.weight BF16 128 1903518780 1903519036 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_norm.weight BF16 256 2170081292 2170081804 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_proj.biases BF16 512x40 4 64 affine 1373361676 1373402636 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_proj.scales BF16 512x40 4 64 affine 2117100204 2117141164 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.k_proj.weight U32 512x320 4 64 affine 1210979756 1211635116 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.o_proj.biases BF16 2560x96 4 64 affine 501081268 501572788 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.o_proj.scales BF16 2560x96 4 64 affine 3753015724 3753507244 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.o_proj.weight U32 2560x768 4 64 affine 3612365516 3620229836 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_norm.weight BF16 256 1664525500 1664526012 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1662067804 1663050844 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1663542460 1664525500 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.q_proj.weight U32 12288x320 4 64 affine 460858548 476587188 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.v_proj.biases BF16 512x40 4 64 affine 1661320284 1661361244 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.v_proj.scales BF16 512x40 4 64 affine 2268044380 2268085340 +model-00016-of-00017.safetensors language_model.model.layers.23.self_attn.v_proj.weight U32 512x320 4 64 affine 1661412444 1662067804 +model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2077080476 2077162396 +model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.hc_norm.weight BF16 10240 1661299804 1661320284 +model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3753507244 3760060844 +model-00016-of-00017.safetensors language_model.model.layers.24.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1654746204 1661299804 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.A_log BF16 48 1538069244 1538069340 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.conv1d.weight BF16 10240x4x1 2238579884 2238661804 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.dt_bias BF16 48 1293793340 1293793436 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1654660444 1654664284 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1706172444 1706176284 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3796069116 3796130556 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3514890268 3514894108 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3003279036 3003282876 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3523762044 3523823484 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 3620741964 3621561164 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1612078508 1612897708 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1633688844 1646796044 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1707108124 1707599644 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1632971948 1633463468 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1919712316 1927576636 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.norm.weight BF16 128 2044013724 2044013980 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1498984012 1499475532 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1832011036 1832502556 +model-00016-of-00017.safetensors language_model.model.layers.24.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1689419708 1697284028 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.gate.biases BF16 512x40 8 64 affine 1613061548 1613102508 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.gate.scales BF16 512x40 8 64 affine 1633463468 1633504428 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.gate.weight U32 512x640 8 64 affine 2213214044 2214524764 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2261275740 2261326940 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3110898108 3110949308 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2042999964 2043819164 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3016584636 3016635836 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2296691804 2296743004 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1601265068 1602084268 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2053395532 2053446732 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1601209676 1601260876 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3611546316 3612365516 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3553257244 3553257324 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1601168636 1601168716 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1719343116 1719344396 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1601066236 1601148156 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.hc_norm.weight BF16 10240 1706070044 1706090524 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2150189484 2156743084 +model-00016-of-00017.safetensors language_model.model.layers.24.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2110491564 2117045164 +model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3752790444 3752872364 +model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.hc_norm.weight BF16 10240 1601004796 1601025276 +model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1594447356 1601000956 +model-00016-of-00017.safetensors language_model.model.layers.25.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1587893756 1594447356 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.A_log BF16 48 1697284028 1697284124 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.conv1d.weight BF16 10240x4x1 1708191084 1708273004 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.dt_bias BF16 48 3735695596 3735695692 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1203111596 1203115436 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1581192796 1581196636 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1581278556 1581339996 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1581167196 1581171036 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2496715772 2496719612 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1053057156 1053118596 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2101187676 2102006876 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1552740956 1553560156 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1553560156 1566667356 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1550586716 1551078236 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1654168924 1654660444 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2970724636 2978588956 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.norm.weight BF16 128 3786480124 3786480380 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1542230876 1542722396 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1824965916 1825457436 +model-00016-of-00017.safetensors language_model.model.layers.25.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1672021692 1679886012 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.gate.biases BF16 512x40 8 64 affine 1540807516 1540848476 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.gate.scales BF16 512x40 8 64 affine 1098331428 1098372388 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.gate.weight U32 512x640 8 64 affine 1540920156 1542230876 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1540602716 1540653916 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1540756316 1540807516 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 754229636 755048836 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1538237020 1538288220 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1539650140 1539701340 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1539701340 1540520540 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2185482764 2185533964 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1538130780 1538181980 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2149370284 2150189484 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1538067804 1538067884 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1538067884 1538067964 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1538067964 1538069244 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1537125724 1537207644 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.hc_norm.weight BF16 10240 1529711964 1529732444 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1522666844 1529220444 +model-00016-of-00017.safetensors language_model.model.layers.25.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3096458940 3103012540 +model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1522584844 1522666764 +model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.hc_norm.weight BF16 10240 2206588684 2206609164 +model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1514556684 1521110284 +model-00016-of-00017.safetensors language_model.model.layers.26.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1646796044 1653349644 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.A_log BF16 48 1514556588 1514556684 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.conv1d.weight BF16 10240x4x1 3496737052 3496818972 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.dt_bias BF16 48 1514556492 1514556588 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 3514394908 3514398748 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1513897292 1513901132 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 704044500 704105940 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1513893452 1513897292 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 912881956 912885796 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1707046684 1707108124 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1499967052 1500786252 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 508902164 509721364 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1500786252 1513893452 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1499475532 1499967052 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2087425116 2087916636 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1439112812 1446977132 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.norm.weight BF16 128 1498983756 1498984012 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1496997196 1497488716 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1497488716 1497980236 +model-00016-of-00017.safetensors language_model.model.layers.26.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3829001564 3836865884 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.gate.biases BF16 512x40 8 64 affine 2269038012 2269078972 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.gate.scales BF16 512x40 8 64 affine 1495549516 1495590476 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.gate.weight U32 512x640 8 64 affine 1495594316 1496905036 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1494576716 1494627916 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1494627916 1494679116 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1494730316 1495549516 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1883567324 1883618524 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2025302428 2025353628 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2338453580 2339272780 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3596082284 3596133484 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1487069516 1487120716 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1487120716 1487939916 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3460400236 3460400316 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3767925164 3767925244 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1486839116 1486840396 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1486675180 1486757100 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.hc_norm.weight BF16 10240 3069461772 3069482252 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1473567980 1480121580 +model-00016-of-00017.safetensors language_model.model.layers.26.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1467010540 1473564140 +model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1465617900 1465699820 +model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.hc_norm.weight BF16 10240 1463887340 1463907820 +model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1457309420 1463863020 +model-00016-of-00017.safetensors language_model.model.layers.27.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3070629484 3077183084 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.gate.biases BF16 512x40 8 64 affine 1448543980 1448584940 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.gate.scales BF16 512x40 8 64 affine 1457268460 1457309420 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.gate.weight U32 512x640 8 64 affine 2327414860 2328725580 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1661361244 1661412444 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1446977260 1447028460 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1447724780 1448543980 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1439061612 1439112812 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2011611468 2011662668 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1748310892 1749130092 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1857293676 1857344876 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1779773628 1779824828 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1438242412 1439061612 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2300871340 2300871420 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2441172652 2441172732 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1488021836 1488023116 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1438160492 1438242412 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.hc_norm.weight BF16 10240 2507128572 2507149052 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1431606892 1438160492 +model-00016-of-00017.safetensors language_model.model.layers.27.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1883669820 1890223420 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 2252240220 2252291420 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 1358112444 1358163644 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1429866092 1431504492 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.k_layernorm.weight BF16 128 1429865836 1429866092 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.indexer.q_layernorm.weight BF16 128 1429865580 1429865836 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_norm.weight BF16 256 1429865068 1429865580 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_proj.biases BF16 512x40 4 64 affine 1295605916 1295646876 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_proj.scales BF16 512x40 4 64 affine 1429824108 1429865068 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.k_proj.weight U32 512x320 4 64 affine 2410475596 2411130956 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.o_proj.biases BF16 2560x96 4 64 affine 1422236268 1422727788 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1429281388 1429772908 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.o_proj.weight U32 2560x768 4 64 affine 2252291420 2260155740 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_norm.weight BF16 256 2102192812 2102193324 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_proj.biases BF16 12288x40 4 64 affine 3727284716 3728267756 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1421253228 1422236268 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.q_proj.weight U32 12288x320 4 64 affine 3497806108 3513534748 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.v_proj.biases BF16 512x40 4 64 affine 1419471468 1419512428 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.v_proj.scales BF16 512x40 4 64 affine 2240801964 2240842924 +model-00016-of-00017.safetensors language_model.model.layers.27.self_attn.v_proj.weight U32 512x320 4 64 affine 1419727468 1420382828 +model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2395484236 2395566156 +model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.hc_norm.weight BF16 10240 1581171036 1581191516 +model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1412917868 1419471468 +model-00016-of-00017.safetensors language_model.model.layers.28.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1406364268 1412917868 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.A_log BF16 48 2269037916 2269038012 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.conv1d.weight BF16 10240x4x1 1406237548 1406319468 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.dt_bias BF16 48 2521284364 2521284460 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1406168428 1406172268 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1406172268 1406176108 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1406176108 1406237548 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3490138652 3490142492 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1406164588 1406168428 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1540694876 1540756316 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1404505708 1405324908 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1405345388 1406164588 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1271898940 1285006140 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 831152084 831643604 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1688928188 1689419708 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3103033276 3110897596 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.norm.weight BF16 128 1396641132 1396641388 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 677821044 678312564 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2067526460 2068017980 +model-00016-of-00017.safetensors language_model.model.layers.28.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1388776812 1396641132 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.gate.biases BF16 512x40 8 64 affine 1387380236 1387421196 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.gate.scales BF16 512x40 8 64 affine 1387421292 1387462252 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.gate.weight U32 512x640 8 64 affine 1387462252 1388772972 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1373197836 1373249036 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1373402636 1373453836 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1386561036 1387380236 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1816927260 1816978460 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 649359636 649410836 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3523823484 3524642684 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1364882956 1364934156 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1364934156 1364985356 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1365804556 1366623756 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1540541100 1540541180 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1358163644 1358163724 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1364799244 1364800524 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1358030428 1358112348 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.hc_norm.weight BF16 10240 1357149788 1357170268 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1366623756 1373177356 +model-00016-of-00017.safetensors language_model.model.layers.28.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3798803292 3805356892 +model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1349203548 1349285468 +model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.hc_norm.weight BF16 10240 1349183068 1349203548 +model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3589477484 3596031084 +model-00016-of-00017.safetensors language_model.model.layers.29.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1342629468 1349183068 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.A_log BF16 48 1663050844 1663050940 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.conv1d.weight BF16 10240x4x1 1633606924 1633688844 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.dt_bias BF16 48 1342629372 1342629468 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1750268012 1750271852 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1335979676 1335983516 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1883256204 1883317644 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1334497436 1334501276 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1334501276 1334505116 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1813093564 1813155004 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1306644636 1307463836 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1320571036 1321390236 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1321390236 1334497436 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1297449116 1297940636 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 3031006652 3031498172 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1297940636 1305804956 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.norm.weight BF16 128 2067391132 2067391388 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1295646876 1296138396 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1296957596 1297449116 +model-00016-of-00017.safetensors language_model.model.layers.29.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1449404140 1457268460 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.gate.biases BF16 512x40 8 64 affine 1295483036 1295523996 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.gate.scales BF16 512x40 8 64 affine 1601168716 1601209676 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.gate.weight U32 512x640 8 64 affine 3553343084 3554653804 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1293742140 1293793340 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1293793436 1293844636 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1293844636 1294663836 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 4262849884 4262901084 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3776332284 3776383484 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3000708796 3001527996 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1285007420 1285058620 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3110949308 3111000508 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1292922940 1293742140 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1709635436 1709635516 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1271898860 1271898940 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1285006140 1285007420 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2215118764 2215200684 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.hc_norm.weight BF16 10240 1263952620 1263973100 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1566667356 1573220956 +model-00016-of-00017.safetensors language_model.model.layers.29.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2117141164 2123694764 +model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1263870188 1263952108 +model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.hc_norm.weight BF16 10240 1263849708 1263870188 +model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1257292268 1263845868 +model-00016-of-00017.safetensors language_model.model.layers.3.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1242874268 1249427868 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.gate.biases BF16 512x40 8 64 affine 3588609644 3588650604 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.gate.scales BF16 512x40 8 64 affine 1242833228 1242874188 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.gate.weight U32 512x640 8 64 affine 3784247804 3785558524 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1241092428 1241143628 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1241143628 1241194828 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1241194828 1242014028 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 343788804 343840004 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2000783772 2000834972 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1232408908 1233228108 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1224943948 1224995148 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 4300667244 4300718444 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1224995148 1225814348 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1224942412 1224942492 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1224942588 1224942668 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1224942668 1224943948 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3725349276 3725431196 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.hc_norm.weight BF16 10240 1224918012 1224938492 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1218363900 1224917500 +model-00016-of-00017.safetensors language_model.model.layers.3.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2314307660 2320861260 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 678312564 678363764 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 1218312620 1218363820 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 3094769340 3096407740 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.k_layernorm.weight BF16 128 1218312364 1218312620 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.indexer.q_layernorm.weight BF16 128 1211676588 1211676844 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_norm.weight BF16 256 1211635116 1211635628 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_proj.biases BF16 512x40 4 64 affine 1824924956 1824965916 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_proj.scales BF16 512x40 4 64 affine 2481703932 2481744892 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.k_proj.weight U32 512x320 4 64 affine 2050772556 2051427916 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.o_proj.biases BF16 2560x96 4 64 affine 1201637036 1202128556 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1202620076 1203111596 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.o_proj.weight U32 2560x768 4 64 affine 1203115436 1210979756 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_norm.weight BF16 256 1364800524 1364801036 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1777592172 1778575212 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1200653996 1201637036 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.q_proj.weight U32 12288x320 4 64 affine 3536658124 3552386764 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.v_proj.biases BF16 512x40 4 64 affine 704105940 704146900 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.v_proj.scales BF16 512x40 4 64 affine 1903426620 1903467580 +model-00016-of-00017.safetensors language_model.model.layers.3.self_attn.v_proj.weight U32 512x320 4 64 affine 1061079076 1061734436 +model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1486840396 1486922316 +model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.hc_norm.weight BF16 10240 3052080572 3052101052 +model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 912885796 919439396 +model-00016-of-00017.safetensors language_model.model.layers.30.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1077843748 1084397348 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.A_log BF16 48 1187546700 1187546796 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.conv1d.weight BF16 10240x4x1 1419594348 1419676268 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.dt_bias BF16 48 1224942492 1224942588 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1187481420 1187485260 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2950903500 2950907340 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1187485260 1187546700 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1186658380 1186662220 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3751475628 3751479468 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3788139516 3788200956 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1163531340 1164350540 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1166173260 1166992460 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1166996300 1180103500 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2214627244 2215118764 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1161964524 1162456044 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1542722396 1550586716 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.norm.weight BF16 128 4298485692 4298485948 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1162548300 1163039820 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1163039820 1163531340 +model-00016-of-00017.safetensors language_model.model.layers.30.linear_attn.out_proj.weight U32 2560x768 4 64 affine 4282664284 4290528604 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.gate.biases BF16 512x40 8 64 affine 1162507340 1162548300 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.gate.scales BF16 512x40 8 64 affine 1846992156 1847033116 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.gate.weight U32 512x640 8 64 affine 1465699820 1467010540 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1161913324 1161964524 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1162456044 1162507244 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1060259796 1061078996 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2132091564 2132142764 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1161862124 1161913324 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2329391180 2330210380 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1160940524 1160991724 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2267880540 2267931740 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1160991724 1161810924 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 434988212 434988292 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1160121244 1160121324 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3573431404 3573432684 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1159003780 1159085700 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.hc_norm.weight BF16 10240 1158932100 1158952580 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 632936964 639490564 +model-00016-of-00017.safetensors language_model.model.layers.30.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1529752924 1536306524 +model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1158850180 1158932100 +model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.hc_norm.weight BF16 10240 1061734436 1061754916 +model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 453813428 460367028 +model-00016-of-00017.safetensors language_model.model.layers.31.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1152162180 1158715780 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.gate.biases BF16 512x40 8 64 affine 1033105540 1033146500 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.gate.scales BF16 512x40 8 64 affine 1150810500 1150851460 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.gate.weight U32 512x640 8 64 affine 1150851460 1152162180 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3786377724 3786428924 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 537682452 537733652 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1149991300 1150810500 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1336024476 1336075676 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1149940020 1149991220 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 365587556 366406756 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1539107420 1539158620 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1149069620 1149120820 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1149120820 1149940020 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1149048980 1149049060 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1149049060 1149049140 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2951011356 2951012636 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3496823068 3496904988 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.hc_norm.weight BF16 10240 2252219644 2252240124 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1142413460 1148967060 +model-00016-of-00017.safetensors language_model.model.layers.31.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1069866452 1076420052 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 1126582420 1126633620 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 1142362260 1142413460 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1789441724 1791080124 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.k_layernorm.weight BF16 128 1126582084 1126582340 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.indexer.q_layernorm.weight BF16 128 1125721668 1125721924 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_norm.weight BF16 256 988290948 988291460 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_proj.biases BF16 512x40 4 64 affine 3092598460 3092639420 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_proj.scales BF16 512x40 4 64 affine 1117816292 1117857252 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.k_proj.weight U32 512x320 4 64 affine 3017455036 3018110396 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.o_proj.biases BF16 2560x96 4 64 affine 1104991620 1105483140 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1117324772 1117816292 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.o_proj.weight U32 2560x768 4 64 affine 3821137244 3829001564 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_norm.weight BF16 256 3110897596 3110898108 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1113614052 1114597092 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1114597092 1115580132 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.q_proj.weight U32 12288x320 4 64 affine 2551271116 2566999756 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.v_proj.biases BF16 512x40 4 64 affine 552349972 552390932 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.v_proj.scales BF16 512x40 4 64 affine 1496905036 1496945996 +model-00016-of-00017.safetensors language_model.model.layers.31.self_attn.v_proj.weight U32 512x320 4 64 affine 1112958692 1113614052 +model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1112876772 1112958692 +model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.hc_norm.weight BF16 10240 1112856292 1112876772 +model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 626383364 632936964 +model-00016-of-00017.safetensors language_model.model.layers.32.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1106302436 1112856036 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.A_log BF16 48 1106302340 1106302436 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.conv1d.weight BF16 10240x4x1 2260155740 2260237660 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.dt_bias BF16 48 1104991524 1104991620 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1098434084 1098437924 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2042996124 2042999964 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1883190828 1883252268 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1084397348 1084401188 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1084401188 1084405028 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1098372388 1098433828 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1077024548 1077843748 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2625349324 2626168524 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2300936700 2314043900 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1076532948 1077024468 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1006313860 1006805380 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2185615884 2193480204 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.norm.weight BF16 128 1076471252 1076471508 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1856802076 1857293596 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3526725244 3527216764 +model-00016-of-00017.safetensors language_model.model.layers.32.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1285058620 1292922940 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.gate.biases BF16 512x40 8 64 affine 2496623612 2496664572 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.gate.scales BF16 512x40 8 64 affine 3573339244 3573380204 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.gate.weight U32 512x640 8 64 affine 1814797340 1816108060 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1076420052 1076471252 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1431555692 1431606892 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1813978140 1814797340 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2238948524 2238999724 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1061858772 1061909972 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1068514772 1069333972 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2206609164 2206660364 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1061807492 1061858692 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3522860924 3523680124 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1059723396 1059723476 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1061078996 1061079076 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1061754916 1061756196 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 3069328652 3069410572 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.hc_norm.weight BF16 10240 3522758524 3522779004 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1053118596 1059672196 +model-00016-of-00017.safetensors language_model.model.layers.32.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1046503300 1053056900 +model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1033019780 1033101700 +model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.hc_norm.weight BF16 10240 1046431620 1046452100 +model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1039826820 1046380420 +model-00016-of-00017.safetensors language_model.model.layers.33.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2287607564 2294161164 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.A_log BF16 48 1061756196 1061756292 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.conv1d.weight BF16 10240x4x1 386733236 386815156 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.dt_bias BF16 48 1633555628 1633555724 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1789434044 1789437884 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 2149366444 2149370284 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1033207940 1033269380 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1033101700 1033105540 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3788200956 3788204796 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1033146500 1033207940 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2336815180 2337634380 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2481744892 2482564092 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1006805380 1019912580 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1005822340 1006313860 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 1539158620 1539650140 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1264014060 1271878380 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.norm.weight BF16 128 1824924700 1824924956 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1059723476 1060214996 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 997466500 997958020 +model-00016-of-00017.safetensors language_model.model.layers.33.linear_attn.out_proj.weight U32 2560x768 4 64 affine 997958020 1005822340 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.gate.biases BF16 512x40 8 64 affine 988249988 988290948 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.gate.scales BF16 512x40 8 64 affine 3513534748 3513575708 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.gate.weight U32 512x640 8 64 affine 988291460 989602180 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1993499292 1993550492 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3084887660 3084938860 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3496904988 3497724188 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 973449348 973500548 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3092547260 3092598460 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 986611588 987430788 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1804238524 1804289724 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 973398148 973449348 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3535572684 3536391884 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 973377412 973377492 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 973397972 973398052 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1159085700 1159086980 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1865895372 1865977292 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.hc_norm.weight BF16 10240 973356932 973377412 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2483516412 2490070012 +model-00016-of-00017.safetensors language_model.model.layers.33.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 966803332 973356932 +model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2978589036 2978670956 +model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.hc_norm.weight BF16 10240 973377492 973397972 +model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 958939012 965492612 +model-00016-of-00017.safetensors language_model.model.layers.34.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1180104780 1186658380 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.A_log BF16 48 1162507244 1162507340 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.conv1d.weight BF16 10240x4x1 958805892 958887812 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.dt_bias BF16 48 952252196 952252292 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 986607748 986611588 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 952248356 952252196 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 4300718444 4300779884 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 846880804 846884644 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1039822980 1039826820 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 536258836 536320276 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 928716836 929536036 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 929587236 930406436 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 930457636 943564836 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1164350540 1164842060 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 928225316 928716836 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2025395868 2033260188 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.norm.weight BF16 128 2297563484 2297563740 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1202128556 1202620076 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1529220444 1529711964 +model-00016-of-00017.safetensors language_model.model.layers.34.linear_attn.out_proj.weight U32 2560x768 4 64 affine 919439396 927303716 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.gate.biases BF16 512x40 8 64 affine 858364196 858405156 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.gate.scales BF16 512x40 8 64 affine 878270756 878311716 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.gate.weight U32 512x640 8 64 affine 878311716 879622436 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 858261796 858312996 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 858312996 858364196 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1160121324 1160940524 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1725898092 1725949292 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2163402924 2163454124 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2521284460 2522103660 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1496945996 1496997196 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1788563388 1788614588 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 856787236 857606436 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 849690756 849690836 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 849690836 849690916 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3574794604 3574795884 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2075483036 2075564956 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.hc_norm.weight BF16 10240 849670276 849690756 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2427195052 2433748652 +model-00016-of-00017.safetensors language_model.model.layers.34.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1118676548 1125230148 +model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 849588356 849670276 +model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.hc_norm.weight BF16 10240 2339354700 2339375180 +model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 719679700 726233300 +model-00016-of-00017.safetensors language_model.model.layers.35.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2538163916 2544717516 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.gate.biases BF16 512x40 8 64 affine 846884644 846925604 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.gate.scales BF16 512x40 8 64 affine 848236420 848277380 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.gate.weight U32 512x640 8 64 affine 848277380 849588100 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 3588558444 3588609644 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3685156764 3685207964 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 839508004 840327204 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3602728044 3602779244 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1633504428 1633555628 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 831643604 832462804 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1883618524 1883669724 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1158717060 1158768260 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1364985356 1365804556 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 824105604 824105684 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3527708284 3527708364 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 831150804 831152084 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1218230444 1218312364 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.hc_norm.weight BF16 10240 1498963276 1498983756 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 817500804 824054404 +model-00016-of-00017.safetensors language_model.model.layers.35.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1061909972 1068463572 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 816087684 816138884 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 817449604 817500804 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1463979500 1465617900 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.k_layernorm.weight BF16 128 2950907340 2950907596 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.indexer.q_layernorm.weight BF16 128 816001668 816001924 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_norm.weight BF16 256 2602696396 2602696908 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_proj.biases BF16 512x40 4 64 affine 1487980876 1488021836 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_proj.scales BF16 512x40 4 64 affine 1406323308 1406364268 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.k_proj.weight U32 512x320 4 64 affine 3574795884 3575451244 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.o_proj.biases BF16 2560x96 4 64 affine 807645828 808137348 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1993673372 1994164892 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.o_proj.weight U32 2560x768 4 64 affine 808137348 816001668 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_norm.weight BF16 256 807645316 807645828 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_proj.biases BF16 12288x40 4 64 affine 2260237660 2261220700 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_proj.scales BF16 12288x40 4 64 affine 3083904364 3084887404 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.q_proj.weight U32 12288x320 4 64 affine 2221754284 2237482924 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.v_proj.biases BF16 512x40 4 64 affine 2604336844 2604377804 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.v_proj.scales BF16 512x40 4 64 affine 2603649308 2603690268 +model-00016-of-00017.safetensors language_model.model.layers.35.self_attn.v_proj.weight U32 512x320 4 64 affine 1447069420 1447724780 +model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 755052676 755134596 +model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.hc_norm.weight BF16 10240 754209156 754229636 +model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2507214428 2513768028 +model-00016-of-00017.safetensors language_model.model.layers.36.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 4291931996 4298485596 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.A_log BF16 48 3735746892 3735746988 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.conv1d.weight BF16 10240x4x1 3536525004 3536606924 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.dt_bias BF16 48 1601264972 1601265068 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 2314125820 2314129660 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1905198396 1905202236 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3068775692 3068837132 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 442463572 442467412 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1708187244 1708191084 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 508840724 508902164 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1855982876 1856802076 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 728998180 729817380 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3807978844 3821086044 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 753174916 753666436 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 753666436 754157956 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 603156324 611020644 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.norm.weight BF16 128 2087424860 2087425116 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 752191876 752683396 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 752683396 753174916 +model-00016-of-00017.safetensors language_model.model.layers.36.linear_attn.out_proj.weight U32 2560x768 4 64 affine 989602180 997466500 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.gate.biases BF16 512x40 8 64 affine 728957220 728998180 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.gate.scales BF16 512x40 8 64 affine 729817380 729858340 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.gate.weight U32 512x640 8 64 affine 1609948588 1611259308 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1538185820 1538237020 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1846940956 1846992156 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3573924204 3574743404 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 509938196 509989396 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 728854820 728906020 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2268085340 2268904540 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1046380420 1046431620 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 754157956 754209156 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 728035620 728854820 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 726233300 726233380 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1708187164 1708187244 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 719678420 719679700 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 719555540 719637460 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.hc_norm.weight BF16 10240 2252199164 2252219644 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 712182740 718736340 +model-00016-of-00017.safetensors language_model.model.layers.36.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 704154580 710708180 +model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 703962580 704044500 +model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.hc_norm.weight BF16 10240 703942100 703962580 +model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 905836836 912390436 +model-00016-of-00017.safetensors language_model.model.layers.37.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1581340156 1587893756 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.A_log BF16 48 2044095980 2044096076 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.conv1d.weight BF16 10240x4x1 1486926156 1487008076 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.dt_bias BF16 48 703942004 703942100 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1740934252 1740938092 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 703938164 703942004 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 2011550028 2011611468 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 704146900 704150740 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 755048836 755052676 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1076471508 1076532948 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 689086324 689905524 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 689905524 690724724 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 690724724 703831924 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 1732506732 1732998252 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 680669044 681160564 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 681160564 689024884 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.norm.weight BF16 128 2051550796 2051551052 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1522093324 1522584844 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 2123735724 2124227244 +model-00016-of-00017.safetensors language_model.model.layers.37.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2360043596 2367907916 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.gate.biases BF16 512x40 8 64 affine 1335983516 1336024476 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.gate.scales BF16 512x40 8 64 affine 679317364 679358324 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.gate.weight U32 512x640 8 64 affine 679358324 680669044 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2441172732 2441223932 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 856244516 856295716 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 678498164 679317364 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1463907820 1463959020 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3637392204 3637443404 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1708816236 1709635436 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 678446964 678498164 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 929536036 929587236 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1664577212 1665396412 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 677765492 677765572 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 677765572 677765652 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 678363764 678365044 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 676321652 676403572 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.hc_norm.weight BF16 10240 675737876 675758356 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1033269380 1039822980 +model-00016-of-00017.safetensors language_model.model.layers.37.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 662630676 669184276 +model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 755134596 755216516 +model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.hc_norm.weight BF16 10240 655964436 655984916 +model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2215200684 2221754284 +model-00016-of-00017.safetensors language_model.model.layers.38.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 649410836 655964436 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.A_log BF16 48 649267380 649267476 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.conv1d.weight BF16 10240x4x1 1486757196 1486839116 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.dt_bias BF16 48 2427187276 2427187372 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 649212340 649216180 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1903371580 1903375420 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1679886092 1679947532 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 649208500 649212340 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2086714268 2086718108 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 3067403532 3067464972 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 648389204 649208404 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 2522820460 2523639660 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3038113212 3051220412 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 639490564 639982084 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 639982084 640473604 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 640524804 648389124 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.norm.weight BF16 128 3806176348 3806176604 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1824433180 1824924700 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 625891844 626383364 +model-00016-of-00017.safetensors language_model.model.layers.38.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1349285468 1357149788 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.gate.biases BF16 512x40 8 64 affine 625809924 625850884 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.gate.scales BF16 512x40 8 64 affine 625850884 625891844 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.gate.weight U32 512x640 8 64 affine 2298618460 2299929180 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2538057580 2538108780 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1494679116 1494730316 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2010235292 2011054492 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 625676804 625728004 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2238661804 2238713004 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3126606876 3127426076 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1068463572 1068514772 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 625625604 625676804 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 1420434028 1421253228 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 624967524 624967604 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 625622964 625623044 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 625623044 625624324 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1612979628 1613061548 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.hc_norm.weight BF16 10240 624947044 624967524 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 618393444 624947044 +model-00016-of-00017.safetensors language_model.model.layers.38.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 611020644 617574244 +model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2278950332 2279032252 +model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.hc_norm.weight BF16 10240 602644324 602664804 +model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 596086628 602640228 +model-00016-of-00017.safetensors language_model.model.layers.39.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2163458476 2170012076 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.gate.biases BF16 512x40 8 64 affine 662589716 662630676 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.gate.scales BF16 512x40 8 64 affine 596045668 596086628 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.gate.weight U32 512x640 8 64 affine 1757869932 1759180652 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 675758452 675809652 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 595994468 596045668 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 595154788 595973988 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1059672196 1059723396 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3596031084 3596082284 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 594335588 595154788 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 677222772 677273972 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1161810924 1161862124 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2033915548 2034734748 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1149991220 1149991300 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 593842708 593842788 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 594334308 594335588 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1158768260 1158850180 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.hc_norm.weight BF16 10240 3786480636 3786501116 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1781135548 1787689148 +model-00016-of-00017.safetensors language_model.model.layers.39.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 587289108 593842708 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 585548308 585599508 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 585599508 585650708 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 585650708 587289108 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.k_layernorm.weight BF16 128 585548052 585548308 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.indexer.q_layernorm.weight BF16 128 3125787164 3125787420 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_norm.weight BF16 256 2077985436 2077985948 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_proj.biases BF16 512x40 4 64 affine 1979855772 1979896732 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_proj.scales BF16 512x40 4 64 affine 585507092 585548052 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.k_proj.weight U32 512x320 4 64 affine 2086769404 2087424764 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.o_proj.biases BF16 2560x96 4 64 affine 577151252 577642772 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.o_proj.scales BF16 2560x96 4 64 affine 2986535356 2987026876 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.o_proj.weight U32 2560x768 4 64 affine 577642772 585507092 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_norm.weight BF16 256 1708273004 1708273516 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_proj.biases BF16 12288x40 4 64 affine 559620372 560603412 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_proj.scales BF16 12288x40 4 64 affine 1159086980 1160070020 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.q_proj.weight U32 12288x320 4 64 affine 560603412 576332052 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.v_proj.biases BF16 512x40 4 64 affine 2076547996 2076588956 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.v_proj.scales BF16 512x40 4 64 affine 1357170268 1357211228 +model-00016-of-00017.safetensors language_model.model.layers.39.self_attn.v_proj.weight U32 512x320 4 64 affine 552411412 553066772 +model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1419512428 1419594348 +model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.hc_norm.weight BF16 10240 552390932 552411412 +model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 832462804 839016404 +model-00016-of-00017.safetensors language_model.model.layers.4.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 545796372 552349972 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.A_log BF16 48 973398052 973398148 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.conv1d.weight BF16 10240x4x1 537798932 537880852 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.dt_bias BF16 48 744276260 744276356 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 703883124 703886964 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 537733652 537737492 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 537737492 537798932 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2287603724 2287607564 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3083900524 3083904364 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1883505884 1883567324 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1105483140 1106302340 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1115580132 1116399332 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3655829404 3668936604 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3573432684 3573924204 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2035707804 2036199324 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 357063940 364928260 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.norm.weight BF16 128 537682196 537682452 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3557115244 3557606764 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 537190676 537682196 +model-00016-of-00017.safetensors language_model.model.layers.4.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1249427948 1257292268 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.gate.biases BF16 512x40 8 64 affine 1540653916 1540694876 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.gate.scales BF16 512x40 8 64 affine 1125721924 1125762884 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.gate.weight U32 512x640 8 64 affine 816138884 817449604 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1158952580 1159003780 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 536320276 536371476 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1125762884 1126582084 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 677766004 677817204 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 536207636 536258836 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1816978460 1817797660 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1046452100 1046503300 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 509886996 509938196 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3725595036 3726414236 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1522666764 1522666844 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1224942332 1224942412 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 509885204 509886484 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 509803284 509885204 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.hc_norm.weight BF16 10240 1306624156 1306644636 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1913107516 1919661116 +model-00016-of-00017.safetensors language_model.model.layers.4.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1422727788 1429281388 +model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 509721364 509803284 +model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.hc_norm.weight BF16 10240 508820244 508840724 +model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 502211604 508765204 +model-00016-of-00017.safetensors language_model.model.layers.40.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 655984916 662538516 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.A_log BF16 48 1358112348 1358112444 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.conv1d.weight BF16 10240x4x1 502129684 502211604 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.dt_bias BF16 48 501638068 501638164 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 501634228 501638068 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1538181980 1538185820 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 3610665676 3610727116 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 2339508300 2339512140 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 3553339244 3553343084 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 501572788 501634228 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 500262068 501081268 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3016635836 3017455036 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1085224228 1098331428 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 856295716 856787236 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 2170081804 2170573324 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 492315828 500180148 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.norm.weight BF16 128 3752790188 3752790444 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 460367028 460858548 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 476587188 477078708 +model-00016-of-00017.safetensors language_model.model.layers.40.linear_attn.out_proj.weight U32 2560x768 4 64 affine 484451508 492315828 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.gate.biases BF16 512x40 8 64 affine 452461748 452502708 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.gate.scales BF16 512x40 8 64 affine 1537207644 1537248604 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.gate.weight U32 512x640 8 64 affine 452502708 453813428 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 442412372 442463572 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3786428924 3786480124 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 442467412 443286612 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1750072172 1750123372 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2514628364 2514679564 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 441593172 442412372 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 441541972 441593172 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 752140676 752191876 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 576332052 577151252 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 441541892 441541972 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1581340076 1581340156 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3483501852 3483503132 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 816005764 816087684 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.hc_norm.weight BF16 10240 3092362940 3092383420 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1480121580 1486675180 +model-00016-of-00017.safetensors language_model.model.layers.40.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 434988292 441541892 +model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 434906292 434988212 +model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.hc_norm.weight BF16 10240 434885812 434906292 +model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 428332212 434885812 +model-00016-of-00017.safetensors language_model.model.layers.41.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 421778612 428332212 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.A_log BF16 48 677765908 677766004 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.conv1d.weight BF16 10240x4x1 2067444540 2067526460 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.dt_bias BF16 48 1118676452 1118676548 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 1388772972 1388776812 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 421713332 421717172 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 421717172 421778612 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1406319468 1406323308 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 1116460772 1116464612 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 2497376252 2497437692 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 394679476 395498676 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3124967964 3125787164 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 395498676 408605876 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 3526172284 3526663804 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 711691220 712182740 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 386815156 394679476 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.norm.weight BF16 128 3070629228 3070629484 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 1125230148 1125721668 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 3752298668 3752790188 +model-00016-of-00017.safetensors language_model.model.layers.41.linear_attn.out_proj.weight U32 2560x768 4 64 affine 944384036 952248356 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.gate.biases BF16 512x40 8 64 affine 386692276 386733236 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.gate.scales BF16 512x40 8 64 affine 2035666844 2035707804 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.gate.weight U32 512x640 8 64 affine 1779824828 1781135548 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 1757818732 1757869932 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 2521233164 2521284364 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 2085895068 2086714268 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 3554745964 3554797164 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 3729090796 3729141996 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 3724529820 3725349020 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 386641076 386692276 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3821086044 3821137244 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3637465164 3638284364 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2012154188 2012154268 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 386640996 386641076 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 3637463884 3637465164 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1148967060 1149048980 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.hc_norm.weight BF16 10240 1529732444 1529752924 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 380087396 386640996 +model-00016-of-00017.safetensors language_model.model.layers.41.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 373533796 380087396 +model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 372960356 373042276 +model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.hc_norm.weight BF16 10240 676301172 676321652 +model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2951063836 2957617436 +model-00016-of-00017.safetensors language_model.model.layers.42.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 366406756 372960356 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.A_log BF16 48 848236324 848236420 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.conv1d.weight BF16 10240x4x1 2149284524 2149366444 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.dt_bias BF16 48 365587460 365587556 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 357060100 357063940 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 364928260 364932100 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1373249036 1373310476 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 509989396 509993236 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2110487724 2110491564 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 356998404 357059844 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1537248604 1538067804 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1816108060 1816927260 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 343840004 356947204 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 373042276 373533796 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 602664804 603156324 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1233228108 1241092428 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.norm.weight BF16 128 343788548 343788804 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 334941188 335432708 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 335432708 335924228 +model-00016-of-00017.safetensors language_model.model.layers.42.linear_attn.out_proj.weight U32 2560x768 4 64 affine 335924228 343788548 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.gate.biases BF16 512x40 8 64 affine 333548548 333589508 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.gate.scales BF16 512x40 8 64 affine 333589508 333630468 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.gate.weight U32 512x640 8 64 affine 333630468 334941188 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 332626948 332678148 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 332678148 332729348 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 332729348 333548548 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 332524548 332575748 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 332575748 332626948 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1551901276 1552720476 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 3552386764 3552437964 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 332473348 332524548 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 3806176604 3806995804 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1249427868 1249427948 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1857293596 1857293676 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 332420868 332422148 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 332338948 332420868 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.hc_norm.weight BF16 10240 332318468 332338948 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2353438796 2359992396 +model-00016-of-00017.safetensors language_model.model.layers.42.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 2068017980 2074571580 +model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 332236548 332318468 +model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.hc_norm.weight BF16 10240 2238506924 2238527404 +model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 325682948 332236548 +model-00016-of-00017.safetensors language_model.model.layers.43.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 319129348 325682948 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.gate.biases BF16 512x40 8 64 affine 3490142492 3490183452 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.gate.scales BF16 512x40 8 64 affine 317777668 317818628 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.gate.weight U32 512x640 8 64 affine 317818628 319129348 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2602778908 2602830108 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 316907268 316958468 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 316958468 317777668 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 4298977468 4299028668 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 316856068 316907268 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2295800924 2296620124 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 1697284124 1697335324 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 315985668 316036868 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 316036868 316856068 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 315984228 315984308 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 315984308 315984388 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 315984388 315985668 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1295523996 1295605916 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.hc_norm.weight BF16 10240 1164842060 1164862540 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 309430628 315984228 +model-00016-of-00017.safetensors language_model.model.layers.43.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 302877028 309430628 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 302733668 302784868 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 302784868 302836068 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 1964137276 1965775676 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.k_layernorm.weight BF16 128 357059844 357060100 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.indexer.q_layernorm.weight BF16 128 302733412 302733668 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_norm.weight BF16 256 296179300 296179812 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_proj.biases BF16 512x40 4 64 affine 2139515564 2139556524 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_proj.scales BF16 512x40 4 64 affine 296138340 296179300 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.k_proj.weight U32 512x320 4 64 affine 2496719612 2497374972 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.o_proj.biases BF16 2560x96 4 64 affine 287782500 288274020 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.o_proj.scales BF16 2560x96 4 64 affine 1334505116 1334996636 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.o_proj.weight U32 2560x768 4 64 affine 288274020 296138340 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_norm.weight BF16 256 287730788 287731300 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_proj.biases BF16 12288x40 4 64 affine 1521110284 1522093324 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_proj.scales BF16 12288x40 4 64 affine 286747748 287730788 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.q_proj.weight U32 12288x320 4 64 affine 3735746988 3751475628 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.v_proj.biases BF16 512x40 4 64 affine 2238465964 2238506924 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.v_proj.scales BF16 512x40 4 64 affine 1211635628 1211676588 +model-00016-of-00017.safetensors language_model.model.layers.43.self_attn.v_proj.weight U32 512x320 4 64 affine 1513901132 1514556492 +model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2052453452 2052535372 +model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.hc_norm.weight BF16 10240 286727268 286747748 +model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1935301436 1941855036 +model-00016-of-00017.safetensors language_model.model.layers.44.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 477078708 483632308 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.A_log BF16 48 286727172 286727268 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.conv1d.weight BF16 10240x4x1 286645252 286727172 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.dt_bias BF16 48 286645156 286645252 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 286637476 286641316 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 286641316 286645156 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1777530732 1777592172 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1166992460 1166996300 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 286572196 286576036 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 286576036 286637476 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1242014028 1242833228 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 272645796 273464996 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 273464996 286572196 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 270843556 271335076 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 272154276 272645796 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 3767925244 3775789564 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.norm.weight BF16 128 3496822812 3496823068 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 262487716 262979236 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1069374932 1069866452 +model-00016-of-00017.safetensors language_model.model.layers.44.linear_attn.out_proj.weight U32 2560x768 4 64 affine 262979236 270843556 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.gate.biases BF16 512x40 8 64 affine 262405796 262446756 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.gate.scales BF16 512x40 8 64 affine 262446756 262487716 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.gate.weight U32 512x640 8 64 affine 726233380 727544100 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 262354596 262405796 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1420382828 1420434028 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1294663836 1295483036 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2298464860 2298516060 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 1919661116 1919712316 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 1788614588 1789433788 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 2314150140 2314201340 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 261484196 261535396 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 261535396 262354596 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 1126582340 1126582420 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 261482836 261482916 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 261482916 261484196 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1612897708 1612979628 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.hc_norm.weight BF16 10240 1463866860 1463887340 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 254929236 261482836 +model-00016-of-00017.safetensors language_model.model.layers.44.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 248375636 254929236 +model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 3629425484 3629507404 +model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.hc_norm.weight BF16 10240 3103012540 3103033020 +model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 241822036 248375636 +model-00016-of-00017.safetensors language_model.model.layers.45.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 235268436 241822036 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.A_log BF16 48 1387421196 1387421292 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.conv1d.weight BF16 10240x4x1 235186516 235268436 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.dt_bias BF16 48 235186420 235186516 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 235178740 235182580 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 235182580 235186420 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1540541276 1540602716 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 235174900 235178740 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 816001924 816005764 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 1993550492 1993611932 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 3084990060 3085809260 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 3805356892 3806176092 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2987518396 3000625596 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 234683380 235174900 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 824105684 824597204 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 2530193260 2538057580 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.norm.weight BF16 128 234683124 234683380 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3514398748 3514890268 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 1948408636 1948900156 +model-00016-of-00017.safetensors language_model.model.layers.45.linear_attn.out_proj.weight U32 2560x768 4 64 affine 3776383484 3784247804 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.gate.biases BF16 512x40 8 64 affine 2603804364 2603845324 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.gate.scales BF16 512x40 8 64 affine 1263973100 1264014060 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.gate.weight U32 512x640 8 64 affine 965492612 966803332 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 233812724 233863924 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 1804187324 1804238524 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 233863924 234683124 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 232942324 232993524 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 287731300 287782500 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 232993524 233812724 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 649267476 649318676 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 232891124 232942324 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2419814476 2420633676 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 3553257164 3553257244 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1581339996 1581340076 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 232889844 232891124 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1364801036 1364882956 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.hc_norm.weight BF16 10240 1149049140 1149069620 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 226336244 232889844 +model-00016-of-00017.safetensors language_model.model.layers.45.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 219782644 226336244 +model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 1847115036 1847196956 +model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.hc_norm.weight BF16 10240 219762164 219782644 +model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 213208564 219762164 +model-00016-of-00017.safetensors language_model.model.layers.46.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 3691761820 3698315420 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.A_log BF16 48 1865895276 1865895372 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.conv1d.weight BF16 10240x4x1 213126644 213208564 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.dt_bias BF16 48 213126548 213126644 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 213118868 213122708 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 213122708 213126548 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 1682241292 1682302732 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 3524642684 3524646524 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 213053588 213057428 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 213057428 213118868 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 1653349644 1654168844 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 192573588 193392788 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 193392788 206499988 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 183726228 184217748 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 184217748 184709268 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 184709268 192573588 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.norm.weight BF16 128 183725972 183726228 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 182742932 183234452 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 183234452 183725972 +model-00016-of-00017.safetensors language_model.model.layers.46.linear_attn.out_proj.weight U32 2560x768 4 64 affine 1733069932 1740934252 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.gate.biases BF16 512x40 8 64 affine 1060214996 1060255956 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.gate.scales BF16 512x40 8 64 affine 182701972 182742932 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.gate.weight U32 512x640 8 64 affine 2250888364 2252199084 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 182650772 182701972 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 356947204 356998404 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 1538288220 1539107420 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 649216180 649267380 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2950960156 2951011356 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 271335076 272154276 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 508765204 508816404 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 181780372 181831572 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 181831572 182650772 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 181779012 181779092 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 839016404 839016484 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 181779092 181780372 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 181697092 181779012 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.hc_norm.weight BF16 10240 181676612 181697092 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 175123012 181676612 +model-00016-of-00017.safetensors language_model.model.layers.46.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 168569412 175123012 +model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 160623172 160705092 +model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.hc_norm.weight BF16 10240 1540520540 1540541020 +model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1098437924 1104991524 +model-00016-of-00017.safetensors language_model.model.layers.47.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 154069572 160623172 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.gate.biases BF16 512x40 8 64 affine 152717892 152758852 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.gate.scales BF16 512x40 8 64 affine 302836068 302877028 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.gate.weight U32 512x640 8 64 affine 152758852 154069572 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 151796292 151847492 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 151847492 151898692 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 151898692 152717892 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 150925892 150977092 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 545745172 545796372 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 150977092 151796292 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 150004292 150055492 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 150055492 150106692 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 150106692 150925892 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 150002852 150002932 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 150002932 150003012 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 150003012 150004292 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 149920932 150002852 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.hc_norm.weight BF16 10240 149900452 149920932 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 143346852 149900452 +model-00016-of-00017.safetensors language_model.model.layers.47.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 130239652 136793252 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 130137252 130188452 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 130188452 130239652 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 3786501116 3788139516 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.k_layernorm.weight BF16 128 3725349020 3725349276 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.indexer.q_layernorm.weight BF16 128 3018929596 3018929852 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_norm.weight BF16 256 130094500 130095012 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_proj.biases BF16 512x40 4 64 affine 2944237260 2944278220 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_proj.scales BF16 512x40 4 64 affine 1069333972 1069374932 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.k_proj.weight U32 512x320 4 64 affine 129439140 130094500 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.o_proj.biases BF16 2560x96 4 64 affine 128456100 128947620 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.o_proj.scales BF16 2560x96 4 64 affine 128947620 129439140 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.o_proj.weight U32 2560x768 4 64 affine 160705092 168569412 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_norm.weight BF16 256 2163454124 2163454636 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_proj.biases BF16 12288x40 4 64 affine 127473060 128456100 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_proj.scales BF16 12288x40 4 64 affine 2237482924 2238465964 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.q_proj.weight U32 12288x320 4 64 affine 3669428124 3685156764 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.v_proj.biases BF16 512x40 4 64 affine 127432100 127473060 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.v_proj.scales BF16 512x40 4 64 affine 1487939916 1487980876 +model-00016-of-00017.safetensors language_model.model.layers.47.self_attn.v_proj.weight U32 512x320 4 64 affine 126776740 127432100 +model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 126694820 126776740 +model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.hc_norm.weight BF16 10240 126674340 126694820 +model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3565474924 3572028524 +model-00016-of-00017.safetensors language_model.model.layers.5.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 136793252 143346852 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.A_log BF16 48 126674244 126674340 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.conv1d.weight BF16 10240x4x1 126592324 126674244 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.dt_bias BF16 48 1540541180 1540541276 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 126522948 126526788 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 126527044 126530884 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 126530884 126592324 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1463863020 1463866860 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2314201420 2314205260 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 126461508 126522948 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 125642308 126461508 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1448584940 1449404140 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3470394652 3483501852 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 2008719772 2009211292 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 117286468 117777988 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 117777988 125642308 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.norm.weight BF16 128 117286212 117286468 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 108438852 108930372 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 108930372 109421892 +model-00016-of-00017.safetensors language_model.model.layers.5.linear_attn.out_proj.weight U32 2560x768 4 64 affine 109421892 117286212 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.gate.biases BF16 512x40 8 64 affine 3124927004 3124967964 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.gate.scales BF16 512x40 8 64 affine 108397892 108438852 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.gate.weight U32 512x640 8 64 affine 4290569564 4291880284 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 107527492 107578692 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3077295724 3077346924 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 107578692 108397892 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 1779722428 1779773628 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 106657092 106708292 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 106708292 107527492 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 105786692 105837892 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 2268986716 2269037916 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 105837892 106657092 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 105786612 105786692 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 3727233436 3727233516 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 130135972 130137252 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 2328725580 2328807500 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.hc_norm.weight BF16 10240 2507149052 2507169532 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 99212532 105766132 +model-00016-of-00017.safetensors language_model.model.layers.5.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1225814348 1232367948 +model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 2339272780 2339354700 +model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.hc_norm.weight BF16 10240 105766132 105786612 +model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 4276049244 4282602844 +model-00016-of-00017.safetensors language_model.model.layers.6.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1488023116 1494576716 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.A_log BF16 48 99212436 99212532 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.conv1d.weight BF16 10240x4x1 1654664284 1654746204 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.dt_bias BF16 48 99212340 99212436 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 99143220 99147060 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 99147060 99150900 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 99150900 99212340 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 99077940 99081780 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 2117096364 2117100204 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 99081780 99143220 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 97439540 98258740 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 98258740 99077940 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 2193481484 2206588684 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 96456500 96948020 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 96948020 97439540 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 744276356 752140676 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.norm.weight BF16 128 96456244 96456500 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 87608884 88100404 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 88100404 88591924 +model-00016-of-00017.safetensors language_model.model.layers.6.linear_attn.out_proj.weight U32 2560x768 4 64 affine 88591924 96456244 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.gate.biases BF16 512x40 8 64 affine 1979814716 1979855676 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.gate.scales BF16 512x40 8 64 affine 4290528604 4290569564 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.gate.weight U32 512x640 8 64 affine 86298164 87608884 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 2433748652 2433799852 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 85427764 85478964 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 85478964 86298164 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2238528684 2238579884 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 2238846124 2238897324 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 84608564 85427764 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 84557364 84608564 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 1419676268 1419727468 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 943564836 944384036 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 84557204 84557284 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 84557284 84557364 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2296743004 2296744284 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 84475284 84557204 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.hc_norm.weight BF16 10240 595973988 595994468 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 3596133484 3602687084 +model-00016-of-00017.safetensors language_model.model.layers.6.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 77921684 84475284 +model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 77839764 77921684 +model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.hc_norm.weight BF16 10240 77819284 77839764 +model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 1832523036 1839076636 +model-00016-of-00017.safetensors language_model.model.layers.7.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 71265684 77819284 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.gate.biases BF16 512x40 8 64 affine 1883149868 1883190828 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.gate.scales BF16 512x40 8 64 affine 69914004 69954964 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.gate.weight U32 512x640 8 64 affine 69954964 71265684 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 332422148 332473348 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 3574743404 3574794604 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 69094804 69914004 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 69043604 69094804 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 930406436 930457636 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 2337634380 2338453580 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 68992404 69043604 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 3735695692 3735746892 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 536371476 537190676 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2602696908 2602696988 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 1061858692 1061858772 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 68991124 68992404 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 1581085276 1581167196 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.hc_norm.weight BF16 10240 68970644 68991124 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 62417044 68970644 +model-00016-of-00017.safetensors language_model.model.layers.7.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 296179812 302733412 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 824054404 824105604 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 60727444 60778644 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 60778644 62417044 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.k_layernorm.weight BF16 128 408605876 408606132 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.indexer.q_layernorm.weight BF16 128 60727188 60727444 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_norm.weight BF16 256 60726676 60727188 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_proj.biases BF16 512x40 4 64 affine 59989396 60030356 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_proj.scales BF16 512x40 4 64 affine 60030356 60071316 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.k_proj.weight U32 512x320 4 64 affine 60071316 60726676 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.o_proj.biases BF16 2560x96 4 64 affine 2604377804 2604869324 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.o_proj.scales BF16 2560x96 4 64 affine 59497876 59989396 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.o_proj.weight U32 2560x768 4 64 affine 537880852 545745172 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_norm.weight BF16 256 59497364 59497876 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_proj.biases BF16 12288x40 4 64 affine 710708180 711691220 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_proj.scales BF16 12288x40 4 64 affine 42785684 43768724 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.q_proj.weight U32 12288x320 4 64 affine 43768724 59497364 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.v_proj.biases BF16 512x40 4 64 affine 42703764 42744724 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.v_proj.scales BF16 512x40 4 64 affine 42744724 42785684 +model-00016-of-00017.safetensors language_model.model.layers.7.self_attn.v_proj.weight U32 512x320 4 64 affine 364932100 365587460 +model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 42621844 42703764 +model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.hc_norm.weight BF16 10240 1271878380 1271898860 +model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 36068244 42621844 +model-00016-of-00017.safetensors language_model.model.layers.8.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 29514644 36068244 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.A_log BF16 48 1486757100 1486757196 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.conv1d.weight BF16 10240x4x1 1706090524 1706172444 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.dt_bias BF16 48 2075482940 2075483036 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 29510804 29514644 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 1551078236 1551082076 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 2603690348 2603751788 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 1732502892 1732506732 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 29445524 29449364 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 29449364 29510804 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 2280230412 2281049612 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 28626324 29445524 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 1187546796 1200653996 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 19778964 20270484 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 20270484 20762004 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 20762004 28626324 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.norm.weight BF16 128 126526788 126527044 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 3052101052 3052592572 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 19287444 19778964 +model-00016-of-00017.safetensors language_model.model.layers.8.linear_attn.out_proj.weight U32 2560x768 4 64 affine 2124227244 2132091564 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.gate.biases BF16 512x40 8 64 affine 1708095004 1708135964 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.gate.scales BF16 512x40 8 64 affine 130095012 130135972 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.gate.weight U32 512x640 8 64 affine 17976724 19287444 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 17874324 17925524 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 17925524 17976724 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 3728271596 3729090796 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 2043870364 2043921564 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 17003924 17055124 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 17055124 17874324 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 16901524 16952724 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 16952724 17003924 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 2034735004 2035554204 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 16901364 16901444 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 16901444 16901524 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 1180103500 1180104780 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 16819444 16901364 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.hc_norm.weight BF16 10240 2300799580 2300820060 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 849690916 856244516 +model-00016-of-00017.safetensors language_model.model.layers.8.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 206499988 213053588 +model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 16737524 16819444 +model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.hc_norm.weight BF16 10240 16717044 16737524 +model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 2132961964 2139515564 +model-00016-of-00017.safetensors language_model.model.layers.9.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 10163444 16717044 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.A_log BF16 48 2538163820 2538163916 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.conv1d.weight BF16 10240x4x1 10081524 10163444 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.dt_bias BF16 48 1336075676 1336075772 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_a.biases BF16 48x40 4 64 affine 10012404 10016244 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_a.scales BF16 48x40 4 64 affine 10016244 10020084 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_a.weight U32 48x320 4 64 affine 10020084 10081524 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_b.biases BF16 48x40 4 64 affine 9943284 9947124 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_b.scales BF16 48x40 4 64 affine 9947124 9950964 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_b.weight U32 48x320 4 64 affine 9950964 10012404 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_qkv.biases BF16 10240x40 4 64 affine 9124084 9943284 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_qkv.scales BF16 10240x40 4 64 affine 1116464612 1117283812 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_qkv.weight U32 10240x320 4 64 affine 3003364796 3016471996 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_z.biases BF16 6144x40 4 64 affine 8141044 8632564 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_z.scales BF16 6144x40 4 64 affine 8632564 9124084 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.in_proj_z.weight U32 6144x320 4 64 affine 1602084268 1609948588 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.norm.weight BF16 128 3610664140 3610664396 +model-00016-of-00017.safetensors language_model.model.layers.9.linear_attn.out_proj.weight U32 2560x768 4 64 affine 276724 8141044 +model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.gate_proj.biases BF16 512x640x40 4 64 affine 509993236 536207636 +model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.gate_proj.scales BF16 512x640x40 4 64 affine 879622436 905836836 +model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.up_proj.biases BF16 512x640x40 4 64 affine 2367959116 2394173516 +model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.up_proj.scales BF16 512x640x40 4 64 affine 3698315420 3724529820 +model-00016-of-00017.safetensors language_model.model.layers.9.mlp.switch_mlp.up_proj.weight U32 512x640x320 4 64 affine 3836865884 4256296284 +model-00017-of-00017.safetensors language_model.model.layers.9.linear_attn.out_proj.biases BF16 2560x96 4 64 affine 16552042 17043562 +model-00017-of-00017.safetensors language_model.model.layers.9.linear_attn.out_proj.scales BF16 2560x96 4 64 affine 17862762 18354282 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.gate.biases BF16 512x40 8 64 affine 7704682 7745642 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.gate.scales BF16 512x40 8 64 affine 8564842 8605802 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.gate.weight U32 512x640 8 64 affine 8605802 9916522 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 209002 260202 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 260202 311402 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 311402 1130602 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 106602 157802 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 157802 209002 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 17043562 17862762 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 4202 55402 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 55402 106602 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 7745642 8564842 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 2842 2922 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 2762 2842 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 2922 4202 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 16470122 16552042 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.hc_norm.weight BF16 10240 7684202 7704682 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 9916522 16470122 +model-00017-of-00017.safetensors language_model.model.layers.9.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 1130602 7684202 +ngram-table.safetensors ngram.biases BF16 320001536x5 4 32 affine 28800138648 32000154008 +ngram-table.safetensors ngram.scales BF16 320001536x5 4 32 affine 25600123288 28800138648 +ngram-table.safetensors ngram.weight U32 320001536x20 4 32 affine 408 25600123288 +mtp.safetensors mtp.fc_embedding.weight BF16 2560x2560 620917036 634024236 +mtp.safetensors mtp.fc_hidden.weight BF16 2560x2560 1179744812 1192852012 +mtp.safetensors mtp.hyper_connection_mixer.hc_norm.weight BF16 10240 640577836 640598316 +mtp.safetensors mtp.hyper_connection_mixer.input_mix_weight_down.weight BF16 320x10240 746326572 752880172 +mtp.safetensors mtp.hyper_connection_mixer.input_mix_weight_up.weight BF16 10240x320 614310956 620864556 +mtp.safetensors mtp.layers.0.attn_hyper_connection.block_inject_weight.weight BF16 4x10240 80980620 81062540 +mtp.safetensors mtp.layers.0.attn_hyper_connection.hc_norm.weight BF16 10240 614290476 614310956 +mtp.safetensors mtp.layers.0.attn_hyper_connection.input_mix_weight_down.weight BF16 320x10240 753760812 760314412 +mtp.safetensors mtp.layers.0.attn_hyper_connection.input_mix_weight_up.weight BF16 10240x320 634024236 640577836 +mtp.safetensors mtp.layers.0.mlp.gate.biases BF16 512x40 8 64 affine 194706476 194747436 +mtp.safetensors mtp.layers.0.mlp.gate.scales BF16 512x40 8 64 affine 194747436 194788396 +mtp.safetensors mtp.layers.0.mlp.gate.weight U32 512x640 8 64 affine 193395756 194706476 +mtp.safetensors mtp.layers.0.mlp.shared_expert.down_proj.biases BF16 2560x10 4 64 affine 140915756 140966956 +mtp.safetensors mtp.layers.0.mlp.shared_expert.down_proj.scales BF16 2560x10 4 64 affine 614239276 614290476 +mtp.safetensors mtp.layers.0.mlp.shared_expert.down_proj.weight U32 2560x80 4 64 affine 640649516 641468716 +mtp.safetensors mtp.layers.0.mlp.shared_expert.gate_proj.biases BF16 640x40 4 64 affine 640598316 640649516 +mtp.safetensors mtp.layers.0.mlp.shared_expert.gate_proj.scales BF16 640x40 4 64 affine 19141260 19192460 +mtp.safetensors mtp.layers.0.mlp.shared_expert.gate_proj.weight U32 640x320 4 64 affine 752941612 753760812 +mtp.safetensors mtp.layers.0.mlp.shared_expert.up_proj.biases BF16 640x40 4 64 affine 140045356 140096556 +mtp.safetensors mtp.layers.0.mlp.shared_expert.up_proj.scales BF16 640x40 4 64 affine 620864556 620915756 +mtp.safetensors mtp.layers.0.mlp.shared_expert.up_proj.weight U32 640x320 4 64 affine 140096556 140915756 +mtp.safetensors mtp.layers.0.mlp.shared_expert_gate.biases BF16 1x40 4 64 affine 140045196 140045276 +mtp.safetensors mtp.layers.0.mlp.shared_expert_gate.scales BF16 1x40 4 64 affine 140045276 140045356 +mtp.safetensors mtp.layers.0.mlp.shared_expert_gate.weight U32 1x320 4 64 affine 620915756 620917036 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.down_proj.biases BF16 512x2560x20 2 64 affine 140966956 193395756 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.down_proj.scales BF16 512x2560x20 2 64 affine 641468716 693897516 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.down_proj.weight U32 512x2560x80 2 64 affine 1253145132 1672575532 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.gate_proj.biases BF16 512x640x80 2 64 affine 693897772 746326572 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.gate_proj.scales BF16 512x640x80 2 64 affine 20277900 72706700 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.gate_proj.weight U32 512x640x320 2 64 affine 760314412 1179744812 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.up_proj.biases BF16 512x640x80 2 64 affine 87616396 140045196 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.up_proj.scales BF16 512x640x80 2 64 affine 1192852012 1245280812 +mtp.safetensors mtp.layers.0.mlp.switch_mlp.up_proj.weight U32 512x640x320 2 64 affine 194808876 614239276 +mtp.safetensors mtp.layers.0.mlp_hyper_connection.block_inject_weight.weight BF16 4x10240 74345100 74427020 +mtp.safetensors mtp.layers.0.mlp_hyper_connection.hc_norm.weight BF16 10240 752921132 752941612 +mtp.safetensors mtp.layers.0.mlp_hyper_connection.input_mix_weight_down.weight BF16 320x10240 81062540 87616140 +mtp.safetensors mtp.layers.0.mlp_hyper_connection.input_mix_weight_up.weight BF16 10240x320 74427020 80980620 +mtp.safetensors mtp.layers.0.self_attn.indexer.index_qk_proj.biases BF16 640x40 8 64 affine 19192460 19243660 +mtp.safetensors mtp.layers.0.self_attn.indexer.index_qk_proj.scales BF16 640x40 8 64 affine 19243660 19294860 +mtp.safetensors mtp.layers.0.self_attn.indexer.index_qk_proj.weight U32 640x640 8 64 affine 72706700 74345100 +mtp.safetensors mtp.layers.0.self_attn.indexer.k_layernorm.weight BF16 128 693897516 693897772 +mtp.safetensors mtp.layers.0.self_attn.indexer.q_layernorm.weight BF16 128 87616140 87616396 +mtp.safetensors mtp.layers.0.self_attn.k_norm.weight BF16 256 19140748 19141260 +mtp.safetensors mtp.layers.0.self_attn.k_proj.biases BF16 512x40 4 64 affine 752880172 752921132 +mtp.safetensors mtp.layers.0.self_attn.k_proj.scales BF16 512x40 4 64 affine 18403468 18444428 +mtp.safetensors mtp.layers.0.self_attn.k_proj.weight U32 512x320 4 64 affine 18444428 19099788 +mtp.safetensors mtp.layers.0.self_attn.o_proj.biases BF16 2560x96 4 64 affine 17420428 17911948 +mtp.safetensors mtp.layers.0.self_attn.o_proj.scales BF16 2560x96 4 64 affine 17911948 18403468 +mtp.safetensors mtp.layers.0.self_attn.o_proj.weight U32 2560x768 4 64 affine 1245280812 1253145132 +mtp.safetensors mtp.layers.0.self_attn.q_norm.weight BF16 256 17419916 17420428 +mtp.safetensors mtp.layers.0.self_attn.q_proj.biases BF16 12288x40 4 64 affine 708236 1691276 +mtp.safetensors mtp.layers.0.self_attn.q_proj.scales BF16 12288x40 4 64 affine 19294860 20277900 +mtp.safetensors mtp.layers.0.self_attn.q_proj.weight U32 12288x320 4 64 affine 1691276 17419916 +mtp.safetensors mtp.layers.0.self_attn.v_proj.biases BF16 512x40 4 64 affine 19099788 19140748 +mtp.safetensors mtp.layers.0.self_attn.v_proj.scales BF16 512x40 4 64 affine 11916 52876 +mtp.safetensors mtp.layers.0.self_attn.v_proj.weight U32 512x320 4 64 affine 52876 708236 +mtp.safetensors mtp.pre_fc_norm_embedding.weight BF16 2560 6796 11916 +mtp.safetensors mtp.pre_fc_norm_hidden.weight BF16 10240 194788396 194808876 diff --git a/assets/models/qwen38-flash-next-bare-speed.json b/assets/models/qwen38-flash-next-bare-speed.json new file mode 100644 index 0000000..3f584b8 --- /dev/null +++ b/assets/models/qwen38-flash-next-bare-speed.json @@ -0,0 +1,115 @@ +{ + "format": 1, + "source": { + "repository": "Youssofal/Qwen3.8-Flash-Next-MTPLX-Bare-Speed", + "url": "https://huggingface.co/Youssofal/Qwen3.8-Flash-Next-MTPLX-Bare-Speed/tree/74559cdf34fbfc0b593de72d17e93f37fd4f9ea7", + "revision": "74559cdf34fbfc0b593de72d17e93f37fd4f9ea7", + "base_repository": "Qwen/Qwen3.8-Flash-Next", + "base_url": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/tree/de4b8e4d43b917e7706784d8bb445c9af86a3540", + "base_revision": "de4b8e4d43b917e7706784d8bb445c9af86a3540", + "license": "Qwen Community License 1.0", + "converter": "qwen38-artifacts-v1-identity" + }, + "tensor_inventory": "qwen38-flash-next-bare-speed-tensors.tsv", + "tensor_inventory_sha256": "369cdc7e53c09eaa5eff5cdcc49f0d0eab0ddc175f34965ba2ed1be2673d5858", + "config": { + "/architectures/0": "Qwen4ExpForConditionalGeneration", + "/model_type": "qwen4_exp", + "/quantization_config/bits": 4, + "/quantization_config/group_size": 64, + "/text_config/full_attention_interval": 4, + "/text_config/hc_count": 4, + "/text_config/hc_lowrank": 320, + "/text_config/hidden_size": 2560, + "/text_config/indexer_budget": 2048, + "/text_config/indexer_compress_ratio": 4, + "/text_config/indexer_head_dim": 128, + "/text_config/indexer_kv_heads": 1, + "/text_config/indexer_n_heads": 4, + "/text_config/linear_conv_kernel_dim": 4, + "/text_config/linear_key_head_dim": 128, + "/text_config/linear_num_key_heads": 16, + "/text_config/linear_num_value_heads": 48, + "/text_config/linear_value_head_dim": 128, + "/text_config/max_position_embeddings": 262144, + "/text_config/moe_intermediate_size": 640, + "/text_config/mtp_num_hidden_layers": 1, + "/text_config/ngram_size": 3, + "/text_config/ngram_vocab_size_base": 20000000, + "/text_config/num_attention_heads": 24, + "/text_config/num_experts": 512, + "/text_config/num_experts_per_tok": 10, + "/text_config/num_hidden_layers": 48, + "/text_config/num_key_value_heads": 2, + "/text_config/output_gate_type": "sigmoid", + "/text_config/partial_rotary_factor": 0.25, + "/text_config/ple_conv_kernel_size": 4, + "/text_config/ple_embed_dim": 2560, + "/text_config/rms_norm_eps": 0.000001, + "/text_config/rope_parameters/rope_theta": 10000000, + "/text_config/shared_expert_intermediate_size": 640, + "/text_config/split_ngram_parts": 128, + "/text_config/vocab_size": 248320 + }, + "runtime": { + "/arch_id": "qwen4-next", + "/mtp_contract/base_hidden_variant": "post_norm", + "/mtp_contract/concat_order": "embedding_hidden", + "/mtp_contract/hidden_variant": "post_norm", + "/mtp_contract/mtp_position_mode": "cache", + "/mtp_contract/mtp_quant_group_size": 64, + "/mtp_contract/mtp_quant_mode": "affine", + "/mtp_depth_max": 3, + "/mtp_sidecar": "prequantized-mlx-affine-qwen-moe-switch-mlx", + "/mtplx_version": "2.9.3", + "/public_model_id": "mtplx-flash-next-bare-speed", + "/sampler/temperature": 1.0, + "/sampler/top_k": 20, + "/sampler/top_p": 0.95 + }, + "representative_rows": [ + {"class":"expert","file":"model-00001-of-00017.safetensors","tensor":"language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight","row":0,"values":64,"bits":4,"group_size":64,"sha256":"0fe4ae597854a1483e2d63f07cada72d4656c4178e1d696ce4311b838ba05964"}, + {"class":"dense","file":"model-00016-of-00017.safetensors","tensor":"language_model.model.layers.0.linear_attn.in_proj_qkv.weight","row":0,"values":64,"bits":4,"group_size":64,"sha256":"59d7d7bf84e88823704d9f7b2c84e41b46f481c658ed8a485f2fd063ecf9ca34"}, + {"class":"qsa","file":"model-00016-of-00017.safetensors","tensor":"language_model.model.layers.11.self_attn.indexer.index_qk_proj.weight","row":0,"values":64,"bits":8,"group_size":64,"sha256":"c803ea77148621a5d6dfa4060a52c75f57185234a0bec4771671f69336bb8346"}, + {"class":"gdn","file":"model-00016-of-00017.safetensors","tensor":"language_model.model.layers.0.linear_attn.A_log","row":0,"values":48,"sha256":"88c53a2a04bda1d96ee1ade6fa7dfa9c49d3245b1cc1e83673f02b7a36e07a85"}, + {"class":"ple","file":"ngram-table.safetensors","tensor":"ngram.weight","row":0,"values":64,"bits":4,"group_size":32,"sha256":"2243dd9766046bb80d98e3baf5e59ba246958cfe4f340ebce8c1fff57a2810d9"}, + {"class":"mtp","file":"mtp.safetensors","tensor":"mtp.layers.0.mlp.switch_mlp.gate_proj.weight","row":0,"values":64,"bits":2,"group_size":64,"sha256":"6293cce5e1b25a4f35cb8af46e6e13b8378ec4ca84ec7f68859066dda48049cc"} + ], + "files": [ + {"path":"model-00001-of-00017.safetensors","role":"core","size":4666167150,"sha256":"a27232c9434b9d8961f198cf36f44e228b3f23a5f8d874ba4ca8af32a1b23ffc"}, + {"path":"model-00002-of-00017.safetensors","role":"core","size":4299165712,"sha256":"6268ae80950f6b935fca672fae5aa455eb3ea4f139730e54e6e5f8581faa94dc"}, + {"path":"model-00003-of-00017.safetensors","role":"core","size":4666167168,"sha256":"71a9abbd420dce567f2c06593bfee2d6cea681989d51eaef1b1c5dffd65a844b"}, + {"path":"model-00004-of-00017.safetensors","role":"core","size":4299165702,"sha256":"c8112da735bb1c8dbc4757926ab8ec1eba7a707a8767c7e43c6150586cf63a4c"}, + {"path":"model-00005-of-00017.safetensors","role":"core","size":4666167169,"sha256":"48258b931e7596c1b43c8b38c044060787ef6d85cca2c1935852bc962801f792"}, + {"path":"model-00006-of-00017.safetensors","role":"core","size":4299165708,"sha256":"f357d2d66f7b768ebb19ba464556bbe46c572cf3cd1f8226faddf9216b150d0c"}, + {"path":"model-00007-of-00017.safetensors","role":"core","size":4666167168,"sha256":"d3152e808d013e3afa4a9c391f6d8088e74a10303c89d45fb61dad7e457160f6"}, + {"path":"model-00008-of-00017.safetensors","role":"core","size":4299165697,"sha256":"1d93bd4872f338fa19416bac64ebeb05aedf2721a07ac48e37c53921252c95c1"}, + {"path":"model-00009-of-00017.safetensors","role":"core","size":4666167162,"sha256":"d22d4be580296e3d4fa5f4fe5902c8fc448d8204b6dcde4641f87b9c675930ca"}, + {"path":"model-00010-of-00017.safetensors","role":"core","size":4299165708,"sha256":"111d0166c888cabb7d6fc19c4bb7eb42e2b51946f71b4983dd750b031755637e"}, + {"path":"model-00011-of-00017.safetensors","role":"core","size":4666167159,"sha256":"73dad882cf94a5486cfe0dcda876b0f740556bc469608d2ced7a07540e699803"}, + {"path":"model-00012-of-00017.safetensors","role":"core","size":4299165706,"sha256":"a6057d5a06276461760a4f4e88991550a43abdc5f8afacef06e860f5e7843393"}, + {"path":"model-00013-of-00017.safetensors","role":"core","size":4666167174,"sha256":"ff25dc6e1c250c9b9ece83d59111a4c4055dad1b1f160546c614dcc17395b152"}, + {"path":"model-00014-of-00017.safetensors","role":"core","size":4299165704,"sha256":"d8c9de3ae28abccc8e3da09dc92fbbed4039d6d65dabb2657edebc11b2b09652"}, + {"path":"model-00015-of-00017.safetensors","role":"core","size":4666167146,"sha256":"a5438d468074c709febdeadb9f6a2db54fa9fb30a37c33de78584f28d6fa1191"}, + {"path":"model-00016-of-00017.safetensors","role":"core","size":4300831084,"sha256":"4a07409bd568cd1435641495d90c805dcb0d0e5d8dc64409273d604a7d06b342"}, + {"path":"model-00017-of-00017.safetensors","role":"core","size":18354282,"sha256":"9c064e0b8dc0054e2b453fe2b8450c287e1ede6e74bb74ecf5aaed1ef9271913"}, + {"path":"ngram-table.safetensors","role":"ple","size":32000154008,"sha256":"af7f0880c1798e60353b07cc6b0faa0d41f78734818d858ff013ab7e3bce9de5"}, + {"path":"mtp.safetensors","role":"mtp","size":1672575532,"sha256":"1bd4b2fc20a93b53f22b4cab0b14d68bc0fe666bb64e4e2f898872c30d7d26ae"}, + {"path":"model.safetensors.index.json","role":"core-index","size":284868,"sha256":"81e77f5b25bb29e25e239ae2f9f324c9cada379ee600441d6c229f1d95a13443"}, + {"path":"config.json","role":"config","size":227092,"sha256":"826302af9ff3293cfc9975c9a5a96546fa0aa9f0808505adfc30a62fef806965"}, + {"path":"generation_config.json","role":"config","size":202,"sha256":"e70c136c1b78ddc1fb0905bac8e733a4dc448d4f852a5dd75143fffc70be550e"}, + {"path":"mtplx_runtime.json","role":"config","size":2676,"sha256":"381c7666872f878542494a48a874bb0784c72a955d18cc077324d3c9e74b8cdb"}, + {"path":"chat_template.jinja","role":"tokenizer","size":8952,"sha256":"c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041"}, + {"path":"tokenizer.json","role":"tokenizer","size":12809320,"sha256":"0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3"}, + {"path":"tokenizer_config.json","role":"tokenizer","size":17928,"sha256":"b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27"}, + {"path":"vocab.json","role":"tokenizer","size":6722759,"sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"}, + {"path":"merges.txt","role":"tokenizer","size":3353259,"sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"}, + {"path":"LICENSE","role":"license","size":3235,"sha256":"a0dc422560841fd68e06d974907f8b4c709bca44a67daad2b528437bdf676c08"} + ], + "excluded": [ + {"path":"model-vision.safetensors","reason":"Vision tensors are outside the DS4Server text artifact.","size":897899145,"sha256":"5aff6983f36f52acbecc1f61f703ae7b927e74525bf3b1133ba276bb7a5b52b1"}, + {"path":"preprocessor_config.json","reason":"Vision preprocessing is outside the DS4Server text artifact.","size":390,"sha256":"27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"}, + {"path":"processor_config.json","reason":"Vision preprocessing is outside the DS4Server text artifact.","size":367,"sha256":"dcc7ce0121d6d1c514804e68e5673e04c239ad2547339ea6330289d272a748f8"}, + {"path":"video_preprocessor_config.json","reason":"Video preprocessing is outside the DS4Server text artifact.","size":385,"sha256":"7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"} + ] +} diff --git a/docs/QWEN38_ARTIFACTS.md b/docs/QWEN38_ARTIFACTS.md new file mode 100644 index 0000000..4481d8f --- /dev/null +++ b/docs/QWEN38_ARTIFACTS.md @@ -0,0 +1,81 @@ +# Qwen3.8 Flash Next Bare Speed artifacts + +DS4Server uses the text artifacts published at +`Youssofal/Qwen3.8-Flash-Next-MTPLX-Bare-Speed` revision +`74559cdf34fbfc0b593de72d17e93f37fd4f9ea7`. The upstream base is +`Qwen/Qwen3.8-Flash-Next` revision +`de4b8e4d43b917e7706784d8bb445c9af86a3540`, under the Qwen Community +License 1.0 included in the artifact set. + +The source already separates the 71,742,682,599-byte resident text core, +32,000,154,008-byte PLE table, and 1,672,575,532-byte MTP head. DS4Server's +artifact conversion is therefore deliberately byte-preserving: the Rust tool +validates and materializes the selected files without changing their tensor +containers or numerical interpretation. The complete selected download is +105,438,842,430 bytes. + +The manifest at `assets/models/qwen38-flash-next-bare-speed.json` pins every +selected file by byte size and SHA-256. Its companion TSV records all 2,527 +text, PLE, and MTP tensors with their source file, name, dtype, shape, +quantization parameters, and byte range. Validation also compares the 2,466 +core tensors with the source index, rejects duplicate names, invalid byte +ranges, and precision-map drift, and checks the fixed `qwen4_exp` architecture +and MTPLX serving contract before any payload is accepted. + +The precision contract follows the pinned files and their per-tensor +`quantization_config`: affine 4-bit matrices use 64-weight groups; router +matrices and QSA `index_qk_proj` use the source's declared 8-bit groups; GDN +convolution and recurrent parameters, norms, hyper-connection parameters, and +the MTP mixing/norm parameters remain BF16. The MTP layer's attention and MoE +matrices remain in their published 2-, 4-, or 8-bit affine representation; +they are not up-converted to satisfy a broader prose description. The PLE +sidecar's own header pins its 4-bit rows to 32-weight groups. No Python, MLX, +llama.cpp runtime, or foreign converter is part of the tool or product. + +`model-vision.safetensors` and its image/video processor configs are explicitly +excluded. The full config is retained because its `text_config` is the loader +authority. llama.cpp commit `b356fa2624643b6d5753162ae43efff8cdd4d8cb` +is not used as a converter because its reviewed path did not preserve this +separate MTP artifact. + +Download and authenticate every selected file without loading tensors: + +```sh +cargo run --release --bin qwen38-artifacts -- \ + fetch assets/models/qwen38-flash-next-bare-speed.json OUTPUT_DIRECTORY +``` + +Interrupted downloads remain as resumable `.part` files. The fetcher uses the +manifest's immutable revision URLs, validates resumed ranges, authenticates +each completed file, and then runs the complete inventory and precision +verifier. + +Verify a pinned checkout: + +```sh +cargo run --release --bin qwen38-artifacts -- \ + verify assets/models/qwen38-flash-next-bare-speed.json SOURCE_DIRECTORY +``` + +Materialize a clean, authenticated artifact directory: + +```sh +cargo run --release --bin qwen38-artifacts -- \ + materialize assets/models/qwen38-flash-next-bare-speed.json \ + SOURCE_DIRECTORY OUTPUT_DIRECTORY +``` + +The output directory must not already exist. Materialization uses hard links +when possible and otherwise copies bytes, then re-runs the same full verifier +against the output. + +## Verification receipt + +On 2026-09-03 the pinned source passed full authentication and inventory, +precision, and representative-row verification. Two separately empty output +directories were then materialized and independently verified. Each contained +the same 29 selected files and 105,438,842,430 artifact bytes; normalized +SHA-256 listings for all 31 output files (artifacts, manifest, and inventory) +were byte-identical and had list digest +`3e0233c2356b26859de43b25b2a3df08169dd8263071432bdf2fc497da74007c`. +The six dequantized-row receipts are pinned directly in the manifest. diff --git a/tools/qwen38-artifacts.rs b/tools/qwen38-artifacts.rs new file mode 100644 index 0000000..5afca3f --- /dev/null +++ b/tools/qwen38-artifacts.rs @@ -0,0 +1,1250 @@ +use serde::de::{MapAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::env; +use std::fmt; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +const MAX_SAFETENSORS_HEADER: u64 = 16 * 1024 * 1024; +const CONVERTER: &str = "qwen38-artifacts-v1-identity"; +const LICENSE: &str = "Qwen Community License 1.0"; +const SOURCE_REPOSITORY: &str = "Youssofal/Qwen3.8-Flash-Next-MTPLX-Bare-Speed"; +const SOURCE_REVISION: &str = "74559cdf34fbfc0b593de72d17e93f37fd4f9ea7"; +const BASE_REPOSITORY: &str = "Qwen/Qwen3.8-Flash-Next"; +const BASE_REVISION: &str = "de4b8e4d43b917e7706784d8bb445c9af86a3540"; +const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Deserialize)] +struct Manifest { + format: u32, + source: Source, + tensor_inventory: String, + tensor_inventory_sha256: String, + files: Vec, + config: BTreeMap, + runtime: BTreeMap, + representative_rows: Vec, + excluded: Vec, +} + +#[derive(Deserialize)] +struct Source { + repository: String, + url: String, + revision: String, + base_repository: String, + base_url: String, + base_revision: String, + license: String, + converter: String, +} + +#[derive(Deserialize)] +struct Artifact { + path: String, + role: String, + size: u64, + sha256: String, +} + +#[derive(Deserialize)] +struct Excluded { + path: String, + reason: String, + size: u64, + sha256: String, +} + +#[derive(Deserialize)] +struct Sample { + class: String, + file: String, + tensor: String, + row: u64, + values: u64, + bits: Option, + group_size: Option, + sha256: String, +} + +fn main() { + if let Err(error) = run() { + eprintln!("qwen38-artifacts: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let args = env::args_os().skip(1).collect::>(); + match args.as_slice() { + [command, manifest, source] if command == "verify" => { + let manifest = Path::new(manifest); + verify(manifest, Path::new(source))?; + println!("verified {}", manifest.display()); + Ok(()) + } + [command, manifest, source, output] if command == "inventory" => { + let (manifest, _) = load_manifest(Path::new(manifest))?; + let inventory = build_inventory(&manifest, Path::new(source), false)?; + fs::write(output, inventory).map_err(|error| error.to_string()) + } + [command, manifest, source, output] if command == "materialize" => { + materialize(Path::new(manifest), Path::new(source), Path::new(output)) + } + [command, manifest, output] if command == "fetch" => { + fetch(Path::new(manifest), Path::new(output)) + } + [command, manifest, source] if command == "sample-digests" => { + let (manifest, _) = load_manifest(Path::new(manifest))?; + for sample in &manifest.representative_rows { + println!("{}\t{}", sample.class, sample_digest(Path::new(source), sample)?); + } + Ok(()) + } + _ => Err("usage: qwen38-artifacts fetch MANIFEST OUTPUT | verify MANIFEST SOURCE | inventory MANIFEST SOURCE OUTPUT | sample-digests MANIFEST SOURCE | materialize MANIFEST SOURCE OUTPUT".into()), + } +} + +fn load_manifest(path: &Path) -> Result<(Manifest, PathBuf), String> { + let bytes = fs::read(path).map_err(|error| format!("{}: {error}", path.display()))?; + let manifest: Manifest = + serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?; + if manifest.format != 1 { + return Err(format!("unsupported manifest format {}", manifest.format)); + } + validate_source(&manifest.source)?; + let directory = path + .parent() + .ok_or_else(|| format!("{} has no parent directory", path.display()))?; + Ok((manifest, directory.to_owned())) +} + +fn validate_source(source: &Source) -> Result<(), String> { + for (name, value) in [ + ("source repository", source.repository.as_str()), + ("source URL", source.url.as_str()), + ("source revision", source.revision.as_str()), + ("base repository", source.base_repository.as_str()), + ("base URL", source.base_url.as_str()), + ("base revision", source.base_revision.as_str()), + ("license", source.license.as_str()), + ("converter", source.converter.as_str()), + ] { + if value.is_empty() { + return Err(format!("{name} is empty")); + } + } + if source.revision.len() != 40 || source.base_revision.len() != 40 { + return Err("source revisions must be full 40-character commits".into()); + } + if source.repository != SOURCE_REPOSITORY + || source.revision != SOURCE_REVISION + || source.base_repository != BASE_REPOSITORY + || source.base_revision != BASE_REVISION + { + return Err("manifest does not identify the pinned Bare Speed source and base".into()); + } + if source.url + != format!( + "https://huggingface.co/{}/tree/{}", + source.repository, source.revision + ) + || source.base_url + != format!( + "https://huggingface.co/{}/tree/{}", + source.base_repository, source.base_revision + ) + { + return Err("source URLs do not match their repositories and revisions".into()); + } + if source.converter != CONVERTER || source.license != LICENSE { + return Err("manifest converter or license does not match this tool".into()); + } + Ok(()) +} + +fn verify(manifest_path: &Path, source: &Path) -> Result<(), String> { + let (manifest, manifest_directory) = load_manifest(manifest_path)?; + validate_file_contract(&manifest)?; + for artifact in &manifest.files { + verify_artifact(&source.join(&artifact.path), artifact)?; + } + validate_json_contract(source, "config.json", &manifest.config)?; + validate_json_contract(source, "mtplx_runtime.json", &manifest.runtime)?; + validate_index(&manifest, source)?; + validate_samples(&manifest, source)?; + let actual_inventory = build_inventory(&manifest, source, true)?; + let inventory_path = manifest_directory.join(&manifest.tensor_inventory); + let expected_inventory = fs::read_to_string(&inventory_path) + .map_err(|error| format!("{}: {error}", inventory_path.display()))?; + let inventory_sha256 = sha256_file(&inventory_path)?; + if inventory_sha256 != manifest.tensor_inventory_sha256 { + return Err(format!( + "{} has SHA-256 {inventory_sha256}, expected {}", + inventory_path.display(), + manifest.tensor_inventory_sha256 + )); + } + if actual_inventory != expected_inventory { + return Err(format!( + "tensor inventory does not match {}", + inventory_path.display() + )); + } + Ok(()) +} + +fn validate_samples(manifest: &Manifest, source: &Path) -> Result<(), String> { + let classes = manifest + .representative_rows + .iter() + .map(|sample| sample.class.as_str()) + .collect::>(); + let required = ["dense", "expert", "gdn", "mtp", "ple", "qsa"] + .into_iter() + .collect::>(); + if classes != required || manifest.representative_rows.len() != required.len() { + return Err("representative rows must cover dense, expert, GDN, MTP, PLE, and QSA".into()); + } + let precision = precision_assignments(manifest, source, true)?; + for sample in &manifest.representative_rows { + if sample.values == 0 || sample.values > 4096 { + return Err(format!("{} sample has an invalid width", sample.class)); + } + let expected_role = match sample.class.as_str() { + "ple" => "ple", + "mtp" => "mtp", + _ => "core", + }; + if !manifest + .files + .iter() + .any(|artifact| artifact.path == sample.file && artifact.role == expected_role) + { + return Err(format!( + "{} sample is not in its artifact role", + sample.class + )); + } + match precision.get(&sample.tensor) { + Some(contract) + if sample.bits == Some(contract.bits) + && sample.group_size == Some(contract.group_size) => {} + None if sample.bits.is_none() && sample.group_size.is_none() => {} + _ => { + return Err(format!( + "{} sample does not match its tensor precision", + sample.class + )); + } + } + let actual = sample_digest(source, sample)?; + if actual != sample.sha256 { + return Err(format!( + "{} sample has SHA-256 {actual}, expected {}", + sample.class, sample.sha256 + )); + } + } + Ok(()) +} + +fn sample_digest(source: &Path, sample: &Sample) -> Result { + let path = source.join(&sample.file); + let (data_start, tensors) = read_safetensors_header(&path, true)?; + let tensor = tensors + .get(&sample.tensor) + .ok_or_else(|| format!("{} is missing {}", path.display(), sample.tensor))?; + let mut values = Vec::with_capacity(sample.values as usize); + match tensor.dtype.as_str() { + "BF16" => { + if sample.bits.is_some() || sample.group_size.is_some() { + return Err(format!( + "{} BF16 sample declares quantization", + sample.class + )); + } + let columns = *tensor + .shape + .last() + .ok_or_else(|| format!("{} has no dimensions", sample.tensor))?; + if sample.values > columns { + return Err(format!( + "{} sample is wider than its tensor row", + sample.class + )); + } + let start = tensor_byte_offset(data_start, tensor, sample.row, columns * 2)?; + let bytes = read_bytes(&path, start, sample.values * 2)?; + for bytes in bytes.chunks_exact(2) { + values.push(f32::from_bits( + u32::from(u16::from_le_bytes([bytes[0], bytes[1]])) << 16, + )); + } + } + "U32" => { + let bits = sample + .bits + .ok_or_else(|| format!("{} sample has no bit width", sample.class))?; + let group_size = sample + .group_size + .ok_or_else(|| format!("{} sample has no group size", sample.class))?; + if !matches!(bits, 2 | 4 | 8) || group_size == 0 { + return Err(format!("{} sample has invalid quantization", sample.class)); + } + let packed_columns = *tensor + .shape + .last() + .ok_or_else(|| format!("{} has no dimensions", sample.tensor))?; + let pack = u64::from(32 / bits); + let columns = packed_columns + .checked_mul(pack) + .ok_or_else(|| format!("{} row width overflows", sample.tensor))?; + if sample.values > columns || !columns.is_multiple_of(group_size) { + return Err(format!("{} sample has invalid row width", sample.class)); + } + let base = sample + .tensor + .strip_suffix(".weight") + .ok_or_else(|| format!("{} is not a quantized weight", sample.tensor))?; + let scales = tensors + .get(&format!("{base}.scales")) + .ok_or_else(|| format!("{} has no scales", sample.tensor))?; + let biases = tensors + .get(&format!("{base}.biases")) + .ok_or_else(|| format!("{} has no biases", sample.tensor))?; + let groups = columns / group_size; + if scales.dtype != "BF16" + || biases.dtype != "BF16" + || scales.shape.last() != Some(&groups) + || biases.shape != scales.shape + { + return Err(format!( + "{} has incompatible affine parameters", + sample.tensor + )); + } + let packed_start = + tensor_byte_offset(data_start, tensor, sample.row, packed_columns * 4)?; + let scale_start = tensor_byte_offset(data_start, scales, sample.row, groups * 2)?; + let bias_start = tensor_byte_offset(data_start, biases, sample.row, groups * 2)?; + let packed = read_bytes(&path, packed_start, packed_columns * 4)?; + let scales = read_bf16(&path, scale_start, groups)?; + let biases = read_bf16(&path, bias_start, groups)?; + let mask = (1_u32 << bits) - 1; + for index in 0..sample.values { + let word_index = (index / pack) as usize * 4; + let word = u32::from_le_bytes([ + packed[word_index], + packed[word_index + 1], + packed[word_index + 2], + packed[word_index + 3], + ]); + let quantized = (word >> ((index % pack) * u64::from(bits))) & mask; + let group = (index / group_size) as usize; + values.push(quantized as f32 * scales[group] + biases[group]); + } + } + dtype => return Err(format!("{} sample uses unsupported {dtype}", sample.class)), + } + let mut hash = Sha256::new(); + for value in values { + hash.update(value.to_le_bytes()); + } + Ok(hex_digest(hash.finalize())) +} + +fn tensor_byte_offset( + data_start: u64, + tensor: &Tensor, + row: u64, + row_bytes: u64, +) -> Result { + let offset = row + .checked_mul(row_bytes) + .and_then(|offset| tensor.offsets[0].checked_add(offset)) + .and_then(|offset| data_start.checked_add(offset)) + .ok_or_else(|| "sample offset overflows".to_owned())?; + let end = offset + .checked_add(row_bytes) + .ok_or_else(|| "sample range overflows".to_owned())?; + if end > data_start.saturating_add(tensor.offsets[1]) { + return Err("sample row exceeds its tensor".into()); + } + Ok(offset) +} + +fn read_bytes(path: &Path, offset: u64, length: u64) -> Result, String> { + let mut file = File::open(path).map_err(|error| format!("{}: {error}", path.display()))?; + file.seek(SeekFrom::Start(offset)) + .map_err(|error| format!("{}: {error}", path.display()))?; + let mut bytes = vec![0_u8; length as usize]; + file.read_exact(&mut bytes) + .map_err(|error| format!("{}: {error}", path.display()))?; + Ok(bytes) +} + +fn read_bf16(path: &Path, offset: u64, count: u64) -> Result, String> { + Ok(read_bytes(path, offset, count * 2)? + .chunks_exact(2) + .map(|bytes| f32::from_bits(u32::from(u16::from_le_bytes([bytes[0], bytes[1]])) << 16)) + .collect()) +} + +fn validate_file_contract(manifest: &Manifest) -> Result<(), String> { + validate_name(&manifest.tensor_inventory)?; + if manifest.tensor_inventory_sha256.len() != 64 + || !manifest + .tensor_inventory_sha256 + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err("tensor inventory has an invalid SHA-256".into()); + } + let mut paths = HashSet::new(); + let mut roles = BTreeMap::<&str, usize>::new(); + for artifact in &manifest.files { + validate_name(&artifact.path)?; + if !paths.insert(artifact.path.as_str()) { + return Err(format!("duplicate artifact {}", artifact.path)); + } + if artifact.size == 0 + || artifact.sha256.len() != 64 + || !artifact.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(format!("invalid SHA-256 for {}", artifact.path)); + } + *roles.entry(&artifact.role).or_default() += 1; + } + for excluded in &manifest.excluded { + validate_name(&excluded.path)?; + if !paths.insert(excluded.path.as_str()) { + return Err(format!("{} is both included and excluded", excluded.path)); + } + if excluded.reason.is_empty() { + return Err(format!("{} has no exclusion reason", excluded.path)); + } + if excluded.size == 0 + || excluded.sha256.len() != 64 + || !excluded.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(format!("{} has invalid exclusion metadata", excluded.path)); + } + } + for (role, count) in [("core", 17), ("ple", 1), ("mtp", 1)] { + if roles.get(role).copied() != Some(count) { + return Err(format!("manifest must contain {count} {role} artifact(s)")); + } + } + for required in [ + "model.safetensors.index.json", + "config.json", + "generation_config.json", + "mtplx_runtime.json", + "chat_template.jinja", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + "LICENSE", + ] { + if !manifest + .files + .iter() + .any(|artifact| artifact.path == required) + { + return Err(format!("manifest is missing {required}")); + } + } + if manifest.files.len() != 29 { + return Err("manifest must contain exactly 29 selected files".into()); + } + let excluded = manifest + .excluded + .iter() + .map(|entry| entry.path.as_str()) + .collect::>(); + if excluded + != [ + "model-vision.safetensors", + "preprocessor_config.json", + "processor_config.json", + "video_preprocessor_config.json", + ] + .into_iter() + .collect() + { + return Err("manifest must explicitly exclude the four vision artifacts".into()); + } + Ok(()) +} + +fn validate_name(name: &str) -> Result<(), String> { + let path = Path::new(name); + if path.components().count() != 1 || path.file_name().is_none() { + return Err(format!("artifact path must be one file name: {name}")); + } + Ok(()) +} + +fn validate_json_contract( + source: &Path, + file_name: &str, + expected_values: &BTreeMap, +) -> Result<(), String> { + let path = source.join(file_name); + let config: Value = serde_json::from_slice( + &fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?, + ) + .map_err(|error| format!("{}: {error}", path.display()))?; + for (pointer, expected) in expected_values { + let actual = config + .pointer(pointer) + .ok_or_else(|| format!("{file_name} is missing {pointer}"))?; + if actual != expected { + return Err(format!( + "{file_name} {pointer} is {actual}, expected {expected}" + )); + } + } + Ok(()) +} + +fn validate_index(manifest: &Manifest, source: &Path) -> Result<(), String> { + let path = source.join("model.safetensors.index.json"); + let value: Value = serde_json::from_slice( + &fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?, + ) + .map_err(|error| format!("{}: {error}", path.display()))?; + let map = value + .get("weight_map") + .and_then(Value::as_object) + .ok_or_else(|| "model index has no weight_map object".to_owned())?; + let core_files = manifest + .files + .iter() + .filter(|artifact| artifact.role == "core") + .map(|artifact| artifact.path.as_str()) + .collect::>(); + let indexed = map + .iter() + .filter_map(|(name, file)| { + file.as_str() + .filter(|file| core_files.contains(file)) + .map(|_| name.clone()) + }) + .collect::>(); + let actual = tensor_headers(manifest, source, Some("core"), true)? + .keys() + .cloned() + .collect::>(); + if indexed != actual { + return Err("model index and core shard tensor inventories differ".into()); + } + let vision_count = map + .values() + .filter(|file| file.as_str() == Some("model-vision.safetensors")) + .count(); + if vision_count == 0 + || !manifest + .excluded + .iter() + .any(|entry| entry.path == "model-vision.safetensors") + { + return Err("vision tensors are not explicitly excluded".into()); + } + Ok(()) +} + +fn build_inventory(manifest: &Manifest, source: &Path, complete: bool) -> Result { + tensor_headers(manifest, source, None, complete)?; + let precision = precision_assignments(manifest, source, complete)?; + let mut output = String::from( + "file\tname\tdtype\tshape\tquant_bits\tgroup_size\tquant_mode\tdata_start\tdata_end\n", + ); + for artifact in manifest + .files + .iter() + .filter(|artifact| artifact.path.ends_with(".safetensors")) + { + let path = source.join(&artifact.path); + let (data_start, tensors) = read_safetensors_header(&path, complete)?; + for (name, tensor) in tensors { + let start = data_start + .checked_add(tensor.offsets[0]) + .ok_or_else(|| format!("{name} offset overflows"))?; + let end = data_start + .checked_add(tensor.offsets[1]) + .ok_or_else(|| format!("{name} offset overflows"))?; + let quantization = precision.get(&name); + output.push_str(&format!( + "{}\t{name}\t{}\t{}\t{}\t{}\t{}\t{start}\t{end}\n", + artifact.path, + tensor.dtype, + tensor + .shape + .iter() + .map(u64::to_string) + .collect::>() + .join("x"), + quantization.map_or_else(String::new, |value| value.bits.to_string()), + quantization.map_or_else(String::new, |value| value.group_size.to_string()), + quantization.map_or("", |value| value.mode.as_str()), + )); + } + } + Ok(output) +} + +#[derive(Clone)] +struct Quantization { + bits: u32, + group_size: u64, + mode: String, +} + +fn precision_assignments( + manifest: &Manifest, + source: &Path, + complete: bool, +) -> Result, String> { + let core = tensor_headers(manifest, source, Some("core"), complete)?; + let config_path = source.join("config.json"); + let config: Value = serde_json::from_slice( + &fs::read(&config_path).map_err(|error| format!("{}: {error}", config_path.display()))?, + ) + .map_err(|error| format!("{}: {error}", config_path.display()))?; + if config.get("quantization") != config.get("quantization_config") { + return Err("config quantization maps differ".into()); + } + let quantization = config + .get("quantization") + .and_then(Value::as_object) + .ok_or_else(|| "config has no quantization object".to_owned())?; + let mut assignments = BTreeMap::new(); + for (base, contract) in quantization { + if matches!(base.as_str(), "bits" | "group_size" | "mode") { + continue; + } + match contract { + Value::Bool(false) => validate_unquantized(base, &core)?, + Value::Object(contract) => { + let bits = contract_u64(contract, "bits", base)? as u32; + let group_size = contract_u64(contract, "group_size", base)?; + let mode = contract + .get("mode") + .and_then(Value::as_str) + .ok_or_else(|| format!("{base} has no quantization mode"))?; + add_quantized( + base, + Quantization { + bits, + group_size, + mode: mode.to_owned(), + }, + &core, + &mut assignments, + )?; + } + _ => return Err(format!("{base} has an invalid quantization contract")), + } + } + reject_unassigned_quantized(&core, &assignments)?; + + let mtp = tensor_headers(manifest, source, Some("mtp"), complete)?; + add_inferred_sidecar(&mtp, 64, &mut assignments)?; + let ple = tensor_headers(manifest, source, Some("ple"), complete)?; + let ple_group = config + .pointer("/mtplx_recipe/ngram/group_size") + .and_then(Value::as_u64) + .ok_or_else(|| "config has no PLE group size".to_owned())?; + let ple_bits = config + .pointer("/mtplx_recipe/ngram/bits") + .and_then(Value::as_u64) + .ok_or_else(|| "config has no PLE bit width".to_owned())? as u32; + for tensor in ple.keys().filter(|name| name.ends_with(".weight")) { + let base = tensor.trim_end_matches(".weight"); + add_quantized( + base, + Quantization { + bits: ple_bits, + group_size: ple_group, + mode: "affine".into(), + }, + &ple, + &mut assignments, + )?; + } + reject_unassigned_quantized(&ple, &assignments)?; + Ok(assignments) +} + +fn contract_u64( + contract: &serde_json::Map, + field: &str, + base: &str, +) -> Result { + contract + .get(field) + .and_then(Value::as_u64) + .ok_or_else(|| format!("{base} has no quantization {field}")) +} + +fn validate_unquantized(base: &str, tensors: &BTreeMap) -> Result<(), String> { + if let Some(weight) = tensors.get(&format!("{base}.weight")) + && weight.dtype != "BF16" + { + return Err(format!("{base} is unexpectedly quantized")); + } + if tensors.contains_key(&format!("{base}.scales")) + || tensors.contains_key(&format!("{base}.biases")) + { + return Err(format!("{base} has unexpected affine parameters")); + } + Ok(()) +} + +fn add_inferred_sidecar( + tensors: &BTreeMap, + group_size: u64, + assignments: &mut BTreeMap, +) -> Result<(), String> { + for (name, tensor) in tensors { + if tensor.dtype != "U32" { + continue; + } + let base = name + .strip_suffix(".weight") + .ok_or_else(|| format!("quantized sidecar tensor {name} is not a weight"))?; + let scales = tensors + .get(&format!("{base}.scales")) + .ok_or_else(|| format!("{base} has no scales"))?; + let packed = *tensor + .shape + .last() + .ok_or_else(|| format!("{name} has no dimensions"))?; + let groups = *scales + .shape + .last() + .ok_or_else(|| format!("{base}.scales has no dimensions"))?; + let pack = groups + .checked_mul(group_size) + .and_then(|columns| columns.checked_div(packed)) + .filter(|pack| matches!(pack, 4 | 8 | 16)) + .ok_or_else(|| format!("{base} has an invalid packed width"))?; + add_quantized( + base, + Quantization { + bits: (32 / pack) as u32, + group_size, + mode: "affine".into(), + }, + tensors, + assignments, + )?; + } + reject_unassigned_quantized(tensors, assignments) +} + +fn add_quantized( + base: &str, + quantization: Quantization, + tensors: &BTreeMap, + assignments: &mut BTreeMap, +) -> Result<(), String> { + if !matches!(quantization.bits, 2 | 4 | 8) + || quantization.group_size == 0 + || quantization.mode != "affine" + { + return Err(format!("{base} has unsupported quantization")); + } + let names = [ + format!("{base}.weight"), + format!("{base}.scales"), + format!("{base}.biases"), + ]; + let weight = tensors + .get(&names[0]) + .ok_or_else(|| format!("{base} is missing its weight"))?; + let scales = tensors + .get(&names[1]) + .ok_or_else(|| format!("{base} is missing its scales"))?; + let biases = tensors + .get(&names[2]) + .ok_or_else(|| format!("{base} is missing its biases"))?; + let packed = *weight + .shape + .last() + .ok_or_else(|| format!("{base} has no dimensions"))?; + let groups = *scales + .shape + .last() + .ok_or_else(|| format!("{base}.scales has no dimensions"))?; + let pack = u64::from(32 / quantization.bits); + if weight.dtype != "U32" + || scales.dtype != "BF16" + || biases.dtype != "BF16" + || scales.shape != biases.shape + || weight.shape[..weight.shape.len() - 1] != scales.shape[..scales.shape.len() - 1] + || packed.checked_mul(pack) != groups.checked_mul(quantization.group_size) + { + return Err(format!("{base} does not match its affine contract")); + } + for name in names { + if assignments + .insert(name.clone(), quantization.clone()) + .is_some() + { + return Err(format!("duplicate quantization assignment for {name}")); + } + } + Ok(()) +} + +fn reject_unassigned_quantized( + tensors: &BTreeMap, + assignments: &BTreeMap, +) -> Result<(), String> { + for (name, tensor) in tensors { + let affine_parameter = name.ends_with(".scales") || name.ends_with(".biases"); + if (tensor.dtype == "U32" || affine_parameter) && !assignments.contains_key(name) { + return Err(format!("{name} is unexpectedly quantized")); + } + } + Ok(()) +} + +fn tensor_headers( + manifest: &Manifest, + source: &Path, + role: Option<&str>, + complete: bool, +) -> Result, String> { + let mut all = BTreeMap::new(); + for artifact in manifest.files.iter().filter(|artifact| { + artifact.path.ends_with(".safetensors") && role.is_none_or(|role| artifact.role == role) + }) { + let (_, tensors) = read_safetensors_header(&source.join(&artifact.path), complete)?; + for (name, tensor) in tensors { + if all.insert(name.clone(), tensor).is_some() { + return Err(format!("duplicate tensor {name}")); + } + } + } + Ok(all) +} + +#[derive(Deserialize)] +struct Tensor { + dtype: String, + shape: Vec, + #[serde(rename = "data_offsets")] + offsets: [u64; 2], +} + +fn read_safetensors_header( + path: &Path, + complete: bool, +) -> Result<(u64, BTreeMap), String> { + let mut file = File::open(path).map_err(|error| format!("{}: {error}", path.display()))?; + let size = file + .metadata() + .map_err(|error| format!("{}: {error}", path.display()))? + .len(); + let mut length = [0_u8; 8]; + file.read_exact(&mut length) + .map_err(|error| format!("{}: {error}", path.display()))?; + let length = u64::from_le_bytes(length); + if length == 0 || length > MAX_SAFETENSORS_HEADER { + return Err(format!( + "{} has invalid header size {length}", + path.display() + )); + } + let data_start = 8_u64 + .checked_add(length) + .ok_or_else(|| format!("{} header size overflows", path.display()))?; + let mut bytes = vec![0_u8; length as usize]; + file.read_exact(&mut bytes) + .map_err(|error| format!("{}: {error}", path.display()))?; + let mut value = serde_json::from_slice::(&bytes) + .map_err(|error| format!("{}: {error}", path.display()))?; + value.0.remove("__metadata__"); + let mut tensors = BTreeMap::new(); + for (name, value) in value.0 { + let tensor: Tensor = serde_json::from_value(value) + .map_err(|error| format!("{} tensor {name}: {error}", path.display()))?; + if tensor.shape.is_empty() || tensor.offsets[0] > tensor.offsets[1] { + return Err(format!( + "{} tensor {name} has invalid layout", + path.display() + )); + } + let element_size = match tensor.dtype.as_str() { + "BF16" => 2_u64, + "I64" => 8_u64, + "U32" => 4_u64, + _ => { + return Err(format!( + "{} tensor {name} has unsupported dtype", + path.display() + )); + } + }; + let expected_bytes = tensor + .shape + .iter() + .try_fold(element_size, |bytes, dimension| { + bytes.checked_mul(*dimension) + }); + if expected_bytes != tensor.offsets[1].checked_sub(tensor.offsets[0]) { + return Err(format!( + "{} tensor {name} has a shape/byte-length mismatch", + path.display() + )); + } + if complete && data_start.saturating_add(tensor.offsets[1]) > size { + return Err(format!("{} tensor {name} exceeds its file", path.display())); + } + tensors.insert(name, tensor); + } + Ok((data_start, tensors)) +} + +struct UniqueObject(BTreeMap); + +impl<'de> Deserialize<'de> for UniqueObject { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct UniqueObjectVisitor; + + impl<'de> Visitor<'de> for UniqueObjectVisitor { + type Value = UniqueObject; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a safetensors header object with unique tensor names") + } + + fn visit_map(self, mut map: M) -> Result + where + M: MapAccess<'de>, + { + let mut values = BTreeMap::new(); + while let Some((name, value)) = map.next_entry::()? { + if values.insert(name.clone(), value).is_some() { + return Err(serde::de::Error::custom(format!( + "duplicate tensor name {name}" + ))); + } + } + Ok(UniqueObject(values)) + } + } + + deserializer.deserialize_map(UniqueObjectVisitor) + } +} + +fn sha256_file(path: &Path) -> Result { + let mut file = File::open(path).map_err(|error| format!("{}: {error}", path.display()))?; + let mut hash = Sha256::new(); + let mut buffer = vec![0_u8; 8 * 1024 * 1024]; + loop { + let read = file + .read(&mut buffer) + .map_err(|error| format!("{}: {error}", path.display()))?; + if read == 0 { + break; + } + hash.update(&buffer[..read]); + } + Ok(hex_digest(hash.finalize())) +} + +fn hex_digest(digest: impl AsRef<[u8]>) -> String { + let mut output = String::with_capacity(64); + for byte in digest.as_ref() { + output.push_str(&format!("{byte:02x}")); + } + output +} + +fn fetch(manifest_path: &Path, output: &Path) -> Result<(), String> { + let (manifest, _) = load_manifest(manifest_path)?; + validate_file_contract(&manifest)?; + fs::create_dir_all(output).map_err(|error| format!("{}: {error}", output.display()))?; + for artifact in &manifest.files { + let destination = output.join(&artifact.path); + if destination.exists() { + verify_artifact(&destination, artifact)?; + continue; + } + let partial = output.join(format!("{}.part", artifact.path)); + if partial.metadata().map_or(0, |metadata| metadata.len()) > artifact.size { + File::create(&partial).map_err(|error| format!("{}: {error}", partial.display()))?; + } + if partial.metadata().map_or(0, |metadata| metadata.len()) == artifact.size { + verify_artifact(&partial, artifact)?; + fs::rename(&partial, &destination) + .map_err(|error| format!("{}: {error}", destination.display()))?; + println!("authenticated {}", artifact.path); + continue; + } + let url = format!( + "https://huggingface.co/{}/resolve/{}/{}", + manifest.source.repository, manifest.source.revision, artifact.path + ); + download(&url, &partial, true)?; + verify_artifact(&partial, artifact)?; + fs::rename(&partial, &destination) + .map_err(|error| format!("{}: {error}", destination.display()))?; + println!("fetched {}", artifact.path); + } + verify(manifest_path, output)?; + Ok(()) +} + +fn verify_artifact(path: &Path, artifact: &Artifact) -> Result<(), String> { + let size = path + .metadata() + .map_err(|error| format!("{}: {error}", path.display()))? + .len(); + if size != artifact.size { + return Err(format!( + "{} has {size} bytes, expected {}", + path.display(), + artifact.size + )); + } + let actual = sha256_file(path)?; + if actual != artifact.sha256 { + return Err(format!( + "{} has SHA-256 {actual}, expected {}", + path.display(), + artifact.sha256 + )); + } + Ok(()) +} + +fn download(url: &str, partial: &Path, https_only: bool) -> Result<(), String> { + let offset = partial.metadata().map_or(0, |metadata| metadata.len()); + let agent: ureq::Agent = ureq::Agent::config_builder() + .https_only(https_only) + .timeout_connect(Some(DOWNLOAD_TIMEOUT)) + .timeout_recv_response(Some(DOWNLOAD_TIMEOUT)) + .timeout_recv_body(Some(DOWNLOAD_TIMEOUT)) + .build() + .into(); + let mut request = agent.get(url); + if offset > 0 { + request = request.header("Range", format!("bytes={offset}-")); + } + let mut response = request + .call() + .map_err(|error| format!("artifact download failed: {error}"))?; + let status = response.status().as_u16(); + let append = offset > 0 && status == 206; + if offset > 0 && !matches!(status, 200 | 206) { + return Err(format!( + "artifact server returned HTTP {status} while resuming at byte {offset}" + )); + } + if append { + let expected = format!("bytes {offset}-"); + let content_range = response + .headers() + .get("content-range") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + if !content_range.starts_with(&expected) { + return Err(format!( + "artifact server returned an invalid Content-Range while resuming at byte {offset}" + )); + } + } + let mut file = OpenOptions::new() + .create(true) + .write(true) + .append(append) + .truncate(!append) + .open(partial) + .map_err(|error| format!("{}: {error}", partial.display()))?; + let mut body = response.body_mut().as_reader(); + let mut buffer = vec![0_u8; 1024 * 1024]; + loop { + let read = body + .read(&mut buffer) + .map_err(|error| format!("artifact download failed: {error}"))?; + if read == 0 { + break; + } + file.write_all(&buffer[..read]) + .map_err(|error| format!("{}: {error}", partial.display()))?; + } + Ok(()) +} + +fn materialize(manifest_path: &Path, source: &Path, output: &Path) -> Result<(), String> { + verify(manifest_path, source)?; + fs::create_dir(output).map_err(|error| format!("{}: {error}", output.display()))?; + let (manifest, manifest_directory) = load_manifest(manifest_path)?; + for artifact in &manifest.files { + let from = source.join(&artifact.path); + let to = output.join(&artifact.path); + if fs::hard_link(&from, &to).is_err() { + fs::copy(&from, &to).map_err(|error| format!("{}: {error}", to.display()))?; + } + } + fs::copy(manifest_path, output.join("manifest.json")) + .map_err(|error| format!("{}: {error}", output.display()))?; + fs::copy( + manifest_directory.join(&manifest.tensor_inventory), + output.join(&manifest.tensor_inventory), + ) + .map_err(|error| format!("{}: {error}", output.display()))?; + verify(manifest_path, output)?; + println!( + "materialized {} files ({} bytes) in {}", + manifest.files.len(), + manifest + .files + .iter() + .map(|artifact| artifact.size) + .sum::(), + output.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + use std::thread; + + fn safetensors_fixture(name: &str, header: &str, payload: &[u8]) -> PathBuf { + let path = env::temp_dir().join(format!( + "ds4-qwen38-{name}-{}-{}.safetensors", + std::process::id(), + std::thread::current().name().unwrap_or("test") + )); + let mut bytes = (header.len() as u64).to_le_bytes().to_vec(); + bytes.extend_from_slice(header.as_bytes()); + bytes.extend_from_slice(payload); + fs::write(&path, bytes).unwrap(); + path + } + + #[test] + fn artifact_names_cannot_escape_the_output_directory() { + assert!(validate_name("model.safetensors").is_ok()); + assert!(validate_name("../model.safetensors").is_err()); + assert!(validate_name("nested/model.safetensors").is_err()); + assert!(validate_name("").is_err()); + } + + #[test] + fn safetensors_validation_rejects_duplicate_names_and_invalid_ranges() { + let duplicate = safetensors_fixture( + "duplicate", + r#"{"x":{"dtype":"BF16","shape":[1],"data_offsets":[0,2]},"x":{"dtype":"BF16","shape":[1],"data_offsets":[0,2]}}"#, + &[0, 0], + ); + assert!(read_safetensors_header(&duplicate, true).is_err()); + fs::remove_file(duplicate).unwrap(); + + let outside = safetensors_fixture( + "outside", + r#"{"x":{"dtype":"BF16","shape":[1],"data_offsets":[0,2]}}"#, + &[0], + ); + assert!(read_safetensors_header(&outside, true).is_err()); + fs::remove_file(outside).unwrap(); + } + + #[test] + fn affine_sample_digest_uses_packed_values_scales_and_biases() { + let header = r#"{"x.biases":{"dtype":"BF16","shape":[1,1],"data_offsets":[6,8]},"x.scales":{"dtype":"BF16","shape":[1,1],"data_offsets":[4,6]},"x.weight":{"dtype":"U32","shape":[1,1],"data_offsets":[0,4]}}"#; + let mut payload = 0x7654_3210_u32.to_le_bytes().to_vec(); + payload.extend_from_slice(&0x4000_u16.to_le_bytes()); + payload.extend_from_slice(&0x3f80_u16.to_le_bytes()); + let path = safetensors_fixture("affine", header, &payload); + let sample = Sample { + class: "fixture".into(), + file: path.file_name().unwrap().to_string_lossy().into_owned(), + tensor: "x.weight".into(), + row: 0, + values: 8, + bits: Some(4), + group_size: Some(8), + sha256: String::new(), + }; + let mut expected = Sha256::new(); + for value in [1.0_f32, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0] { + expected.update(value.to_le_bytes()); + } + assert_eq!( + sample_digest(path.parent().unwrap(), &sample).unwrap(), + hex_digest(expected.finalize()) + ); + fs::remove_file(path).unwrap(); + } + + #[test] + fn artifact_download_resumes_an_authenticated_partial() { + let content = b"authenticated artifact"; + let offset = 7; + let directory = env::temp_dir().join(format!("ds4-qwen38-fetch-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let partial = directory.join("artifact.part"); + fs::write(&partial, &content[..offset]).unwrap(); + let listener = match TcpListener::bind("127.0.0.1:0") { + Ok(listener) => listener, + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => { + fs::remove_dir_all(directory).unwrap(); + return; + } + Err(error) => panic!("could not start test server: {error}"), + }; + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut connection, _) = listener.accept().unwrap(); + let mut request = [0_u8; 2048]; + let read = connection.read(&mut request).unwrap(); + let request = String::from_utf8_lossy(&request[..read]).to_ascii_lowercase(); + assert!(request.contains("range: bytes=7-")); + let remaining = &content[offset..]; + write!( + connection, + "HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nContent-Range: bytes {offset}-{}/{}\r\nConnection: close\r\n\r\n", + remaining.len(), + content.len() - 1, + content.len(), + ) + .unwrap(); + connection.write_all(remaining).unwrap(); + }); + + download(&format!("http://{address}/artifact"), &partial, false).unwrap(); + server.join().unwrap(); + verify_artifact( + &partial, + &Artifact { + path: "artifact".into(), + role: "fixture".into(), + size: content.len() as u64, + sha256: "ef056ce05d5f86154a8498b5504a46bc4b633fb68ec795a3a1cfd0548f791e4c".into(), + }, + ) + .unwrap(); + fs::remove_dir_all(directory).unwrap(); + } +}