hugo/DS4Server
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate Qwen3.8 model intake

Browse Source
This commit is contained in:
Georg Bauer committed 2026-09-03 19:56:23 +02:00
1 parent 4c83ac0360
commit 3773cfda2e
15 files changed
+1805 -35
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/engine/tokenizer.rs
+275 -12
View File
@@ -4,7 +4,10 @@ use super::{
VISION_TOKEN_END, VISION_TOKEN_START,
};
use crate::settings::ReasoningMode;
use serde::Deserialize;
use std::collections::HashMap;
use std::fs;
use std::path::Path;
const HIGH_REASONING_PREFIX: &str = "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n\
You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n\
@@ -20,7 +23,14 @@ fn reasoning_prefix(family: ModelFamily, reasoning: ReasoningMode) -> Option<&'s
(ModelFamily::Glm, ReasoningMode::Low) => Some("Reasoning Effort: Low"),
(ModelFamily::Glm, ReasoningMode::High) => Some("Reasoning Effort: High"),
(ModelFamily::Glm, ReasoningMode::Max) => Some("Reasoning Effort: Max"),
(_, ReasoningMode::Direct | ReasoningMode::Low) => None,
(ModelFamily::Qwen, _) => None,
(
_,
ReasoningMode::Direct
| ReasoningMode::Low
| ReasoningMode::Medium
| ReasoningMode::XHigh,
) => None,
}
}
@@ -38,7 +48,30 @@ pub(super) struct Tokenizer {
sop: i32,
think_start: i32,
think_end: i32,
rendered_specials: Vec<(&'static [u8], i32)>,
alternate_eos: i32,
rendered_specials: Vec<(Vec<u8>, i32)>,
}
#[derive(Deserialize)]
struct QwenTokenizerFile {
added_tokens: Vec<QwenAddedToken>,
model: QwenBpe,
}
#[derive(Deserialize)]
struct QwenAddedToken {
id: usize,
content: String,
special: bool,
}
#[derive(Deserialize)]
struct QwenBpe {
#[serde(rename = "type")]
kind: String,
vocab: HashMap<String, usize>,
merges: Vec<String>,
byte_fallback: bool,
}
impl Tokenizer {
@@ -95,6 +128,9 @@ impl Tokenizer {
lookup(b"<think>"),
lookup(b"</think>"),
),
ModelFamily::Qwen => {
return Err("Qwen tokenizers must be loaded from tokenizer.json".into());
}
};
if [bos, user, assistant, think_start, think_end]
.into_iter()
@@ -128,6 +164,7 @@ impl Tokenizer {
]
.into_iter()
.filter(|(_, token)| *token >= 0)
.map(|(marker, token)| (marker.to_vec(), token))
.collect();
Ok(Self {
@@ -144,6 +181,98 @@ impl Tokenizer {
sop,
think_start,
think_end,
alternate_eos: -1,
rendered_specials,
})
}
pub(super) fn load_qwen(path: &Path) -> Result<Self, String> {
let file: QwenTokenizerFile = serde_json::from_slice(
&fs::read(path).map_err(|error| format!("{}: {error}", path.display()))?,
)
.map_err(|error| format!("{}: {error}", path.display()))?;
if file.model.kind != "BPE" || file.model.byte_fallback {
return Err("Qwen tokenizer must use BPE without byte fallback".into());
}
let maximum = file
.model
.vocab
.values()
.copied()
.chain(file.added_tokens.iter().map(|token| token.id))
.max()
.ok_or_else(|| "Qwen tokenizer vocabulary is empty".to_owned())?;
if maximum != 248_076 {
return Err(format!(
"Qwen tokenizer ends at token id {maximum}, expected 248076"
));
}
// The model head has 248320 rows; the final 243 ids are deliberately
// reserved and have no tokenizer spelling in the pinned source.
let mut tokens = vec![Vec::new(); 248_320];
for (token, id) in file.model.vocab {
if !tokens[id].is_empty() {
return Err(format!("Qwen tokenizer has duplicate token id {id}"));
}
tokens[id] = token.into_bytes();
}
let mut rendered_specials = Vec::new();
for token in file.added_tokens {
if !tokens[token.id].is_empty() {
return Err(format!(
"Qwen tokenizer has duplicate token id {}",
token.id
));
}
tokens[token.id] = token.content.as_bytes().to_vec();
if token.special {
rendered_specials.push((token.content.into_bytes(), token.id as i32));
}
}
if tokens[..=maximum].iter().any(Vec::is_empty) {
return Err("Qwen tokenizer token ids are not contiguous".into());
}
let token_to_id = tokens
.iter()
.enumerate()
.filter(|(_, token)| !token.is_empty())
.map(|(id, token)| (token.clone(), id as i32))
.collect::<HashMap<_, _>>();
let required = |token: &[u8]| {
token_to_id.get(token).copied().ok_or_else(|| {
format!(
"required Qwen tokenizer token is missing: {}",
String::from_utf8_lossy(token)
)
})
};
let merge_rank = file
.model
.merges
.into_iter()
.enumerate()
.map(|(rank, merge)| (merge.into_bytes(), rank))
.collect();
let alternate_eos = required(b"<|endoftext|>")?;
let im_start = required(b"<|im_start|>")?;
let im_end = required(b"<|im_end|>")?;
let think_start = required(b"<think>")?;
let think_end = required(b"</think>")?;
Ok(Self {
family: ModelFamily::Qwen,
tokens,
token_to_id,
merge_rank,
bos: alternate_eos,
eos: im_end,
system: im_start,
user: im_start,
assistant: im_start,
observation: im_start,
sop: im_end,
think_start,
think_end,
alternate_eos,
rendered_specials,
})
}
@@ -152,6 +281,40 @@ impl Tokenizer {
self.tokens.len()
}
pub(super) fn validate_qwen_contract(&self) -> Result<(), String> {
if self.family != ModelFamily::Qwen {
return Err("tokenizer is not Qwen".into());
}
for (token, expected) in [
(b"<|endoftext|>".as_slice(), 248_044),
(b"<|im_start|>".as_slice(), 248_045),
(b"<|im_end|>".as_slice(), 248_046),
(b"<tool_call>".as_slice(), 248_058),
(b"</tool_call>".as_slice(), 248_059),
(b"<tool_response>".as_slice(), 248_066),
(b"</tool_response>".as_slice(), 248_067),
(b"<think>".as_slice(), 248_068),
(b"</think>".as_slice(), 248_069),
] {
if self.token_to_id.get(token).copied() != Some(expected) {
return Err(format!(
"Qwen special token {} does not have id {expected}",
String::from_utf8_lossy(token)
));
}
}
if ![248_044, 248_045, 248_046]
.into_iter()
.all(|token| self.is_ple_reset(token))
|| ![248_044, 248_046]
.into_iter()
.all(|token| self.is_stop(token))
{
return Err("Qwen EOS or PLE reset markers are invalid".into());
}
Ok(())
}
pub(super) fn token_bytes(&self, token: i32) -> Option<Vec<u8>> {
let token = self.tokens.get(usize::try_from(token).ok()?)?;
if token.windows(3).any(|window| window == [0xef, 0xbd, 0x9c]) {
@@ -163,10 +326,10 @@ impl Tokenizer {
pub(super) fn tokenize(&self, text: &str) -> Vec<i32> {
let mut output = Vec::new();
if self.family == ModelFamily::Glm {
self.tokenize_glm(text, &mut output);
} else {
self.tokenize_joyai(text, &mut output);
match self.family {
ModelFamily::DeepSeek => self.tokenize_joyai(text, &mut output),
ModelFamily::Glm => self.tokenize_gpt4(text, &mut output, 3),
ModelFamily::Qwen => self.tokenize_gpt4(text, &mut output, 1),
}
output
}
@@ -211,10 +374,10 @@ impl Tokenizer {
}
fn tokenize_plain(&self, text: &str, output: &mut Vec<i32>) {
if self.family == ModelFamily::Glm {
self.tokenize_glm(text, output);
} else {
self.tokenize_joyai(text, output);
match self.family {
ModelFamily::DeepSeek => self.tokenize_joyai(text, output),
ModelFamily::Glm => self.tokenize_gpt4(text, output, 3),
ModelFamily::Qwen => self.tokenize_gpt4(text, output, 1),
}
}
@@ -264,6 +427,14 @@ impl Tokenizer {
reasoning: ReasoningMode,
continue_assistant: bool,
) -> Vec<i32> {
if self.family == ModelFamily::Qwen {
return self.encode_qwen_messages(
system_prompt,
messages,
reasoning,
continue_assistant,
);
}
let mut output = vec![self.bos];
if self.family == ModelFamily::Glm && self.sop >= 0 {
output.push(self.sop);
@@ -353,12 +524,95 @@ impl Tokenizer {
output
}
fn encode_qwen_messages(
&self,
system_prompt: &str,
messages: &[ChatTurn],
reasoning: ReasoningMode,
continue_assistant: bool,
) -> Vec<i32> {
let instruction = match reasoning {
ReasoningMode::Direct | ReasoningMode::Medium => "",
ReasoningMode::Low => {
"Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration."
}
ReasoningMode::XHigh => {
"Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer."
}
ReasoningMode::High | ReasoningMode::Max => {
unreachable!("legacy reasoning modes are not valid for Qwen")
}
};
let mut rendered = String::new();
let system_prompt = system_prompt.trim();
if !instruction.is_empty() || !system_prompt.is_empty() {
rendered.push_str("<|im_start|>system\n");
if !instruction.is_empty() {
rendered.push_str(instruction);
if !system_prompt.is_empty() {
rendered.push_str("\n\n");
}
}
rendered.push_str(system_prompt);
rendered.push_str("<|im_end|>\n");
}
for (index, message) in messages.iter().enumerate() {
let content = message.content.trim();
if message.system {
rendered.push_str("<|im_start|>system\n");
rendered.push_str(content);
rendered.push_str("<|im_end|>\n");
} else if message.tool {
if index == 0 || !messages[index - 1].tool {
rendered.push_str("<|im_start|>user");
}
rendered.push_str("\n<tool_response>\n");
rendered.push_str(content);
rendered.push_str("\n</tool_response>");
if index + 1 == messages.len() || !messages[index + 1].tool {
rendered.push_str("<|im_end|>\n");
}
} else if message.user {
rendered.push_str("<|im_start|>user\n");
rendered.push_str(content);
rendered.push_str("<|im_end|>\n");
} else {
rendered.push_str("<|im_start|>assistant\n<think>\n");
if let Some(thinking) = &message.reasoning {
rendered.push_str(thinking.trim());
}
rendered.push_str("\n</think>\n\n");
rendered.push_str(content);
rendered.push_str("<|im_end|>\n");
}
}
if continue_assistant {
rendered.push_str("<|im_start|>assistant\n<think>\n");
if reasoning == ReasoningMode::Direct {
rendered.push_str("\n</think>\n\n");
}
}
self.tokenize_rendered(&rendered)
}
pub(super) fn encode_continuation(
&self,
prompt: &str,
reasoning: ReasoningMode,
skip_previous_eos: bool,
) -> Vec<i32> {
if self.family == ModelFamily::Qwen {
let messages = [ChatTurn {
user: true,
tool: false,
system: false,
skip_previous_eos,
reasoning: None,
reasoning_complete: true,
content: prompt.to_owned(),
}];
return self.encode_qwen_messages("", &messages, reasoning, true);
}
let mut output = Vec::new();
if self.family == ModelFamily::DeepSeek && !skip_previous_eos {
output.push(self.eos);
@@ -382,6 +636,7 @@ impl Tokenizer {
pub(super) fn is_stop(&self, token: i32) -> bool {
token == self.eos
|| token == self.alternate_eos
|| (self.family == ModelFamily::Glm
&& [self.system, self.user, self.assistant, self.observation].contains(&token))
}
@@ -394,6 +649,11 @@ impl Tokenizer {
token == self.think_end
}
pub(super) fn is_ple_reset(&self, token: i32) -> bool {
self.family == ModelFamily::Qwen
&& [self.system, self.sop, self.alternate_eos].contains(&token)
}
fn emit_piece(&self, raw: &[u8], output: &mut Vec<i32>) {
if raw.is_empty() {
return;
@@ -512,7 +772,7 @@ impl Tokenizer {
}
}
fn tokenize_glm(&self, text: &str, output: &mut Vec<i32>) {
fn tokenize_gpt4(&self, text: &str, output: &mut Vec<i32>, max_digits: usize) {
let mut position = 0;
while position < text.len() {
let start = position;
@@ -548,7 +808,10 @@ impl Tokenizer {
}
} else if current.number {
let mut digits = 0;
while position < text.len() && char_info(text, position).number && digits < 3 {
while position < text.len()
&& char_info(text, position).number
&& digits < max_digits
{
position = char_info(text, position).next;
digits += 1;
}