hugo/DS4Server
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement batched server parity and observability

Browse Source
This commit is contained in:
Georg Bauer
2026-07-25 08:32:35 +02:00
parent d554b77b9d
commit 2fbe605b5d
15 changed files with 6478 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/engine/gguf.rs
View File

@@ -46,6 +46,8 @@ pub(super) struct Tensor {
pub(super) struct Gguf {
path: PathBuf,
map: Mmap,
data_offset: u64,
max_tensor_bytes: u64,
pub(super) metadata: HashMap<String, Value>,
pub(super) tensors: HashMap<String, Tensor>,
}
@@ -141,6 +143,7 @@ impl Gguf {
}
let data_start = align(cursor.position() as u64, alignment)?;
let mut max_tensor_bytes = 0;
for (name, tensor) in &mut tensors {
tensor.offset = data_start
.checked_add(tensor.offset)
@@ -152,11 +155,14 @@ impl Gguf {
if end > map.len() as u64 {
return Err(format!("tensor {name} points outside the GGUF file"));
}
max_tensor_bytes = max_tensor_bytes.max(tensor.bytes);
}
Ok(Self {
path: path.to_owned(),
map,
data_offset: data_start,
max_tensor_bytes,
metadata,
tensors,
})
@@ -174,6 +180,14 @@ impl Gguf {
self.map.as_ptr()
}
pub(super) fn data_offset(&self) -> u64 {
self.data_offset
}
pub(super) fn max_tensor_bytes(&self) -> u64 {
self.max_tensor_bytes
}
pub(super) fn tensor(&self, name: &str) -> Result<&Tensor, String> {
self.tensors
.get(name)

2122
src/engine/metal.rs
View File

File diff suppressed because it is too large Load Diff

77
src/engine/tokenizer.rs
View File

@@ -21,6 +21,7 @@ pub(super) struct Tokenizer {
sop: i32,
think_start: i32,
think_end: i32,
rendered_specials: Vec<(&'static [u8], i32)>,
}
impl Tokenizer {
@@ -85,6 +86,32 @@ impl Tokenizer {
{
return Err("tokenizer does not provide the required DS4 chat markers".into());
}
let rendered_specials = [
("<｜begin▁of▁sentence｜>".as_bytes(), bos),
("<｜end▁of▁sentence｜>".as_bytes(), eos),
(b"[gMASK]".as_slice(), bos),
(b"<sop>".as_slice(), sop),
(b"<|system|>".as_slice(), system),
("<｜User｜>".as_bytes(), user),
("<｜Assistant｜>".as_bytes(), assistant),
(b"<|user|>".as_slice(), user),
(b"<|assistant|>".as_slice(), assistant),
(b"<|observation|>".as_slice(), observation),
(b"<think>".as_slice(), think_start),
(b"</think>".as_slice(), think_end),
(b"<tool_call>".as_slice(), lookup(b"<tool_call>")),
(b"</tool_call>".as_slice(), lookup(b"</tool_call>")),
(b"<tool_response>".as_slice(), lookup(b"<tool_response>")),
(b"</tool_response>".as_slice(), lookup(b"</tool_response>")),
(b"<arg_key>".as_slice(), lookup(b"<arg_key>")),
(b"</arg_key>".as_slice(), lookup(b"</arg_key>")),
(b"<arg_value>".as_slice(), lookup(b"<arg_value>")),
(b"</arg_value>".as_slice(), lookup(b"</arg_value>")),
("｜DSML｜".as_bytes(), lookup("｜DSML｜".as_bytes())),
]
.into_iter()
.filter(|(_, token)| *token >= 0)
.collect();
Ok(Self {
family,
@@ -100,6 +127,7 @@ impl Tokenizer {
sop,
think_start,
think_end,
rendered_specials,
})
}
@@ -126,6 +154,37 @@ impl Tokenizer {
output
}
pub(super) fn tokenize_rendered(&self, text: &str) -> Vec<i32> {
let bytes = text.as_bytes();
let mut output = Vec::new();
let mut span = 0;
let mut position = 0;
while position < bytes.len() {
let special = self
.rendered_specials
.iter()
.find(|(marker, _)| bytes[position..].starts_with(marker));
if let Some((marker, token)) = special {
self.tokenize_plain(&text[span..position], &mut output);
output.push(*token);
position += marker.len();
span = position;
} else {
position += 1;
}
}
self.tokenize_plain(&text[span..], &mut output);
output
}
fn tokenize_plain(&self, text: &str, output: &mut Vec<i32>) {
if self.family == ModelFamily::Glm {
self.tokenize_glm(text, output);
} else {
self.tokenize_joyai(text, output);
}
}
pub(super) fn encode_chat(
&self,
system_prompt: &str,
@@ -136,6 +195,7 @@ impl Tokenizer {
system_prompt,
&[ChatTurn {
user: true,
skip_previous_eos: false,
reasoning: None,
reasoning_complete: true,
content: prompt.to_owned(),
@@ -172,7 +232,7 @@ impl Tokenizer {
if self.family == ModelFamily::Glm {
output.push(self.system);
}
output.extend(self.tokenize(system_prompt));
output.extend(self.tokenize_rendered(system_prompt));
}
for message in messages {
output.push(if message.user {
@@ -183,7 +243,7 @@ impl Tokenizer {
if !message.user {
if let Some(reasoning) = &message.reasoning {
output.push(self.think_start);
output.extend(self.tokenize(reasoning));
output.extend(self.tokenize_rendered(reasoning));
if message.reasoning_complete {
output.push(self.think_end);
}
@@ -193,7 +253,7 @@ impl Tokenizer {
output.push(self.think_end);
}
}
output.extend(self.tokenize(&message.content));
output.extend(self.tokenize_rendered(&message.content));
if !message.user && self.family == ModelFamily::DeepSeek {
output.push(self.eos);
}
@@ -209,13 +269,18 @@ impl Tokenizer {
output
}
pub(super) fn encode_continuation(&self, prompt: &str, reasoning: ReasoningMode) -> Vec<i32> {
pub(super) fn encode_continuation(
&self,
prompt: &str,
reasoning: ReasoningMode,
skip_previous_eos: bool,
) -> Vec<i32> {
let mut output = Vec::new();
if self.family == ModelFamily::DeepSeek {
if self.family == ModelFamily::DeepSeek && !skip_previous_eos {
output.push(self.eos);
}
output.push(self.user);
output.extend(self.tokenize(prompt));
output.extend(self.tokenize_rendered(prompt));
output.push(self.assistant);
if reasoning != ReasoningMode::Direct {
output.push(self.think_start);