hugo/DS4Server
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: finalized support for chat

Browse Source
This commit is contained in:
Georg Bauer
2026-07-24 20:14:26 +02:00
parent 3154f57a2f
commit 6560f20129
9 changed files with 356 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/engine.rs
View File

@@ -10,6 +10,8 @@ use gguf::{F16, F32, Gguf, I32, IQ2_XXS, Q2_K, Q4_0, Q4_K, Q5_K, Q6_K, Q8_0, Ten
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
#[cfg(target_os = "macos")]
use std::time::Instant;
use tokenizer::Tokenizer;
#[cfg(target_os = "macos")]
@@ -334,7 +336,7 @@ impl Generator {
settings: &TurnSettings,
cancelled: &AtomicBool,
mut emit: impl FnMut(bool, String),
mut progress: impl FnMut(u32, u32),
mut progress: impl FnMut(u32, u32, Option<f32>),
) -> Result<(), String> {
self.select_checkpoint(checkpoint)?;
let (generated, prompt_complete) =
@@ -370,7 +372,7 @@ impl Generator {
settings: &TurnSettings,
cancelled: &AtomicBool,
emit: &mut impl FnMut(bool, String),
progress: &mut impl FnMut(u32, u32),
progress: &mut impl FnMut(u32, u32, Option<f32>),
) -> Result<(ChatTurn, bool), String> {
let tokens = match messages.split_last() {
Some((latest, history))
@@ -407,7 +409,7 @@ impl Generator {
));
}
let reused = self.executor.align_prompt(&tokens)?;
progress(self.executor.position(), self.executor.context());
progress(self.executor.position(), self.executor.context(), None);
let mut rng = Rng::new(settings.seed.unwrap_or(0x4453_3453_4552_5645));
let mut reasoning = settings.reasoning_mode != ReasoningMode::Direct;
let mut generated = ChatTurn {
@@ -423,9 +425,11 @@ impl Generator {
}
self.executor.eval(token)?;
if (index + 1).is_multiple_of(16) || index + 1 == prompt_tokens {
progress(self.executor.position(), self.executor.context());
progress(self.executor.position(), self.executor.context(), None);
}
}
let generation_started = Instant::now();
let mut generated_tokens = 0_u32;
for _ in 0..settings
.max_generated_tokens
.max(0)
@@ -469,7 +473,15 @@ impl Generator {
emit(reasoning, content);
}
self.executor.eval(token)?;
progress(self.executor.position(), self.executor.context());
generated_tokens += 1;
progress(
self.executor.position(),
self.executor.context(),
Some(
generated_tokens as f32
/ generation_started.elapsed().as_secs_f32().max(1.0e-6),
),
);
}
Ok((generated, true))
}