hugo/DS4Server
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

Complete local HTTP endpoint parity

Browse Source
This commit is contained in:
Georg Bauer
2026-07-25 14:18:57 +02:00
parent a1761fa731
commit 668d8b787e
13 changed files with 493 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/server/http.rs
View File

@@ -1,10 +1,16 @@
use super::*;
pub(super) fn send_sse_headers(stream: &mut impl Write) -> Result<(), String> {
pub(super) fn send_sse_headers_with_cors(
stream: &mut impl Write,
cors: bool,
) -> Result<(), String> {
let cors = if cors {
"Access-Control-Allow-Origin: *\r\nAccess-Control-Allow-Methods: GET, POST, OPTIONS\r\nAccess-Control-Allow-Headers: *\r\n"
} else {
""
};
stream
.write_all(
b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\nConnection: close\r\n\r\n",
)
.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\n{cors}Connection: close\r\n\r\n").as_bytes())
.map_err(|error| error.to_string())
}
@@ -27,25 +33,37 @@ pub(super) fn send_sse_error(stream: &mut impl Write, message: &str) -> Result<(
stream.write_all(b"\n\n").map_err(|error| error.to_string())
}
pub(super) fn send_json(stream: &mut TcpStream, code: u16, value: &Value) -> Result<(), String> {
pub(super) fn send_json_with_cors(
stream: &mut TcpStream,
code: u16,
value: &Value,
cors: bool,
) -> Result<(), String> {
let mut body = serde_json::to_vec(value).map_err(|error| error.to_string())?;
body.push(b'\n');
send_response(stream, code, Some("application/json"), &body)
send_response_with_cors(stream, code, Some("application/json"), &body, cors)
}
pub(super) fn send_error(stream: &mut TcpStream, code: u16, message: &str) -> Result<(), String> {
send_json(
pub(super) fn send_error_with_cors(
stream: &mut TcpStream,
code: u16,
message: &str,
cors: bool,
) -> Result<(), String> {
send_json_with_cors(
stream,
code,
&json!({"error": {"message": message, "type": "invalid_request_error"}}),
cors,
)
}
pub(super) fn send_response(
stream: &mut TcpStream,
pub(super) fn send_response_with_cors(
stream: &mut impl Write,
code: u16,
content_type: Option<&str>,
body: &[u8],
cors: bool,
) -> Result<(), String> {
let reason = match code {
200 => "OK",
@@ -65,6 +83,11 @@ pub(super) fn send_response(
header.push_str(content_type);
header.push_str("\r\n");
}
if cors {
header.push_str("Access-Control-Allow-Origin: *\r\n");
header.push_str("Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n");
header.push_str("Access-Control-Allow-Headers: *\r\n");
}
header.push_str("Connection: close\r\n\r\n");
stream
.write_all(header.as_bytes())

106
src/server/response.rs
View File

@@ -7,7 +7,11 @@ pub(super) fn final_response(
active: crate::runtime::ActiveGeneration,
id: &str,
) -> Result<(), (u16, String)> {
let output = wait_for_output(active)?;
let output = match wait_for_output(stream, active) {
Ok(output) => output,
Err(error) if error == "client disconnected" => return Ok(()),
Err(error) => return send_generation_error(stream, request.protocol, request.cors, error),
};
let (content, calls) = parse_generated_tools(state, &output.message.content, request.protocol);
let finish = if calls.is_empty() {
output.finish_reason
@@ -114,7 +118,7 @@ pub(super) fn final_response(
})
}
};
send_json(stream, 200, &body).map_err(|error| (500, error))
send_json_with_cors(stream, 200, &body, request.cors).map_err(|error| (500, error))
}
pub(super) fn stream_response(
@@ -158,7 +162,7 @@ fn completion_stream_response(
active: crate::runtime::ActiveGeneration,
id: &str,
) -> Result<(), (u16, String)> {
send_sse_headers(stream).map_err(|error| (500, error))?;
send_sse_headers_with_cors(stream, request.cors).map_err(|error| (500, error))?;
while let Ok(event) = active.events.recv() {
match event {
GenerationEvent::Chunk {
@@ -215,7 +219,7 @@ fn anthropic_stream_response(
active: crate::runtime::ActiveGeneration,
id: &str,
) -> Result<(), (u16, String)> {
send_sse_headers(stream).map_err(|error| (500, error))?;
send_sse_headers_with_cors(stream, request.cors).map_err(|error| (500, error))?;
let mut prompt_tokens = 0;
let mut started = false;
let mut block = None::<(usize, bool)>;
@@ -298,7 +302,7 @@ fn anthropic_stream_response(
)?;
}
if request.has_tools {
let events = projector.push("", true, "toolu_");
let events = projector.finish("toolu_");
send_anthropic_projection_events(
stream,
events,
@@ -468,7 +472,7 @@ fn responses_stream_response(
active: crate::runtime::ActiveGeneration,
id: &str,
) -> Result<(), (u16, String)> {
send_sse_headers(stream).map_err(|error| (500, error))?;
send_sse_headers_with_cors(stream, request.cors).map_err(|error| (500, error))?;
let created = unix_time();
let message_id = random_id("msg_");
let reasoning_id = random_id("rs_");
@@ -739,7 +743,8 @@ fn stream_response_with_keepalive(
} => {
prefilling = tokens_per_second.is_none();
if prefilling && !headers_sent {
send_sse_headers(stream).map_err(|error| (500, error))?;
send_sse_headers_with_cors(stream, request.cors)
.map_err(|error| (500, error))?;
headers_sent = true;
last_keepalive = Instant::now();
} else if !prefilling {
@@ -785,7 +790,12 @@ fn stream_response_with_keepalive(
let _ = send_sse_error(stream, &error);
return Ok(());
}
return Err((500, error));
return send_generation_error(
stream,
request.protocol,
request.cors,
error,
);
}
}
break;
@@ -801,7 +811,7 @@ fn stream_response_with_keepalive(
None => return Err((500, "The model runtime stopped unexpectedly.".into())),
};
if request.has_tools {
let events = projector.push("", true, "call_");
let events = projector.finish("call_");
send_chat_projection_events(stream, &request, id, events)?;
}
let (_, calls) = parse_generated_tools_with_ids(
@@ -885,7 +895,7 @@ pub(super) fn send_stream_start(
role_sent: &mut bool,
) -> Result<(), (u16, String)> {
if !*headers_sent {
send_sse_headers(stream).map_err(|error| (500, error))?;
send_sse_headers_with_cors(stream, request.cors).map_err(|error| (500, error))?;
*headers_sent = true;
}
if !*role_sent {
@@ -924,10 +934,28 @@ pub(super) fn receive_stream_event(
}
fn wait_for_output(
stream: &mut TcpStream,
active: crate::runtime::ActiveGeneration,
) -> Result<crate::engine::GenerationOutput, (u16, String)> {
) -> Result<crate::engine::GenerationOutput, String> {
let mut content = String::new();
while let Ok(event) = active.events.recv() {
stream
.set_nonblocking(true)
.map_err(|error| error.to_string())?;
loop {
let event = match active.events.recv_timeout(Duration::from_millis(100)) {
Ok(event) => event,
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
let mut byte = [0];
match stream.peek(&mut byte) {
Ok(0) => return Err("client disconnected".into()),
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {}
Err(error) => return Err(error.to_string()),
}
continue;
}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
};
match event {
GenerationEvent::Chunk {
reasoning: false,
@@ -942,12 +970,62 @@ fn wait_for_output(
}
}
GenerationEvent::Finished(result) => {
return result.map_err(|error| (500, error));
stream
.set_nonblocking(false)
.map_err(|error| error.to_string())?;
return result;
}
_ => {}
}
}
Err((500, "The model runtime stopped unexpectedly.".into()))
let _ = stream.set_nonblocking(false);
Err("The model runtime stopped unexpectedly.".into())
}
pub(super) fn send_generation_error(
stream: &mut TcpStream,
protocol: Protocol,
cors: bool,
error: String,
) -> Result<(), (u16, String)> {
let Some((prompt_tokens, context)) = context_error_dimensions(&error) else {
return Err((500, error));
};
let body = if protocol == Protocol::Anthropic {
json!({
"type": "error",
"error": {
"type": "invalid_request_error",
"message": error,
"n_prompt_tokens": prompt_tokens,
"n_ctx": context
}
})
} else {
let parameter = match protocol {
Protocol::Completion => "prompt",
Protocol::Responses => "input",
Protocol::Chat => "messages",
Protocol::Anthropic => unreachable!(),
};
json!({"error": {
"message": error,
"type": "invalid_request_error",
"param": parameter,
"code": "context_length_exceeded",
"n_prompt_tokens": prompt_tokens,
"n_ctx": context
}})
};
send_json_with_cors(stream, 400, &body, cors).map_err(|error| (500, error))
}
pub(super) fn context_error_dimensions(error: &str) -> Option<(u32, u32)> {
let values = error
.strip_prefix("Prompt has ")?
.strip_suffix(" tokens")?
.split_once(" tokens, but the configured context size is ")?;
Some((values.0.parse().ok()?, values.1.parse().ok()?))
}
pub(super) fn usage_json(prompt: u32, cached: u32, completion: u32) -> Value {

107
src/server/tools.rs
View File

@@ -1,5 +1,9 @@
use super::*;
const TOOL_MEMORY_FILE: &str = "tool-replay.json";
const TOOL_MEMORY_MAX_IDS: usize = 100_000;
const TOOL_MEMORY_MAX_BYTES: u64 = 512 * 1024 * 1024;
enum ToolProjectionState {
Seeking,
Invokes,
@@ -226,6 +230,15 @@ impl ToolProjector {
events
}
pub(super) fn finish(&mut self, prefix: &str) -> Vec<ToolProjectionEvent> {
if let Some(repaired) = repair_generated_tools(&self.raw) {
let suffix = repaired[self.raw.len()..].to_owned();
self.push(&suffix, true, prefix)
} else {
self.push("", true, prefix)
}
}
fn emit_value(&mut self, end: usize, events: &mut Vec<ToolProjectionEvent>) {
if end <= self.position {
return;
@@ -506,6 +519,17 @@ pub(super) fn parse_generated_tools_with_ids(
text: &str,
protocol: Protocol,
streamed_ids: &[String],
) -> (String, Vec<ApiToolCall>) {
let repaired = repair_generated_tools(text);
let text = repaired.as_deref().unwrap_or(text);
parse_generated_tools_once(state, text, protocol, streamed_ids)
}
fn parse_generated_tools_once(
state: &State,
text: &str,
protocol: Protocol,
streamed_ids: &[String],
) -> (String, Vec<ApiToolCall>) {
let Some((start, syntax)) = TOOL_SYNTAXES
.iter()
@@ -575,16 +599,95 @@ pub(super) fn parse_generated_tools_with_ids(
.tool_memory
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
// ponytail: one process-local replay table; add LRU eviction if 100k live tool ids is measured insufficient.
if memory.len() >= 100_000 {
// ponytail: clear-at-cap is cheaper than an LRU; add ordered eviction if real clients retain 100k live ids.
if memory.len() >= TOOL_MEMORY_MAX_IDS {
memory.clear();
}
for call in &calls {
memory.insert(call.id.clone(), raw.to_owned());
}
write_tool_memory(&state.cache_path, &memory);
(content.to_owned(), calls)
}
pub(super) fn read_tool_memory(directory: &std::path::Path) -> HashMap<String, String> {
if directory.as_os_str().is_empty() {
return HashMap::new();
}
let path = directory.join(TOOL_MEMORY_FILE);
if std::fs::metadata(&path).is_ok_and(|metadata| metadata.len() > TOOL_MEMORY_MAX_BYTES) {
return HashMap::new();
}
let Ok(file) = File::open(path) else {
return HashMap::new();
};
let Ok(memory) = serde_json::from_reader::<_, HashMap<String, String>>(file) else {
return HashMap::new();
};
if memory.len() <= TOOL_MEMORY_MAX_IDS {
memory
} else {
HashMap::new()
}
}
fn write_tool_memory(directory: &std::path::Path, memory: &HashMap<String, String>) {
if directory.as_os_str().is_empty() || std::fs::create_dir_all(directory).is_err() {
return;
}
let path = directory.join(TOOL_MEMORY_FILE);
let temporary = path.with_extension("json.tmp");
let Ok(mut file) = File::create(&temporary) else {
return;
};
if serde_json::to_writer(&mut file, memory).is_ok()
&& file.sync_all().is_ok()
&& file
.metadata()
.is_ok_and(|metadata| metadata.len() <= TOOL_MEMORY_MAX_BYTES)
{
let _ = std::fs::rename(&temporary, path);
} else {
let _ = std::fs::remove_file(temporary);
}
}
fn repair_generated_tools(text: &str) -> Option<String> {
let scan_start = text
.rfind("</think>")
.map_or(0, |position| position + "</think>".len());
let scan = &text[scan_start..];
let syntax = TOOL_SYNTAXES
.iter()
.filter_map(|syntax| scan.find(syntax.tool_start).map(|start| (start, *syntax)))
.min_by_key(|(start, _)| *start)?
.1;
let tool_open = scan.matches(syntax.tool_start).count();
let tool_close = scan.matches(syntax.tool_end).count();
let invoke_open = scan.matches(syntax.invoke_start).count();
let invoke_close = scan.matches(syntax.invoke_end).count();
let parameter_open = scan.matches(syntax.parameter_start).count();
let parameter_close = scan.matches(syntax.parameter_end).count();
if (tool_open, invoke_open, parameter_open) == (tool_close, invoke_close, parameter_close)
|| tool_close > tool_open
|| invoke_close > invoke_open
|| parameter_close > parameter_open
{
return None;
}
let mut repaired = text.to_owned();
for _ in parameter_close..parameter_open {
repaired.push_str(syntax.parameter_end);
}
for _ in invoke_close..invoke_open {
repaired.push_str(syntax.invoke_end);
}
for _ in tool_close..tool_open {
repaired.push_str(syntax.tool_end);
}
Some(repaired)
}
#[derive(Clone, Copy)]
pub(super) struct ToolSyntax {
pub(super) tool_start: &'static str,