From 4a457350729cc7518bc8476755a45fbe7ad85dee Mon Sep 17 00:00:00 2001 From: Georg Bauer Date: Sat, 25 Jul 2026 14:57:17 +0200 Subject: [PATCH] Add persistent local agent tools --- PLAN.md | 40 +- README.md | 18 +- build.rs | 6 + .../20260725140000_add_tool_messages/down.sql | 1 + .../20260725140000_add_tool_messages/up.sql | 2 + native/web/LICENSE | 22 + native/web/ds4_web.c | 1385 +++++++++++++++++ native/web/ds4_web.h | 33 + src/agent.rs | 1309 ++++++++++++++++ src/app.rs | 26 +- src/app/generation.rs | 148 +- src/app/view/chat.rs | 22 +- src/database.rs | 49 +- src/engine.rs | 8 +- src/engine/tokenizer.rs | 27 + src/engine/validation.rs | 26 + src/main.rs | 1 + src/schema.rs | 1 + src/server.rs | 43 + src/server/tools.rs | 9 +- 20 files changed, 3139 insertions(+), 37 deletions(-) create mode 100644 migrations/20260725140000_add_tool_messages/down.sql create mode 100644 migrations/20260725140000_add_tool_messages/up.sql create mode 100644 native/web/LICENSE create mode 100644 native/web/ds4_web.c create mode 100644 native/web/ds4_web.h create mode 100644 src/agent.rs diff --git a/PLAN.md b/PLAN.md index 9c052a8..c83d937 100644 --- a/PLAN.md +++ b/PLAN.md @@ -413,11 +413,14 @@ the same conversation without prefill when its KV payload is compatible. ## Phase 4 — agent chat and tools -Status: **basic local chat implemented**. The conversation area streams +Status: **solid local agent base implemented**. The conversation area streams multi-turn DeepSeek Flash output, persists and rehydrates structured reasoning and answers, renders Markdown, follows new output, restores sessions at their latest message, reports context use and generation speed, and supports Stop. -Tool turns and the coding-agent runtime remain open. +DeepSeek DSML and GLM tool turns now execute off the UI thread, persist semantic +tool roles, resume the same checkpoint, and expose the complete `ds4_agent.c` +starting tool set. Context compaction and richer partial-tool presentation +remain open. Every chat feature must persist its semantic state and rehydrate it when a session is reopened in the same implementation slice. In-memory-only chat @@ -431,23 +434,27 @@ ephemeral presentation preferences are the only exception. decode, HTTP, and KV telemetry; tool-call cards, queued input, and inline chat-prefill progress remain. Local turns must call the same session registry and KV path used by the HTTP endpoint. -2. Port the native agent turn loop from `ds4_agent.c`, including model-specific - system prompts and DeepSeek DSML/GLM tool-call parsing. -3. Implement the local tool set with the project directory as its boundary: +2. **Implemented:** port the native agent turn loop from `ds4_agent.c`, including + model-specific system prompts and DeepSeek DSML/GLM tool-call parsing. +3. **Implemented starting set:** implement the local tool set with the project directory as its boundary: - shell command execution - file read and continuation - file write - anchored edit - text/file search - - opt-in Dev Brain knowledge lookup and memory storage -4. Stream assistant tokens and partial tool calls into Iced while inference and - tools run on workers. Preserve cooperative interruption and a valid KV - prefix at every stop point. -5. Require confirmation for commands or writes outside the project boundary, - destructive actions, and network tools. Add web search/page visiting only - after the local tool loop is stable. -6. Port context compaction and system-prompt reinjection after ordinary - multi-turn operation is correct. + - directory listing + - asynchronous shell status/stop + - approved visible-Chrome Google search and rendered page visits + - opt-in Dev Brain knowledge lookup and memory storage remains separate +4. **Partially implemented:** assistant tokens and compact completed tool calls + stream into Iced while inference and tools run on workers. Cooperative Stop + covers generation and tool work; richer partial-call cards remain. +5. **Partially implemented:** file tools enforce the project boundary and web + tools require visible-browser approval. Destructive shell-command approval + remains before broader distribution. +6. The full tool system prompt is derived into every local generation, including + reopened sessions. Port context compaction before prompts approach their + configured context limit. ### Native macOS menus (required with chat) @@ -594,8 +601,9 @@ notarization, and delivery remain open. then porting every parser/route/state machine and its fixtures. Keep HTTP conversation state transient except for opaque KV cache files. **In progress; no route is considered complete until differential tests pass.** -3. Local transcript/KV persistence and the conversation UI are implemented; - port the agent tools after the shared endpoint lifecycle is stable. +3. Local transcript/KV persistence, the conversation UI, and the native agent + starting tool set are implemented. Add context compaction and richer + partial-tool surfaces next. 4. Add the opt-in Dev Brain tool after local file/search boundaries and agent approvals are stable. 5. Add bDS2-style A2UI render tools and native inline surfaces after the core diff --git a/README.md b/README.md index 8fea35a..fc333a5 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,11 @@ DS4Server is a native macOS coding-agent application that rewrites the DwarfStar (`ds4`) inference engine in Rust. It uses Rust and Iced and will combine local model loading, an OpenAI-compatible localhost endpoint, and project-scoped agent chat in one app. -DS4Server vendors and adapts the Metal kernels and Objective-C Metal glue from +DS4Server vendors and adapts the Metal kernels, Objective-C Metal glue, and +visible-Chrome web tool runtime from [DwarfStar (`ds4`)](https://github.com/antirez/ds4). Their copyright and license -notices are retained in [`native/metal/LICENSE`](native/metal/LICENSE). +notices are retained in [`native/metal/LICENSE`](native/metal/LICENSE) and +[`native/web/LICENSE`](native/web/LICENSE). The current milestone provides a Codex-inspired project/session layout. A native macOS folder picker selects each workspace, then the app asks for its display @@ -29,8 +31,16 @@ follow-up turns reuse durable transcript and KV state, and the model unloads after the idle timeout. The graph uses the full configured context with the ratio-4 sparse indexer. -The app also listens on `127.0.0.1:4000` by default for `GET /v1/models` and -`POST /v1/chat/completions`; the port is configurable in Preferences. The +Project chat includes the native `ds4_agent.c` starting tool set: bounded file +read/continuation, write, anchored edit, search, directory listing, asynchronous +shell jobs, Google search, and rendered page visits. Tool calls and results are +persisted as transcript roles and automatically continue the same model turn. +File tools stay inside the selected project. Web tools ask before starting a +visible Chrome profile. + +The app also listens on `127.0.0.1:4000` by default for Models, Chat +Completions, Completions, Anthropic Messages, and Responses APIs. The listener, +port, and opt-in CORS are configurable in Preferences. The endpoint and local chat share the single model owner. External conversations are client-managed and never enter the project, session, message, or transcript database; only opaque content-addressed KV cache files are retained. Model diff --git a/build.rs b/build.rs index 49c514e..90717cd 100644 --- a/build.rs +++ b/build.rs @@ -13,6 +13,12 @@ fn main() { .flag("-fobjc-arc") .opt_level(3) .compile("ds4_metal"); + println!("cargo:rerun-if-changed=native/web"); + cc::Build::new() + .include("native/web") + .file("native/web/ds4_web.c") + .opt_level(2) + .compile("ds4_web"); println!("cargo:rustc-link-lib=framework=Foundation"); println!("cargo:rustc-link-lib=framework=Metal"); } diff --git a/migrations/20260725140000_add_tool_messages/down.sql b/migrations/20260725140000_add_tool_messages/down.sql new file mode 100644 index 0000000..b47d1d2 --- /dev/null +++ b/migrations/20260725140000_add_tool_messages/down.sql @@ -0,0 +1 @@ +ALTER TABLE messages DROP COLUMN tool; diff --git a/migrations/20260725140000_add_tool_messages/up.sql b/migrations/20260725140000_add_tool_messages/up.sql new file mode 100644 index 0000000..bf397d0 --- /dev/null +++ b/migrations/20260725140000_add_tool_messages/up.sql @@ -0,0 +1,2 @@ +ALTER TABLE messages ADD COLUMN tool BOOLEAN NOT NULL DEFAULT FALSE + CHECK (tool IN (FALSE, TRUE)); diff --git a/native/web/LICENSE b/native/web/LICENSE new file mode 100644 index 0000000..5973a4c --- /dev/null +++ b/native/web/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2026 The ds4.c authors +Copyright (c) 2023-2026 The ggml authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/native/web/ds4_web.c b/native/web/ds4_web.c new file mode 100644 index 0000000..fb1b666 --- /dev/null +++ b/native/web/ds4_web.c @@ -0,0 +1,1385 @@ +#include "ds4_web.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +#define DS4_WEB_DEFAULT_PORT 9333 +#define DS4_WEB_CONNECT_TIMEOUT_MS 3000 +#define DS4_WEB_CDP_TIMEOUT_MS 20000 +#define DS4_WEB_MAX_RESULT_BYTES (1024*1024) + +typedef struct { + char *ptr; + size_t len; + size_t cap; +} web_buf; + +struct ds4_web { + char home[PATH_MAX]; + char profile_dir[PATH_MAX]; + int port; + pid_t chrome_pid; + bool browser_allowed; + ds4_web_confirm_fn confirm; + void *confirm_privdata; + ds4_web_log_fn log; + void *log_privdata; + ds4_web_cancel_fn cancel; + void *cancel_privdata; + int next_cdp_id; +}; + +typedef struct { + int fd; + int next_id; + ds4_web *web; +} cdp_ws; + +typedef struct { + char *id; + char *ws_url; +} web_tab; + +static void *web_xmalloc(size_t n) { + void *p = malloc(n ? n : 1); + if (!p) { + perror("ds4_web: malloc"); + exit(1); + } + return p; +} + +static char *web_xstrdup(const char *s) { + if (!s) s = ""; + size_t n = strlen(s); + char *p = web_xmalloc(n + 1); + memcpy(p, s, n + 1); + return p; +} + +static void web_buf_append(web_buf *b, const char *s, size_t n) { + if (!n) return; + if (b->len + n + 1 > b->cap) { + size_t cap = b->cap ? b->cap * 2 : 4096; + while (cap < b->len + n + 1) cap *= 2; + char *p = realloc(b->ptr, cap); + if (!p) { + perror("ds4_web: realloc"); + exit(1); + } + b->ptr = p; + b->cap = cap; + } + memcpy(b->ptr + b->len, s, n); + b->len += n; + b->ptr[b->len] = '\0'; +} + +static void web_buf_puts(web_buf *b, const char *s) { + web_buf_append(b, s, strlen(s)); +} + +static char *web_buf_take(web_buf *b) { + if (!b->ptr) return web_xstrdup(""); + char *p = b->ptr; + b->ptr = NULL; + b->len = b->cap = 0; + return p; +} + +static void web_set_err(char *err, size_t err_len, const char *fmt, ...) { + if (!err || err_len == 0) return; + va_list ap; + va_start(ap, fmt); + vsnprintf(err, err_len, fmt, ap); + va_end(ap); +} + +static void web_log(ds4_web *web, const char *msg) { + if (web && web->log) web->log(web->log_privdata, msg); +} + +static double web_now_sec(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0; +} + +static bool web_cancelled(ds4_web *web) { + return web && web->cancel && web->cancel(web->cancel_privdata); +} + +static bool web_set_cancel_err(ds4_web *web, char *err, size_t err_len) { + if (!web_cancelled(web)) return false; + web_set_err(err, err_len, "interrupted"); + return true; +} + +static bool web_sleep_ms(ds4_web *web, int ms) { + int left = ms; + while (left > 0) { + if (web_cancelled(web)) return false; + int step = left < 50 ? left : 50; + usleep((useconds_t)step * 1000u); + left -= step; + } + return !web_cancelled(web); +} + +static bool web_err_is_interrupted(const char *err) { + return err && !strcmp(err, "interrupted"); +} + +static bool web_mkdir_p(const char *path) { + if (!path || !path[0]) return false; + char tmp[PATH_MAX]; + snprintf(tmp, sizeof(tmp), "%s", path); + for (char *p = tmp + 1; *p; p++) { + if (*p != '/') continue; + *p = '\0'; + if (mkdir(tmp, 0700) != 0 && errno != EEXIST) return false; + *p = '/'; + } + return mkdir(tmp, 0700) == 0 || errno == EEXIST; +} + +static int web_tcp_connect(const char *host, int port, int timeout_ms, + char *err, size_t err_len) { + char service[32]; + snprintf(service, sizeof(service), "%d", port); + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; + struct addrinfo *res = NULL; + int gai = getaddrinfo(host, service, &hints, &res); + if (gai != 0) { + web_set_err(err, err_len, "getaddrinfo %s: %s", host, gai_strerror(gai)); + return -1; + } + + int fd = -1; + for (struct addrinfo *ai = res; ai; ai = ai->ai_next) { + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) continue; + int flags = fcntl(fd, F_GETFL, 0); + if (flags >= 0) fcntl(fd, F_SETFL, flags | O_NONBLOCK); + int rc = connect(fd, ai->ai_addr, ai->ai_addrlen); + if (rc == 0) { + if (flags >= 0) fcntl(fd, F_SETFL, flags); + break; + } + if (errno == EINPROGRESS) { + struct pollfd pfd = {.fd = fd, .events = POLLOUT}; + rc = poll(&pfd, 1, timeout_ms); + if (rc > 0) { + int soerr = 0; + socklen_t slen = sizeof(soerr); + getsockopt(fd, SOL_SOCKET, SO_ERROR, &soerr, &slen); + if (soerr == 0) { + if (flags >= 0) fcntl(fd, F_SETFL, flags); + break; + } + errno = soerr; + } + } + close(fd); + fd = -1; + } + freeaddrinfo(res); + if (fd < 0) web_set_err(err, err_len, "connect %s:%d failed: %s", + host, port, strerror(errno)); + return fd; +} + +static int web_write_all(int fd, const void *buf, size_t len) { + const char *p = buf; + while (len) { +#ifdef MSG_NOSIGNAL + ssize_t n = send(fd, p, len, MSG_NOSIGNAL); +#else + ssize_t n = write(fd, p, len); +#endif + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return -1; + p += n; + len -= (size_t)n; + } + return 0; +} + +static ssize_t web_read_some(int fd, char *buf, size_t len, int timeout_ms) { + struct pollfd pfd = {.fd = fd, .events = POLLIN}; + int rc = poll(&pfd, 1, timeout_ms); + if (rc <= 0) return rc == 0 ? 0 : -1; + for (;;) { + ssize_t n = read(fd, buf, len); + if (n < 0 && errno == EINTR) continue; + return n; + } +} + +static char *web_http_request(const char *method, int port, const char *path, + char *err, size_t err_len) { + int fd = web_tcp_connect("127.0.0.1", port, DS4_WEB_CONNECT_TIMEOUT_MS, + err, err_len); + if (fd < 0) return NULL; + web_buf req = {0}; + char line[512]; + snprintf(line, sizeof(line), + "%s %s HTTP/1.1\r\nHost: 127.0.0.1:%d\r\nConnection: close\r\n\r\n", + method, path, port); + web_buf_puts(&req, line); + if (web_write_all(fd, req.ptr, req.len) != 0) { + web_set_err(err, err_len, "write HTTP request failed: %s", strerror(errno)); + close(fd); + free(req.ptr); + return NULL; + } + free(req.ptr); + + web_buf resp = {0}; + char tmp[4096]; + for (;;) { + ssize_t n = web_read_some(fd, tmp, sizeof(tmp), DS4_WEB_CONNECT_TIMEOUT_MS); + if (n < 0) { + web_set_err(err, err_len, "read HTTP response failed: %s", strerror(errno)); + close(fd); + free(resp.ptr); + return NULL; + } + if (n == 0) break; + web_buf_append(&resp, tmp, (size_t)n); + } + close(fd); + if (!resp.ptr) { + web_set_err(err, err_len, "empty HTTP response"); + return NULL; + } + char *body = strstr(resp.ptr, "\r\n\r\n"); + if (!body) { + web_set_err(err, err_len, "malformed HTTP response"); + free(resp.ptr); + return NULL; + } + body += 4; + char *out = web_xstrdup(body); + free(resp.ptr); + return out; +} + +static bool web_cdp_alive(ds4_web *web) { + char err[160] = {0}; + char *body = web_http_request("GET", web->port, "/json/version", err, sizeof(err)); + if (!body) return false; + bool ok = strstr(body, "webSocketDebuggerUrl") != NULL; + free(body); + return ok; +} + +static char *web_json_get_string(const char *json, const char *key); + +static char *web_url_encode(const char *s) { + static const char hex[] = "0123456789ABCDEF"; + web_buf b = {0}; + for (const unsigned char *p = (const unsigned char *)s; p && *p; p++) { + unsigned char c = *p; + if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { + web_buf_append(&b, (const char *)&c, 1); + } else { + char e[3] = {'%', hex[c >> 4], hex[c & 15]}; + web_buf_append(&b, e, 3); + } + } + return web_buf_take(&b); +} + +static void web_random_bytes(unsigned char *buf, size_t len) { + int fd = open("/dev/urandom", O_RDONLY); + if (fd >= 0) { + size_t off = 0; + while (off < len) { + ssize_t n = read(fd, buf + off, len - off); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + off += (size_t)n; + } + close(fd); + if (off == len) return; + } + uint64_t x = (uint64_t)time(NULL) ^ ((uint64_t)getpid() << 32); + for (size_t i = 0; i < len; i++) { + x = x * 6364136223846793005ULL + 1; + buf[i] = (unsigned char)(x >> 32); + } +} + +static char *web_base64(const unsigned char *data, size_t len) { + static const char tab[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + size_t outlen = ((len + 2) / 3) * 4; + char *out = web_xmalloc(outlen + 1); + size_t j = 0; + for (size_t i = 0; i < len; i += 3) { + uint32_t v = (uint32_t)data[i] << 16; + if (i + 1 < len) v |= (uint32_t)data[i + 1] << 8; + if (i + 2 < len) v |= data[i + 2]; + out[j++] = tab[(v >> 18) & 63]; + out[j++] = tab[(v >> 12) & 63]; + out[j++] = (i + 1 < len) ? tab[(v >> 6) & 63] : '='; + out[j++] = (i + 2 < len) ? tab[v & 63] : '='; + } + out[j] = '\0'; + return out; +} + +static char *web_json_quote(const char *s) { + web_buf b = {0}; + web_buf_puts(&b, "\""); + for (const unsigned char *p = (const unsigned char *)s; p && *p; p++) { + unsigned char c = *p; + switch (c) { + case '\\': web_buf_puts(&b, "\\\\"); break; + case '"': web_buf_puts(&b, "\\\""); break; + case '\n': web_buf_puts(&b, "\\n"); break; + case '\r': web_buf_puts(&b, "\\r"); break; + case '\t': web_buf_puts(&b, "\\t"); break; + default: + if (c < 0x20) { + char tmp[8]; + snprintf(tmp, sizeof(tmp), "\\u%04x", c); + web_buf_puts(&b, tmp); + } else { + web_buf_append(&b, (const char *)&c, 1); + } + break; + } + } + web_buf_puts(&b, "\""); + return web_buf_take(&b); +} + +static int web_ws_connect(const char *ws_url, cdp_ws *ws, + char *err, size_t err_len) { + const char *p = ws_url; + if (strncmp(p, "ws://", 5) != 0) { + web_set_err(err, err_len, "unsupported websocket URL: %s", ws_url); + return -1; + } + p += 5; + const char *slash = strchr(p, '/'); + if (!slash) { + web_set_err(err, err_len, "malformed websocket URL"); + return -1; + } + char hostport[256]; + size_t hp_len = (size_t)(slash - p); + if (hp_len >= sizeof(hostport)) hp_len = sizeof(hostport) - 1; + memcpy(hostport, p, hp_len); + hostport[hp_len] = '\0'; + char *colon = strrchr(hostport, ':'); + int port = 80; + if (colon) { + *colon = '\0'; + port = atoi(colon + 1); + } + const char *host = hostport; + int fd = web_tcp_connect(host, port, DS4_WEB_CONNECT_TIMEOUT_MS, err, err_len); + if (fd < 0) return -1; + + unsigned char rnd[16]; + web_random_bytes(rnd, sizeof(rnd)); + char *key = web_base64(rnd, sizeof(rnd)); + web_buf req = {0}; + char line[512]; + snprintf(line, sizeof(line), + "GET %s HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key: %s\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n", + slash, host, port, key); + web_buf_puts(&req, line); + free(key); + if (web_write_all(fd, req.ptr, req.len) != 0) { + web_set_err(err, err_len, "websocket handshake write failed"); + close(fd); + free(req.ptr); + return -1; + } + free(req.ptr); + + web_buf resp = {0}; + char tmp[1024]; + double deadline = web_now_sec() + (double)DS4_WEB_CONNECT_TIMEOUT_MS / 1000.0; + while (!strstr(resp.ptr ? resp.ptr : "", "\r\n\r\n")) { + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) { + close(fd); + free(resp.ptr); + return -1; + } + double now = web_now_sec(); + if (now >= deadline) { + web_set_err(err, err_len, "websocket handshake read failed"); + close(fd); + free(resp.ptr); + return -1; + } + int slice = 100; + int remaining = (int)((deadline - now) * 1000.0); + if (remaining < slice) slice = remaining > 0 ? remaining : 1; + ssize_t n = web_read_some(fd, tmp, sizeof(tmp), slice); + if (n < 0) { + web_set_err(err, err_len, "websocket handshake read failed"); + close(fd); + free(resp.ptr); + return -1; + } + if (n == 0) continue; + web_buf_append(&resp, tmp, (size_t)n); + if (resp.len > 8192) break; + } + bool ok = resp.ptr && strstr(resp.ptr, " 101 ") != NULL; + free(resp.ptr); + if (!ok) { + web_set_err(err, err_len, "websocket handshake rejected"); + close(fd); + return -1; + } + ws->fd = fd; + ws->next_id = 1; + return 0; +} + +static void web_ws_close(cdp_ws *ws) { + if (ws && ws->fd >= 0) { + close(ws->fd); + ws->fd = -1; + } +} + +static int web_read_exact(cdp_ws *ws, unsigned char *buf, size_t len, + int timeout_ms, char *err, size_t err_len) { + size_t off = 0; + double deadline = web_now_sec() + (double)timeout_ms / 1000.0; + while (off < len) { + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) return -1; + double now = web_now_sec(); + if (now >= deadline) { + web_set_err(err, err_len, "websocket read timeout"); + return -1; + } + int slice = 100; + int remaining = (int)((deadline - now) * 1000.0); + if (remaining < slice) slice = remaining > 0 ? remaining : 1; + ssize_t n = web_read_some(ws->fd, (char *)buf + off, len - off, slice); + if (n < 0) { + web_set_err(err, err_len, "websocket frame read failed"); + return -1; + } + if (n == 0) continue; + off += (size_t)n; + } + return 0; +} + +static int web_ws_send_text(cdp_ws *ws, const char *text, + char *err, size_t err_len) { + size_t len = strlen(text); + web_buf frame = {0}; + unsigned char hdr[14]; + size_t h = 0; + hdr[h++] = 0x81; + if (len < 126) { + hdr[h++] = 0x80 | (unsigned char)len; + } else if (len <= 0xffff) { + hdr[h++] = 0x80 | 126; + hdr[h++] = (unsigned char)(len >> 8); + hdr[h++] = (unsigned char)len; + } else { + hdr[h++] = 0x80 | 127; + for (int i = 7; i >= 0; i--) hdr[h++] = (unsigned char)((uint64_t)len >> (i * 8)); + } + unsigned char mask[4]; + web_random_bytes(mask, sizeof(mask)); + for (int i = 0; i < 4; i++) hdr[h++] = mask[i]; + web_buf_append(&frame, (const char *)hdr, h); + for (size_t i = 0; i < len; i++) { + char c = text[i] ^ mask[i & 3]; + web_buf_append(&frame, &c, 1); + } + int rc = web_write_all(ws->fd, frame.ptr, frame.len); + free(frame.ptr); + if (rc != 0) { + web_set_err(err, err_len, "websocket write failed: %s", strerror(errno)); + return -1; + } + return 0; +} + +static int web_ws_send_pong(cdp_ws *ws, const unsigned char *payload, size_t len) { + if (len > 125) len = 125; + unsigned char hdr[2 + 4 + 125]; + hdr[0] = 0x8a; + hdr[1] = 0x80 | (unsigned char)len; + unsigned char mask[4]; + web_random_bytes(mask, sizeof(mask)); + memcpy(hdr + 2, mask, 4); + for (size_t i = 0; i < len; i++) hdr[6 + i] = payload[i] ^ mask[i & 3]; + return web_write_all(ws->fd, hdr, 6 + len); +} + +static char *web_ws_read_message(cdp_ws *ws, char *err, size_t err_len) { + web_buf msg = {0}; + for (;;) { + unsigned char h[2]; + if (web_read_exact(ws, h, 2, DS4_WEB_CDP_TIMEOUT_MS, err, err_len) != 0) { + free(msg.ptr); + return NULL; + } + bool fin = (h[0] & 0x80) != 0; + int opcode = h[0] & 0x0f; + bool masked = (h[1] & 0x80) != 0; + uint64_t len = h[1] & 0x7f; + if (len == 126) { + unsigned char x[2]; + if (web_read_exact(ws, x, 2, DS4_WEB_CDP_TIMEOUT_MS, err, err_len) != 0) goto fail; + len = ((uint64_t)x[0] << 8) | x[1]; + } else if (len == 127) { + unsigned char x[8]; + if (web_read_exact(ws, x, 8, DS4_WEB_CDP_TIMEOUT_MS, err, err_len) != 0) goto fail; + len = 0; + for (int i = 0; i < 8; i++) len = (len << 8) | x[i]; + } + unsigned char mask[4] = {0}; + if (masked && web_read_exact(ws, mask, 4, DS4_WEB_CDP_TIMEOUT_MS, err, err_len) != 0) + goto fail; + if (len > DS4_WEB_MAX_RESULT_BYTES * 4ULL) { + web_set_err(err, err_len, "websocket message too large"); + free(msg.ptr); + return NULL; + } + unsigned char *payload = web_xmalloc((size_t)len + 1); + if (len && web_read_exact(ws, payload, (size_t)len, + DS4_WEB_CDP_TIMEOUT_MS, err, err_len) != 0) { + free(payload); + goto fail; + } + for (uint64_t i = 0; masked && i < len; i++) payload[i] ^= mask[i & 3]; + payload[len] = '\0'; + if (opcode == 0x8) { + free(payload); + web_set_err(err, err_len, "websocket closed"); + free(msg.ptr); + return NULL; + } else if (opcode == 0x9) { + web_ws_send_pong(ws, payload, (size_t)len); + free(payload); + continue; + } else if (opcode == 0x1 || opcode == 0x0) { + web_buf_append(&msg, (const char *)payload, (size_t)len); + free(payload); + if (fin) return web_buf_take(&msg); + } else { + free(payload); + } + } +fail: + if (err && err_len && !err[0]) web_set_err(err, err_len, "websocket frame read failed"); + free(msg.ptr); + return NULL; +} + +static bool web_json_id_matches(const char *json, int id) { + const char *p = strstr(json, "\"id\""); + if (!p) return false; + p = strchr(p, ':'); + if (!p) return false; + p++; + while (*p == ' ' || *p == '\t') p++; + return atoi(p) == id; +} + +static char *web_cdp_call(cdp_ws *ws, const char *method, const char *params, + char *err, size_t err_len) { + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) return NULL; + int id = ws->next_id++; + web_buf req = {0}; + char head[256]; + snprintf(head, sizeof(head), "{\"id\":%d,\"method\":", id); + web_buf_puts(&req, head); + char *qmethod = web_json_quote(method); + web_buf_puts(&req, qmethod); + free(qmethod); + if (params && params[0]) { + web_buf_puts(&req, ",\"params\":"); + web_buf_puts(&req, params); + } + web_buf_puts(&req, "}"); + char *wire = web_buf_take(&req); + if (web_ws_send_text(ws, wire, err, err_len) != 0) { + free(wire); + return NULL; + } + free(wire); + for (;;) { + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) return NULL; + char *msg = web_ws_read_message(ws, err, err_len); + if (!msg) return NULL; + if (web_json_id_matches(msg, id)) return msg; + free(msg); + } +} + +static void web_cdp_call_optional(cdp_ws *ws, const char *method, const char *params) { + char err[160] = {0}; + char *resp = web_cdp_call(ws, method, params, err, sizeof(err)); + free(resp); +} + +static int web_hex4(const char *p) { + int v = 0; + for (int i = 0; i < 4; i++) { + char c = p[i]; + int x; + if (c >= '0' && c <= '9') x = c - '0'; + else if (c >= 'a' && c <= 'f') x = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') x = c - 'A' + 10; + else return -1; + v = (v << 4) | x; + } + return v; +} + +static void web_utf8_append(web_buf *b, unsigned code) { + char out[4]; + if (code <= 0x7f) { + out[0] = (char)code; + web_buf_append(b, out, 1); + } else if (code <= 0x7ff) { + out[0] = (char)(0xc0 | (code >> 6)); + out[1] = (char)(0x80 | (code & 0x3f)); + web_buf_append(b, out, 2); + } else if (code <= 0xffff) { + out[0] = (char)(0xe0 | (code >> 12)); + out[1] = (char)(0x80 | ((code >> 6) & 0x3f)); + out[2] = (char)(0x80 | (code & 0x3f)); + web_buf_append(b, out, 3); + } else { + out[0] = (char)(0xf0 | (code >> 18)); + out[1] = (char)(0x80 | ((code >> 12) & 0x3f)); + out[2] = (char)(0x80 | ((code >> 6) & 0x3f)); + out[3] = (char)(0x80 | (code & 0x3f)); + web_buf_append(b, out, 4); + } +} + +static char *web_json_parse_string_at(const char *q, const char **endp) { + if (*q != '"') return NULL; + q++; + web_buf b = {0}; + while (*q && *q != '"') { + if (*q != '\\') { + web_buf_append(&b, q++, 1); + continue; + } + q++; + switch (*q) { + case '"': web_buf_append(&b, "\"", 1); q++; break; + case '\\': web_buf_append(&b, "\\", 1); q++; break; + case '/': web_buf_append(&b, "/", 1); q++; break; + case 'b': web_buf_append(&b, "\b", 1); q++; break; + case 'f': web_buf_append(&b, "\f", 1); q++; break; + case 'n': web_buf_append(&b, "\n", 1); q++; break; + case 'r': web_buf_append(&b, "\r", 1); q++; break; + case 't': web_buf_append(&b, "\t", 1); q++; break; + case 'u': { + int v = web_hex4(q + 1); + if (v < 0) { free(b.ptr); return NULL; } + q += 5; + if (v >= 0xd800 && v <= 0xdbff && q[0] == '\\' && q[1] == 'u') { + int lo = web_hex4(q + 2); + if (lo >= 0xdc00 && lo <= 0xdfff) { + unsigned code = 0x10000 + (((unsigned)v - 0xd800) << 10) + + ((unsigned)lo - 0xdc00); + web_utf8_append(&b, code); + q += 6; + break; + } + } + web_utf8_append(&b, (unsigned)v); + break; + } + default: + if (*q) web_buf_append(&b, q++, 1); + break; + } + } + if (*q != '"') { + free(b.ptr); + return NULL; + } + if (endp) *endp = q + 1; + return web_buf_take(&b); +} + +static char *web_json_get_string(const char *json, const char *key) { + char pat[128]; + snprintf(pat, sizeof(pat), "\"%s\"", key); + const char *p = json; + while ((p = strstr(p, pat)) != NULL) { + p += strlen(pat); + while (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') p++; + if (*p++ != ':') continue; + while (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') p++; + if (*p == '"') return web_json_parse_string_at(p, NULL); + } + return NULL; +} + +static char *web_cdp_eval_string(cdp_ws *ws, const char *expr, + char *err, size_t err_len) { + char *qexpr = web_json_quote(expr); + web_buf params = {0}; + web_buf_puts(¶ms, "{\"expression\":"); + web_buf_puts(¶ms, qexpr); + web_buf_puts(¶ms, ",\"returnByValue\":true,\"awaitPromise\":true,\"includeCommandLineAPI\":true}"); + free(qexpr); + char *params_s = web_buf_take(¶ms); + char *resp = web_cdp_call(ws, "Runtime.evaluate", params_s, err, err_len); + free(params_s); + if (!resp) return NULL; + if (strstr(resp, "\"exceptionDetails\"")) { + web_set_err(err, err_len, "JavaScript evaluation failed"); + free(resp); + return NULL; + } + char *val = web_json_get_string(resp, "value"); + free(resp); + if (!val) web_set_err(err, err_len, "Runtime.evaluate did not return a string"); + return val; +} + +static bool web_wait_ready(cdp_ws *ws, char *err, size_t err_len) { + const char *expr = "document.readyState"; + for (int i = 0; i < 80; i++) { + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) return false; + char *state = web_cdp_eval_string(ws, expr, err, err_len); + if (state && (!strcmp(state, "complete") || !strcmp(state, "interactive"))) { + free(state); + if (web_sleep_ms(ws ? ws->web : NULL, 800)) return true; + web_set_err(err, err_len, "interrupted"); + return false; + } + free(state); + if (!web_sleep_ms(ws ? ws->web : NULL, 250)) { + web_set_err(err, err_len, "interrupted"); + return false; + } + } + return true; +} + +static bool web_cdp_navigate(cdp_ws *ws, const char *url, + char *err, size_t err_len) { + char *qurl = web_json_quote(url); + web_buf params = {0}; + web_buf_puts(¶ms, "{\"url\":"); + web_buf_puts(¶ms, qurl); + web_buf_puts(¶ms, "}"); + free(qurl); + char *params_s = web_buf_take(¶ms); + char *resp = web_cdp_call(ws, "Page.navigate", params_s, err, err_len); + free(params_s); + if (!resp) return false; + free(resp); + return true; +} + +static bool web_page_probe(cdp_ws *ws, char **href_out, char **ready_out, + long *text_len_out, char *err, size_t err_len) { + const char *expr = + "location.href+'\\n'+document.readyState+'\\n'+" + "((document.body&&document.body.innerText)||'').length"; + char *probe = web_cdp_eval_string(ws, expr, err, err_len); + if (!probe) return false; + + char *nl1 = strchr(probe, '\n'); + char *nl2 = nl1 ? strchr(nl1 + 1, '\n') : NULL; + if (!nl1 || !nl2) { + free(probe); + web_set_err(err, err_len, "page readiness probe returned malformed data"); + return false; + } + *nl1 = '\0'; + *nl2 = '\0'; + if (href_out) *href_out = web_xstrdup(probe); + if (ready_out) *ready_out = web_xstrdup(nl1 + 1); + if (text_len_out) *text_len_out = strtol(nl2 + 1, NULL, 10); + free(probe); + return true; +} + +static bool web_wait_navigated_ready(cdp_ws *ws, const char *url, + char *err, size_t err_len) { + (void)url; + long last_len = -1; + int stable = 0; + bool saw_real_url = false; + + for (int i = 0; i < 100; i++) { + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) return false; + char *href = NULL; + char *ready = NULL; + long text_len = 0; + bool ok = web_page_probe(ws, &href, &ready, &text_len, err, err_len); + if (!ok) { + free(href); + free(ready); + if (!web_sleep_ms(ws ? ws->web : NULL, 250)) { + web_set_err(err, err_len, "interrupted"); + return false; + } + continue; + } + + bool real_url = href && href[0] && + strcmp(href, "about:blank") && + strncmp(href, "chrome://", 9); + bool ready_state = ready && + (!strcmp(ready, "complete") || !strcmp(ready, "interactive")); + if (real_url) saw_real_url = true; + if (text_len > 0 && text_len == last_len) stable++; + else stable = 0; + last_len = text_len; + + free(href); + free(ready); + + if (saw_real_url && ready_state && text_len > 0 && stable >= 2) { + if (web_sleep_ms(ws ? ws->web : NULL, 500)) return true; + web_set_err(err, err_len, "interrupted"); + return false; + } + if (saw_real_url && ready_state && i >= 24) return true; + if (!web_sleep_ms(ws ? ws->web : NULL, 250)) { + web_set_err(err, err_len, "interrupted"); + return false; + } + } + return true; +} + +static bool web_cdp_prepare_page(cdp_ws *ws, char *err, size_t err_len) { + char *resp = web_cdp_call(ws, "Page.enable", "{}", err, err_len); + if (!resp) return false; + free(resp); + resp = web_cdp_call(ws, "Runtime.enable", "{}", err, err_len); + if (!resp) return false; + free(resp); + web_cdp_call_optional(ws, "Emulation.setFocusEmulationEnabled", + "{\"enabled\":true}"); + web_cdp_call_optional(ws, "Emulation.setDeviceMetricsOverride", + "{\"width\":1365,\"height\":900,\"deviceScaleFactor\":1,\"mobile\":false}"); + return web_wait_ready(ws, err, err_len); +} + +static bool web_scroll_dynamic_page(cdp_ws *ws, char *err, size_t err_len) { + const char *expr = + "(() => new Promise(resolve => {" + "const root=()=>document.scrollingElement||document.documentElement||document.body;" + "const blockSel='h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id=\"content-text\"],[class*=\"comment-body\"],[class*=\"comment-content\"],[data-testid*=\"comment-text\"]';" + "const lazySel='[onscroll],[loading=\"lazy\"],[data-src],[data-lazy],[class*=\"lazy\"],[class*=\"infinite\"],[class*=\"virtual\"],[role=\"feed\"],[id*=\"comment\"],[class*=\"comment\"],[data-testid*=\"comment\"]';" + "const hookCount=()=>{let n=0;try{if(window.onscroll)n++;if(document.onscroll)n++;if(document.body&&document.body.onscroll)n++;}catch(e){}" + "try{if(typeof getEventListeners==='function'){for(const o of [window,document,document.body]){if(!o)continue;const ev=getEventListeners(o);if(ev&&ev.scroll)n+=ev.scroll.length;}}}catch(e){}" + "try{n+=document.querySelectorAll(lazySel).length;}catch(e){}return n;};" + "const metrics=()=>{const r=root();return {" + "height:r?r.scrollHeight:0," + "view:innerHeight||900," + "y:scrollY||(r&&r.scrollTop)||0," + "text:((document.body&&document.body.innerText)||'').length," + "links:document.links?document.links.length:0," + "blocks:document.body?document.body.querySelectorAll(blockSel).length:0," + "hooks:hookCount()};};" + "const sig=m=>[m.height,m.text,m.links,m.blocks].join('|');" + "const grew=(a,b)=>b.height>a.height+20||b.text>a.text+200||b.links>a.links+2||b.blocks>a.blocks+2;" + "const scrollOnce=()=>{const r=root();if(!r)return;" + "const h=Math.max(700,Math.floor((innerHeight||900)*0.85));" + "window.scrollTo(0,Math.min(r.scrollHeight,(scrollY||r.scrollTop||0)+h));};" + "let last=metrics(),lastSig=sig(last),same=0,steps=0;" + "const scrollable=last.height>last.view*1.35;" + "if(!scrollable||last.hooks===0){resolve('scroll skipped hooks='+last.hooks+' text='+last.text);return;}" + "const tick=()=>{" + "if(steps>=28){resolve('scrolled '+steps+' text='+last.text);return;}" + "const before=last;" + "scrollOnce();steps++;" + "setTimeout(()=>{const now=metrics(),nowSig=sig(now);" + "if(nowSig===lastSig)same++;else same=0;" + "const loaded=grew(before,now);" + "last=now;lastSig=nowSig;" + "if(steps===1&&!loaded){resolve('scroll probe unchanged text='+now.text);return;}" + "const atBottom=now.y+now.view+20>=now.height;" + "if(same>=4||(atBottom&&same>=1)){resolve('scrolled '+steps+' text='+now.text);return;}" + "tick();},900);" + "};tick();" + "}))()"; + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) return false; + + char local_err[160] = {0}; + char *res = web_cdp_eval_string(ws, expr, local_err, sizeof(local_err)); + if (!res && web_err_is_interrupted(local_err)) { + web_set_err(err, err_len, "interrupted"); + return false; + } + free(res); + if (web_set_cancel_err(ws ? ws->web : NULL, err, err_len)) return false; + return true; +} + +static char *web_chrome_executable(void) { + const char *env = getenv("DS4_CHROME"); + if (env && env[0]) return web_xstrdup(env); +#ifdef __APPLE__ + if (access("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", X_OK) == 0) + return web_xstrdup("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"); + if (access("/Applications/Chromium.app/Contents/MacOS/Chromium", X_OK) == 0) + return web_xstrdup("/Applications/Chromium.app/Contents/MacOS/Chromium"); +#endif + const char *paths[] = { + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + "/opt/google/chrome/chrome", + NULL + }; + for (int i = 0; paths[i]; i++) { + if (access(paths[i], X_OK) == 0) return web_xstrdup(paths[i]); + } + + const char *names[] = { + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + NULL + }; + const char *pathenv = getenv("PATH"); + if (pathenv) { + char *path = web_xstrdup(pathenv); + char *save = NULL; + for (char *dir = strtok_r(path, ":", &save); dir; dir = strtok_r(NULL, ":", &save)) { + for (int i = 0; names[i]; i++) { + char candidate[PATH_MAX]; + snprintf(candidate, sizeof(candidate), "%s/%s", dir[0] ? dir : ".", names[i]); + if (access(candidate, X_OK) == 0) { + char *res = web_xstrdup(candidate); + free(path); + return res; + } + } + } + free(path); + } + return web_xstrdup("google-chrome"); +} + +#ifdef __APPLE__ +static const char *web_macos_chrome_app_name(void) { + if (getenv("DS4_CHROME")) return NULL; + if (access("/Applications/Google Chrome.app", F_OK) == 0) + return "Google Chrome"; + if (access("/Applications/Chromium.app", F_OK) == 0) + return "Chromium"; + return NULL; +} +#endif + +static bool web_spawn_chrome(ds4_web *web, char *err, size_t err_len) { + if (!web_mkdir_p(web->profile_dir)) { + web_set_err(err, err_len, "failed to create Chrome profile dir %s: %s", + web->profile_dir, strerror(errno)); + return false; + } + char *exe = web_chrome_executable(); +#ifdef __APPLE__ + const char *mac_app_name = web_macos_chrome_app_name(); + bool launched_via_open = mac_app_name != NULL && access("/usr/bin/open", X_OK) == 0; +#else + bool launched_via_open = false; +#endif + char port_arg[64], profile_arg[PATH_MAX + 64]; + snprintf(port_arg, sizeof(port_arg), "--remote-debugging-port=%d", web->port); + snprintf(profile_arg, sizeof(profile_arg), "--user-data-dir=%s", web->profile_dir); + pid_t pid = fork(); + if (pid < 0) { + web_set_err(err, err_len, "failed to fork Chrome: %s", strerror(errno)); + free(exe); + return false; + } + if (pid == 0) { + int nullfd = open("/dev/null", O_RDWR); + if (nullfd >= 0) { + dup2(nullfd, STDOUT_FILENO); + dup2(nullfd, STDERR_FILENO); + if (nullfd > 2) close(nullfd); + } +#ifdef __APPLE__ + if (launched_via_open) { + execlp("/usr/bin/open", "open", "-g", "-na", mac_app_name, + "--args", port_arg, "--remote-allow-origins=*", + profile_arg, "--no-first-run", "--no-default-browser-check", + "--disable-sync", "--use-mock-keychain", "--password-store=basic", + "--mute-audio", "about:blank", (char *)NULL); + } else { + execlp(exe, exe, port_arg, "--remote-allow-origins=*", + profile_arg, "--no-first-run", "--no-default-browser-check", + "--disable-sync", "--use-mock-keychain", "--password-store=basic", + "--mute-audio", "about:blank", (char *)NULL); + } +#else + if (geteuid() == 0) { + execlp(exe, exe, port_arg, "--remote-allow-origins=*", + profile_arg, "--no-first-run", "--no-default-browser-check", + "--disable-sync", "--password-store=basic", "--no-sandbox", + "--mute-audio", "about:blank", (char *)NULL); + } else { + execlp(exe, exe, port_arg, "--remote-allow-origins=*", + profile_arg, "--no-first-run", "--no-default-browser-check", + "--disable-sync", "--password-store=basic", + "--mute-audio", "about:blank", (char *)NULL); + } +#endif + _exit(127); + } + free(exe); + web->chrome_pid = pid; + for (int i = 0; i < 80; i++) { + if (web_set_cancel_err(web, err, err_len)) return false; + if (web_cdp_alive(web)) { + web_log(web, "Chrome browser session is ready"); + return true; + } + int status = 0; + pid_t rc = waitpid(pid, &status, WNOHANG); + if (rc == pid) { + web->chrome_pid = 0; + if (launched_via_open) continue; + web_set_err(err, err_len, "Chrome exited before CDP became ready"); + return false; + } + if (!web_sleep_ms(web, 250)) { + web_set_err(err, err_len, "interrupted"); + return false; + } + } + web_set_err(err, err_len, "Chrome did not expose CDP on port %d", web->port); + return false; +} + +static bool web_ensure_browser(ds4_web *web, char *err, size_t err_len) { + if (web_cdp_alive(web)) return true; + if (web->chrome_pid > 0) { + int status = 0; + waitpid(web->chrome_pid, &status, WNOHANG); + web->chrome_pid = 0; + } + if (!web->browser_allowed) { + if (!web->confirm) { + web_set_err(err, err_len, + "starting a visible Chrome browser requires interactive approval"); + return false; + } + if (!web->confirm(web->confirm_privdata, + "The web tool wants to start a visible Chrome browser. Allow? (y/n) ", + err, err_len)) + { + if (err && !err[0]) web_set_err(err, err_len, "user denied Chrome browser start"); + return false; + } + web->browser_allowed = true; + } + return web_spawn_chrome(web, err, err_len); +} + +static void web_tab_free(web_tab *tab) { + if (!tab) return; + free(tab->id); + free(tab->ws_url); + tab->id = NULL; + tab->ws_url = NULL; +} + +static char *web_browser_ws_url(ds4_web *web, char *err, size_t err_len) { + char *body = web_http_request("GET", web->port, "/json/version", err, err_len); + if (!body) return NULL; + char *ws = web_json_get_string(body, "webSocketDebuggerUrl"); + free(body); + if (!ws) web_set_err(err, err_len, "Chrome did not return a browser WebSocket URL"); + return ws; +} + +static bool web_open_tab(ds4_web *web, const char *url, web_tab *tab, + char *err, size_t err_len) { + memset(tab, 0, sizeof(*tab)); + + char *browser_url = web_browser_ws_url(web, err, err_len); + if (!browser_url) return false; + cdp_ws browser = {.fd = -1, .web = web}; + if (web_ws_connect(browser_url, &browser, err, err_len) != 0) { + free(browser_url); + return false; + } + free(browser_url); + + char *qurl = web_json_quote(url); + web_buf params = {0}; + web_buf_puts(¶ms, "{\"url\":"); + web_buf_puts(¶ms, qurl); + web_buf_puts(¶ms, ",\"background\":true,\"newWindow\":false}"); + free(qurl); + char *params_s = web_buf_take(¶ms); + char *resp = web_cdp_call(&browser, "Target.createTarget", + params_s, err, err_len); + free(params_s); + web_ws_close(&browser); + if (!resp) return false; + + tab->id = web_json_get_string(resp, "targetId"); + free(resp); + if (!tab->id) { + web_tab_free(tab); + web_set_err(err, err_len, "Chrome did not return a page target id"); + return false; + } + + char ws_url[PATH_MAX + 128]; + snprintf(ws_url, sizeof(ws_url), "ws://127.0.0.1:%d/devtools/page/%s", + web->port, tab->id); + tab->ws_url = web_xstrdup(ws_url); + return true; +} + +static void web_close_tab(ds4_web *web, const web_tab *tab) { + if (!web || !tab || !tab->id || !tab->id[0]) return; + char *enc = web_url_encode(tab->id); + web_buf path = {0}; + web_buf_puts(&path, "/json/close/"); + web_buf_puts(&path, enc); + free(enc); + + char err[160] = {0}; + char *path_s = web_buf_take(&path); + char *body = web_http_request("GET", web->port, path_s, err, sizeof(err)); + free(path_s); + if (body) { + free(body); + } else if (err[0]) { + web_log(web, err); + } +} + +static const char *web_click_google_consent_js = +"(() => {" +"const clean=s=>(s||'').replace(/\\s+/g,' ').trim();" +"const pats=[/accept all/i,/i agree/i,/agree/i,/accetta tutto/i,/tout accepter/i,/aceptar todo/i,/alle akzeptieren/i];" +"const els=[...document.querySelectorAll('button,[role=button],input[type=submit],a')];" +"for (const el of els){const t=clean(el.innerText||el.value||el.textContent);" +"if(!t)continue; if(pats.some(p=>p.test(t))){el.click(); return 'clicked '+t;}}" +"return '';" +"})()"; + +static const char *web_extract_search_js = +"(() => {" +"const clean=s=>(s||'').replace(/\\s+/g,' ').trim();" +"const esc=s=>clean(s).replace(/\\\\/g,'\\\\\\\\').replace(/\\[/g,'\\\\[').replace(/\\]/g,'\\\\]').replace(/\\n/g,' ');" +"const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};" +"const bad=h=>(/(^|\\.)google\\./.test(h)||/(^|\\.)gstatic\\./.test(h)||/(^|\\.)googleusercontent\\./.test(h));" +"const lines=['# Google search results','',`URL: ${location.href}`,'','## Visible links'];" +"const seen=new Set();" +"for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;let href=a.href||'';" +"try{const u=new URL(href);if(u.pathname==='/url'&&u.searchParams.get('q'))href=u.searchParams.get('q');}catch{}" +"let u;try{u=new URL(href);}catch{continue;}if(!/^https?:$/.test(u.protocol))continue;if(bad(u.hostname))continue;" +"const text=esc(a.innerText||a.textContent);if(text.length<3)continue;if(seen.has(u.href))continue;seen.add(u.href);" +"lines.push(`- [${text.slice(0,180)}](${u.href})`);if(seen.size>=20)break;}" +"lines.push('','## Text snapshot',clean(document.body.innerText).slice(0,1200));" +"return lines.join('\\n');" +"})()"; + +static const char *web_extract_page_js = +"(() => {" +"const clean=s=>(s||'').replace(/\\s+/g,' ').trim();" +"const esc=s=>clean(s).replace(/\\\\/g,'\\\\\\\\').replace(/\\[/g,'\\\\[').replace(/\\]/g,'\\\\]').replace(/\\n/g,' ');" +"const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};" +"const inline=n=>{if(!n)return'';if(n.nodeType===3)return n.nodeValue;if(n.nodeType!==1)return'';const el=n;" +"if(el.tagName==='SCRIPT'||el.tagName==='STYLE'||el.tagName==='NOSCRIPT')return'';" +"if(el.tagName==='A'){const t=esc(el.innerText||el.textContent);const h=el.href||'';return t&&h?`[${t}](${h})`:t;}" +"if(el.tagName==='CODE')return '`'+clean(el.innerText||el.textContent).replace(/`/g,'\\\\`')+'`';" +"return [...el.childNodes].map(inline).join('');};" +"const lines=[`# ${clean(document.title)||location.href}`,'',`URL: ${location.href}`,'','## Content'];" +"const blocks=[...document.body.querySelectorAll('h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id=\"content-text\"],[class*=\"comment-body\"],[class*=\"comment-content\"],[data-testid*=\"comment-text\"]')];" +"const seen=new Set();" +"for(const el of blocks){if(!visible(el))continue;let s='';const tag=el.tagName;" +"if(/^H[1-6]$/.test(tag)){s='#'.repeat(Number(tag[1]))+' '+inline(el);}" +"else if(tag==='LI'){s='- '+inline(el);}" +"else if(tag==='PRE'){s='```\\n'+(el.innerText||el.textContent||'').trimEnd()+'\\n```';}" +"else if(tag==='BLOCKQUOTE'){s='> '+clean(el.innerText||el.textContent);}" +"else{s=inline(el);}s=s.trim();if(!s||seen.has(s))continue;seen.add(s);lines.push('',s);" +"if(lines.join('\\n').length>900000){lines.push('','[Content truncated by browser extractor.]');break;}}" +"lines.push('','## Visible links');let n=0;const linkSeen=new Set();" +"for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;const t=esc(a.innerText||a.textContent);if(t.length<3)continue;" +"let u;try{u=new URL(a.href);}catch{continue;}if(!/^https?:$/.test(u.protocol)||linkSeen.has(u.href))continue;linkSeen.add(u.href);" +"lines.push(`- [${t.slice(0,160)}](${u.href})`);if(++n>=80)break;}" +"return lines.join('\\n');" +"})()"; + +static char *web_run_page_js(ds4_web *web, const char *url, const char *js, + bool dynamic_scroll, + char *err, size_t err_len) { + if (!web_ensure_browser(web, err, err_len)) return NULL; + web_tab tab = {0}; + if (!web_open_tab(web, "about:blank", &tab, err, err_len)) return NULL; + cdp_ws ws = {.fd = -1, .web = web}; + if (web_ws_connect(tab.ws_url, &ws, err, err_len) != 0) { + web_close_tab(web, &tab); + web_tab_free(&tab); + return NULL; + } + if (!web_cdp_prepare_page(&ws, err, err_len)) { + web_ws_close(&ws); + web_close_tab(web, &tab); + web_tab_free(&tab); + return NULL; + } + if (!web_cdp_navigate(&ws, url, err, err_len) || + !web_wait_navigated_ready(&ws, url, err, err_len)) + { + web_ws_close(&ws); + web_close_tab(web, &tab); + web_tab_free(&tab); + return NULL; + } + char *clicked = web_cdp_eval_string(&ws, web_click_google_consent_js, err, err_len); + if (clicked && clicked[0]) { + web_log(web, clicked); + if (!web_sleep_ms(web, 1500)) { + free(clicked); + web_set_err(err, err_len, "interrupted"); + web_ws_close(&ws); + web_close_tab(web, &tab); + web_tab_free(&tab); + return NULL; + } + char wait_err[160] = {0}; + if (!web_wait_navigated_ready(&ws, url, wait_err, sizeof(wait_err)) && + web_err_is_interrupted(wait_err)) + { + free(clicked); + web_set_err(err, err_len, "interrupted"); + web_ws_close(&ws); + web_close_tab(web, &tab); + web_tab_free(&tab); + return NULL; + } + } + free(clicked); + if (dynamic_scroll && !web_scroll_dynamic_page(&ws, err, err_len)) { + web_ws_close(&ws); + web_close_tab(web, &tab); + web_tab_free(&tab); + return NULL; + } + char *out = web_cdp_eval_string(&ws, js, err, err_len); + web_ws_close(&ws); + web_close_tab(web, &tab); + web_tab_free(&tab); + return out; +} + +ds4_web *ds4_web_create(const ds4_web_config *cfg) { + ds4_web *web = web_xmalloc(sizeof(*web)); + memset(web, 0, sizeof(*web)); + const char *home = cfg && cfg->home_dir && cfg->home_dir[0] ? + cfg->home_dir : getenv("HOME"); + if (!home || !home[0]) home = "."; + snprintf(web->home, sizeof(web->home), "%s", home); + snprintf(web->profile_dir, sizeof(web->profile_dir), "%s/.ds4/browser", home); + web->port = cfg && cfg->port > 0 ? cfg->port : DS4_WEB_DEFAULT_PORT; + web->chrome_pid = 0; + web->next_cdp_id = 1; + if (cfg) { + web->confirm = cfg->confirm; + web->confirm_privdata = cfg->confirm_privdata; + web->log = cfg->log; + web->log_privdata = cfg->log_privdata; + web->cancel = cfg->cancel; + web->cancel_privdata = cfg->cancel_privdata; + } + return web; +} + +void ds4_web_free(ds4_web *web) { + if (!web) return; + /* Do not kill Chrome. The browser profile is user-visible state and keeping + * it alive makes repeated web tool calls cheaper and less suspicious. */ + free(web); +} + +char *ds4_web_google_search(ds4_web *web, const char *query, + char *err, size_t err_len) { + if (!web) { + web_set_err(err, err_len, "web subsystem is not initialized"); + return NULL; + } + if (!query || !query[0]) { + web_set_err(err, err_len, "google_search requires query"); + return NULL; + } + char *q = web_url_encode(query); + web_buf url = {0}; + web_buf_puts(&url, "https://www.google.com/search?q="); + web_buf_puts(&url, q); + free(q); + char *url_s = web_buf_take(&url); + char *out = web_run_page_js(web, url_s, web_extract_search_js, false, err, err_len); + free(url_s); + return out; +} + +char *ds4_web_visit_page(ds4_web *web, const char *url, + char *err, size_t err_len) { + if (!web) { + web_set_err(err, err_len, "web subsystem is not initialized"); + return NULL; + } + if (!url || !url[0]) { + web_set_err(err, err_len, "visit_page requires url"); + return NULL; + } + return web_run_page_js(web, url, web_extract_page_js, true, err, err_len); +} diff --git a/native/web/ds4_web.h b/native/web/ds4_web.h new file mode 100644 index 0000000..338f1c6 --- /dev/null +++ b/native/web/ds4_web.h @@ -0,0 +1,33 @@ +#ifndef DS4_WEB_H +#define DS4_WEB_H + +#include +#include + +typedef int (*ds4_web_confirm_fn)(void *privdata, const char *message, + char *err, size_t err_len); +typedef void (*ds4_web_log_fn)(void *privdata, const char *message); +typedef bool (*ds4_web_cancel_fn)(void *privdata); + +typedef struct { + const char *home_dir; + int port; + ds4_web_confirm_fn confirm; + void *confirm_privdata; + ds4_web_log_fn log; + void *log_privdata; + ds4_web_cancel_fn cancel; + void *cancel_privdata; +} ds4_web_config; + +typedef struct ds4_web ds4_web; + +ds4_web *ds4_web_create(const ds4_web_config *cfg); +void ds4_web_free(ds4_web *web); + +char *ds4_web_google_search(ds4_web *web, const char *query, + char *err, size_t err_len); +char *ds4_web_visit_page(ds4_web *web, const char *url, + char *err, size_t err_len); + +#endif diff --git a/src/agent.rs b/src/agent.rs new file mode 100644 index 0000000..89cba3a --- /dev/null +++ b/src/agent.rs @@ -0,0 +1,1309 @@ +use crate::model::ModelChoice; +use rfd::{MessageButtons, MessageDialog, MessageDialogResult, MessageLevel}; +use serde_json::{Map, Value}; +use std::collections::{HashMap, HashSet}; +use std::ffi::{CStr, CString, c_char, c_int, c_void}; +use std::fs::{self, File}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::mpsc::{self, Receiver, TryRecvError}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const MAX_FILE_BYTES: u64 = 16 * 1024 * 1024; + +#[repr(C)] +struct WebConfig { + home_dir: *const c_char, + port: c_int, + confirm: Option c_int>, + confirm_data: *mut c_void, + log: Option, + log_data: *mut c_void, + cancel: Option bool>, + cancel_data: *mut c_void, +} + +unsafe extern "C" { + fn ds4_web_create(config: *const WebConfig) -> *mut c_void; + fn ds4_web_free(web: *mut c_void); + fn ds4_web_google_search( + web: *mut c_void, + query: *const c_char, + error: *mut c_char, + error_len: usize, + ) -> *mut c_char; + fn ds4_web_visit_page( + web: *mut c_void, + url: *const c_char, + error: *mut c_char, + error_len: usize, + ) -> *mut c_char; + fn free(pointer: *mut c_void); +} + +struct WebCallbacks { + cancel: AtomicPtr, +} + +struct Browser { + web: *mut c_void, + callbacks: Box, +} + +// The C browser is used by one agent-tools worker at a time under Tools' mutex. +unsafe impl Send for Browser {} + +impl Browser { + fn new() -> Result { + let mut callbacks = Box::new(WebCallbacks { + cancel: AtomicPtr::new(std::ptr::null_mut()), + }); + let home = CString::new( + std::env::var_os("HOME") + .unwrap_or_else(|| ".".into()) + .to_string_lossy() + .as_bytes(), + ) + .map_err(|_| "The home directory contains a NUL byte.".to_owned())?; + let data = (&mut *callbacks) as *mut WebCallbacks as *mut c_void; + let config = WebConfig { + home_dir: home.as_ptr(), + port: 9333, + confirm: Some(web_confirm), + confirm_data: data, + log: None, + log_data: std::ptr::null_mut(), + cancel: Some(web_cancel), + cancel_data: data, + }; + let web = unsafe { ds4_web_create(&config) }; + if web.is_null() { + Err("Could not initialize browser tools.".into()) + } else { + Ok(Self { web, callbacks }) + } + } + + fn google_search(&self, query: &str, cancel: &AtomicBool) -> Result { + self.call(query, cancel, ds4_web_google_search) + } + + fn visit_page(&self, url: &str, cancel: &AtomicBool) -> Result { + self.call(url, cancel, ds4_web_visit_page) + } + + fn call( + &self, + value: &str, + cancel: &AtomicBool, + operation: unsafe extern "C" fn( + *mut c_void, + *const c_char, + *mut c_char, + usize, + ) -> *mut c_char, + ) -> Result { + let value = CString::new(value).map_err(|_| "web input contains a NUL byte".to_owned())?; + let mut error = [0 as c_char; 256]; + self.callbacks.cancel.store( + cancel as *const AtomicBool as *mut AtomicBool, + Ordering::Relaxed, + ); + let result = + unsafe { operation(self.web, value.as_ptr(), error.as_mut_ptr(), error.len()) }; + self.callbacks + .cancel + .store(std::ptr::null_mut(), Ordering::Relaxed); + if result.is_null() { + let error = unsafe { CStr::from_ptr(error.as_ptr()) } + .to_string_lossy() + .into_owned(); + Err(if error.is_empty() { + "browser tool failed".into() + } else { + error + }) + } else { + let output = unsafe { CStr::from_ptr(result) } + .to_string_lossy() + .into_owned(); + unsafe { free(result.cast()) }; + Ok(output) + } + } +} + +impl Drop for Browser { + fn drop(&mut self) { + unsafe { ds4_web_free(self.web) }; + } +} + +unsafe extern "C" fn web_confirm( + _data: *mut c_void, + message: *const c_char, + error: *mut c_char, + error_len: usize, +) -> c_int { + let message = if message.is_null() { + "The web tool wants to start a visible Chrome browser. Allow?".into() + } else { + unsafe { CStr::from_ptr(message) }.to_string_lossy() + }; + let allowed = MessageDialog::new() + .set_level(MessageLevel::Warning) + .set_title("Allow browser tools?") + .set_description(message) + .set_buttons(MessageButtons::YesNo) + .show() + == MessageDialogResult::Yes; + if !allowed { + write_c_error(error, error_len, "user denied Chrome browser start"); + } + c_int::from(allowed) +} + +unsafe extern "C" fn web_cancel(data: *mut c_void) -> bool { + if data.is_null() { + return false; + } + let callbacks = unsafe { &*(data.cast::()) }; + let cancel = callbacks.cancel.load(Ordering::Relaxed); + !cancel.is_null() && unsafe { (*cancel).load(Ordering::Relaxed) } +} + +fn write_c_error(output: *mut c_char, length: usize, message: &str) { + if output.is_null() || length == 0 { + return; + } + let bytes = message.as_bytes(); + let count = bytes.len().min(length - 1); + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr().cast(), output, count); + *output.add(count) = 0; + } +} + +const TOOL_SCHEMAS: &str = r#"{"type":"function","function":{"name":"google_search","description":"Search Google in a visible browser and return compact Markdown links.","parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}} +{"type":"function","function":{"name":"visit_page","description":"Open a URL in a visible browser and return rendered page text.","parameters":{"type":"object","properties":{"url":{"type":"string"}},"required":["url"]}}} +{"type":"function","function":{"name":"bash","description":"Run a shell command.","parameters":{"type":"object","properties":{"command":{"type":"string"},"timeout_sec":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["command"]}}} +{"type":"function","function":{"name":"bash_status","description":"Report current status and new output for a bash job.","parameters":{"type":"object","properties":{"job":{"type":"number"},"pid":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["job"]}}} +{"type":"function","function":{"name":"bash_stop","description":"Terminate a running bash job and report its final output.","parameters":{"type":"object","properties":{"job":{"type":"number"},"pid":{"type":"number"},"refresh_sec":{"type":"number"}},"required":["job"]}}} +{"type":"function","function":{"name":"read","description":"Read a text file or a range of lines.","parameters":{"type":"object","properties":{"path":{"type":"string"},"start_line":{"type":"number"},"max_lines":{"type":"number"},"whole":{"type":"boolean"},"raw":{"type":"boolean"}},"required":["path"]}}} +{"type":"function","function":{"name":"more","description":"Continue the previous read-like output.","parameters":{"type":"object","properties":{"count":{"type":"number"}}}}} +{"type":"function","function":{"name":"write","description":"Create or overwrite a text file.","parameters":{"type":"object","properties":{"path":{"type":"string"},"content":{"type":"string"}},"required":["path","content"]}}} +{"type":"function","function":{"name":"edit","description":"Replace exactly one old text match; old may contain [upto] between unique head and tail anchors.","parameters":{"type":"object","properties":{"path":{"type":"string"},"old":{"type":"string"},"new":{"type":"string"}},"required":["path","old","new"]}}} +{"type":"function","function":{"name":"search","description":"Search files and return compact edit-friendly matches.","parameters":{"type":"object","properties":{"query":{"type":"string"},"path":{"type":"string"},"mode":{"type":"string"},"glob":{"type":"string"},"context":{"type":"number"},"max_results":{"type":"number"},"case_sensitive":{"type":"boolean"}},"required":["query"]}}} +{"type":"function","function":{"name":"list","description":"List one directory compactly.","parameters":{"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}}}"#; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ToolCall { + pub(crate) name: String, + pub(crate) arguments: Map, +} + +pub(crate) struct ActiveTools { + pub(crate) results: Receiver, + pub(crate) cancel: Arc, +} + +struct BashJob { + id: u32, + command: String, + child: Child, + output: PathBuf, + started: Instant, + timeout: Duration, + observed: usize, +} + +pub(crate) struct Tools { + root: PathBuf, + context_tokens: i32, + more: Option<(PathBuf, usize, bool)>, + jobs: HashMap, + temporary_files: HashSet, + next_job: u32, + browser: Browser, +} + +impl Tools { + pub(crate) fn new(root: &Path, context_tokens: i32) -> Result { + Ok(Self { + root: root + .canonicalize() + .map_err(|error| format!("Could not open the project directory: {error}"))?, + context_tokens, + more: None, + jobs: HashMap::new(), + temporary_files: HashSet::new(), + next_job: 1, + browser: Browser::new()?, + }) + } + + fn execute(&mut self, call: &ToolCall, cancel: &AtomicBool) -> String { + let result = match call.name.as_str() { + "read" => self.read(call), + "more" => self.more(call), + "write" => self.write(call), + "edit" => self.edit(call), + "search" => self.search(call), + "list" => self.list(call), + "bash" => self.bash(call, cancel), + "bash_status" => self.bash_observe(call, false, cancel), + "bash_stop" => self.bash_observe(call, true, cancel), + "google_search" => self.google_search(call, cancel), + "visit_page" => self.visit_page(call, cancel), + name => Err(format!("unknown tool: {name}")), + }; + match result { + Ok(result) if result.len() <= self.result_limit() => result, + Ok(result) => format!( + "Tool error: {} result is too large for this context ({} bytes). Retry with a smaller read/search/bash output.\n", + call.name, + result.len() + ), + Err(error) => format!("Tool error: {error}\n"), + } + } + + fn result_limit(&self) -> usize { + (self.context_tokens.max(4096) as usize * 2).min(512 * 1024) + } + + fn existing_path(&self, value: &str) -> Result { + let path = if Path::new(value).is_absolute() { + PathBuf::from(value) + } else { + self.root.join(value) + }; + let path = path + .canonicalize() + .map_err(|error| format!("open {value}: {error}"))?; + if self.temporary_files.contains(&path) { + Ok(path) + } else { + self.inside_project(path, value) + } + } + + fn writable_path(&self, value: &str) -> Result { + let path = if Path::new(value).is_absolute() { + PathBuf::from(value) + } else { + self.root.join(value) + }; + if path.exists() { + return self.existing_path(value); + } + let parent = path + .parent() + .ok_or_else(|| format!("invalid path: {value}"))? + .canonicalize() + .map_err(|error| format!("open parent of {value}: {error}"))?; + self.inside_project(parent, value)?; + Ok(path) + } + + fn inside_project(&self, path: PathBuf, original: &str) -> Result { + if path.starts_with(&self.root) { + Ok(path) + } else { + Err(format!("path is outside the project: {original}")) + } + } + + fn default_lines(&self) -> usize { + match self.context_tokens { + ..=8192 => 120, + 8193..=16384 => 240, + _ => 500, + } + } + + fn read(&mut self, call: &ToolCall) -> Result { + let path = required_string(call, "path")?; + let start = integer(call, "start_line", 1, 1, usize::MAX); + let count = integer(call, "max_lines", self.default_lines(), 1, usize::MAX); + self.read_range( + &self.existing_path(path)?, + start, + count, + boolean(call, "whole", false), + boolean(call, "raw", false), + ) + } + + fn more(&mut self, call: &ToolCall) -> Result { + let (path, start, raw) = self + .more + .clone() + .ok_or_else(|| "no previous output to continue".to_owned())?; + let count = integer(call, "count", self.default_lines(), 1, usize::MAX); + self.read_range(&path, start, count, false, raw) + } + + fn read_range( + &mut self, + path: &Path, + start: usize, + count: usize, + whole: bool, + raw: bool, + ) -> Result { + let metadata = path.metadata().map_err(|error| error.to_string())?; + if metadata.len() > MAX_FILE_BYTES { + return Err(format!( + "file too large: {} exceeds {MAX_FILE_BYTES} bytes", + path.display() + )); + } + let data = fs::read_to_string(path) + .map_err(|error| format!("read {}: {error}", path.display()))?; + let lines = data.lines().collect::>(); + let first = start.saturating_sub(1).min(lines.len()); + let last = if whole { + lines.len() + } else { + first.saturating_add(count).min(lines.len()) + }; + self.more = (last < lines.len()).then(|| (path.to_owned(), last + 1, raw)); + let mut output = String::new(); + if !raw { + if last < lines.len() { + output.push_str(&format!( + "{}: lines {}-{} of {}; continue_offset={}; call more with count={} to read the next chunk\n", + path.display(), + if lines.is_empty() { 0 } else { first + 1 }, + last, + lines.len(), + last + 1, + count + )); + } else { + output.push_str(&format!( + "{}: lines {}-{} of {}\n", + path.display(), + if lines.is_empty() { 0 } else { first + 1 }, + last, + lines.len() + )); + } + } + for (index, line) in lines[first..last].iter().enumerate() { + if raw { + output.push_str(line); + } else { + output.push_str(&format!("{} {line}", first + index + 1)); + } + output.push('\n'); + } + if raw && last < lines.len() { + output.push_str(&format!( + "[Read truncated at line {} of {}. continue_offset={}. Call more with count={} to read the next chunk.]\n", + last, + lines.len(), + last + 1, + count + )); + } + Ok(output) + } + + fn write(&self, call: &ToolCall) -> Result { + let display = required_string(call, "path")?; + let content = required_string(call, "content")?; + if content.len() as u64 > MAX_FILE_BYTES { + return Err(format!("content exceeds {MAX_FILE_BYTES} bytes")); + } + let path = self.writable_path(display)?; + fs::write(&path, content).map_err(|error| format!("write {display}: {error}"))?; + Ok(format!("Wrote {} bytes to {display}\n", content.len())) + } + + fn edit(&self, call: &ToolCall) -> Result { + let display = required_string(call, "path")?; + let old = required_string(call, "old")?; + let new = required_string(call, "new")?; + if old.is_empty() { + return Err("edit requires non-empty old text".into()); + } + let path = self.existing_path(display)?; + if path.metadata().map_err(|error| error.to_string())?.len() > MAX_FILE_BYTES { + return Err(format!( + "file too large: {display} exceeds {MAX_FILE_BYTES} bytes" + )); + } + let data = fs::read_to_string(&path).map_err(|error| format!("read {display}: {error}"))?; + let (start, end, anchored) = edit_span(&data, old)?; + if data.len() - (end - start) + new.len() > MAX_FILE_BYTES as usize { + return Err(format!("edited file would exceed {MAX_FILE_BYTES} bytes")); + } + let mut output = String::with_capacity(data.len() - (end - start) + new.len()); + output.push_str(&data[..start]); + output.push_str(new); + output.push_str(&data[end..]); + fs::write(&path, output).map_err(|error| format!("write {display}: {error}"))?; + Ok(format!( + "Edited {display} using {} replacement\n", + if anchored { + "anchored old/new" + } else { + "old/new" + } + )) + } + + fn list(&self, call: &ToolCall) -> Result { + let display = string(call, "path").unwrap_or("."); + let path = self.existing_path(display)?; + if !path.is_dir() { + return Err(format!("not a directory: {display}")); + } + let mut entries = fs::read_dir(&path) + .map_err(|error| format!("list {display}: {error}"))? + .filter_map(Result::ok) + .collect::>(); + entries.sort_by_key(|entry| entry.file_name()); + let mut output = format!("{display}:\n"); + for entry in entries.iter().take(300) { + let metadata = fs::symlink_metadata(entry.path()).map_err(|error| error.to_string())?; + let kind = if metadata.file_type().is_symlink() { + 'l' + } else if metadata.is_dir() { + 'd' + } else { + '-' + }; + let suffix = if metadata.is_dir() { "/" } else { "" }; + output.push_str(&format!( + "{kind} {:>10} {}{suffix}\n", + metadata.len(), + entry.file_name().to_string_lossy() + )); + } + if entries.len() > 300 { + output.push_str("... more entries omitted ...\n"); + } + Ok(output) + } + + fn search(&self, call: &ToolCall) -> Result { + let query = required_string(call, "query")?; + let display = string(call, "path").unwrap_or("."); + let path = self.existing_path(display)?; + let context = integer(call, "context", 0, 0, 5); + let limit = integer(call, "max_results", 50, 1, 500); + let options = SearchOptions { + query, + glob: string(call, "glob"), + regex: string(call, "mode") == Some("regex"), + case_sensitive: boolean(call, "case_sensitive", true), + context, + limit, + }; + search_path(&self.root, &path, &options) + } + + fn bash(&mut self, call: &ToolCall, cancel: &AtomicBool) -> Result { + let command = required_string(call, "command")?.to_owned(); + let timeout = integer(call, "timeout_sec", 3600, 1, 24 * 3600); + let refresh = integer(call, "refresh_sec", 60, 1, 3600); + let id = self.next_job; + self.next_job = self.next_job.saturating_add(1).max(1); + let output = std::env::temp_dir().join(format!( + "ds4_agent_output_{}_{}_{}", + std::process::id(), + id, + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + )); + let stdout = File::create(&output).map_err(|error| error.to_string())?; + let output = output.canonicalize().map_err(|error| error.to_string())?; + self.temporary_files.insert(output.clone()); + let stderr = stdout.try_clone().map_err(|error| error.to_string())?; + let child = Command::new("/bin/sh") + .arg("-c") + .arg(&command) + .current_dir(&self.root) + .stdin(Stdio::null()) + .stdout(stdout) + .stderr(stderr) + .process_group(0) + .spawn() + .map_err(|error| format!("bash failed to start: {error}"))?; + self.jobs.insert( + id, + BashJob { + id, + command, + child, + output, + started: Instant::now(), + timeout: Duration::from_secs(timeout as u64), + observed: 0, + }, + ); + self.wait_job(id, refresh, cancel, true) + } + + fn bash_observe( + &mut self, + call: &ToolCall, + stop: bool, + cancel: &AtomicBool, + ) -> Result { + let requested_id = integer(call, "job", 0, 0, u32::MAX as usize) as u32; + let requested_pid = integer(call, "pid", 0, 0, u32::MAX as usize) as u32; + let id = if self.jobs.contains_key(&requested_id) { + requested_id + } else { + self.jobs + .iter() + .find_map(|(id, job)| (job.child.id() == requested_pid).then_some(*id)) + .unwrap_or(requested_id) + }; + let refresh = integer(call, "refresh_sec", 60, 1, 3600); + if !self.jobs.contains_key(&id) { + return Err(format!( + "bash job not found: job={requested_id} pid={requested_pid}" + )); + } + if stop { + stop_job(self.jobs.get_mut(&id).unwrap()); + } + self.wait_job(id, if stop { 1 } else { refresh }, cancel, stop) + } + + fn wait_job( + &mut self, + id: u32, + refresh: usize, + cancel: &AtomicBool, + remove_done: bool, + ) -> Result { + let deadline = Instant::now() + Duration::from_secs(refresh as u64); + loop { + let job = self.jobs.get_mut(&id).unwrap(); + let done = job.child.try_wait().map_err(|error| error.to_string())?; + if done.is_some() || Instant::now() >= deadline { + break; + } + if job.started.elapsed() >= job.timeout { + stop_job(job); + break; + } + if cancel.load(Ordering::Relaxed) { + stop_job(job); + return Err("interrupted".into()); + } + thread::sleep(Duration::from_millis(100)); + } + let job = self.jobs.get_mut(&id).unwrap(); + let status = job.child.try_wait().map_err(|error| error.to_string())?; + let bytes = fs::read(&job.output).unwrap_or_default(); + let first_observation = job.observed == 0; + let new = &bytes[job.observed.min(bytes.len())..]; + job.observed = bytes.len(); + let shown = if first_observation && new.len() > 8 * 1024 { + &new[..8 * 1024] + } else if new.len() > 32 * 1024 { + &new[new.len() - 32 * 1024..] + } else { + new + }; + let mut result = format!( + "bash job={} pid={} status={} command={}\noutput_path={}\n", + job.id, + job.child.id(), + status.map_or("running".into(), |status| status.to_string()), + job.command, + job.output.display() + ); + result.push_str(&String::from_utf8_lossy(shown)); + if shown.len() < new.len() { + if status.is_some() { + let tail = &new[new.len().saturating_sub(32 * 1024).max(shown.len())..]; + result.push_str("\n... middle output omitted ...\n"); + result.push_str(&String::from_utf8_lossy(tail)); + } else { + result.push_str("\n... output truncated; use bash_status for new output ...\n"); + } + } + if !result.ends_with('\n') { + result.push('\n'); + } + if remove_done && status.is_some() { + let job = self.jobs.remove(&id).unwrap(); + self.temporary_files.remove(&job.output); + let _ = fs::remove_file(job.output); + } + Ok(result) + } + + fn google_search(&mut self, call: &ToolCall, cancel: &AtomicBool) -> Result { + let query = required_string(call, "query")?; + self.browser.google_search(query, cancel) + } + + fn visit_page(&mut self, call: &ToolCall, cancel: &AtomicBool) -> Result { + let url = required_string(call, "url")?; + if !matches!( + url.split_once(':').map(|part| part.0), + Some("http" | "https") + ) { + return Err("visit_page requires an HTTP or HTTPS URL".into()); + } + let markdown = self.browser.visit_page(url, cancel)?; + let output = std::env::temp_dir().join(format!( + "ds4_agent_web_{}_{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + )); + fs::write(&output, &markdown).map_err(|error| error.to_string())?; + let output = output.canonicalize().map_err(|error| error.to_string())?; + self.temporary_files.insert(output.clone()); + let head = markdown + .lines() + .take(100) + .collect::>() + .join("\n") + .chars() + .take(8 * 1024) + .collect::(); + Ok(format!( + "visit_page url={url}\noutput_path={} ({} bytes, {} lines)\n\n{head}\n\nUse read with raw=true to inspect more rendered text.\n", + output.display(), + markdown.len(), + markdown.lines().count(), + output.display() + )) + } +} + +struct SearchOptions<'a> { + query: &'a str, + glob: Option<&'a str>, + regex: bool, + case_sensitive: bool, + context: usize, + limit: usize, +} + +fn search_path(root: &Path, path: &Path, options: &SearchOptions<'_>) -> Result { + let mut files = Vec::new(); + collect_search_files(path, 0, &mut files)?; + let mut matches = 0; + let mut body = String::new(); + for file in files { + if matches >= options.limit { + break; + } + let relative = file.strip_prefix(root).unwrap_or(&file); + if options.glob.is_some_and(|glob| { + !wildcard_match(glob, &relative.to_string_lossy()) + && !wildcard_match( + glob, + &file.file_name().unwrap_or_default().to_string_lossy(), + ) + }) { + continue; + } + let remaining = options.limit - matches; + let (count, text) = if options.regex { + regex_search_file(&file, options, remaining)? + } else { + literal_search_file(&file, options, remaining)? + }; + if count > 0 { + body.push_str(&format!("{}\n{text}\n", relative.display())); + matches += count; + } + } + if matches == 0 { + Ok("No matches\n".into()) + } else { + Ok(format!( + "{matches} match{} shown\n\n{body}", + if matches == 1 { "" } else { "es" } + )) + } +} + +fn collect_search_files( + path: &Path, + depth: usize, + output: &mut Vec, +) -> Result<(), String> { + if depth > 24 { + return Ok(()); + } + let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?; + if metadata.file_type().is_symlink() { + return Ok(()); + } + if metadata.is_file() { + if metadata.len() <= MAX_FILE_BYTES { + output.push(path.to_owned()); + } + return Ok(()); + } + if !metadata.is_dir() { + return Ok(()); + } + let mut entries = fs::read_dir(path) + .map_err(|error| error.to_string())? + .filter_map(Result::ok) + .collect::>(); + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + if entry.file_name() == ".git" { + continue; + } + collect_search_files(&entry.path(), depth + 1, output)?; + } + Ok(()) +} + +fn literal_search_file( + path: &Path, + options: &SearchOptions<'_>, + limit: usize, +) -> Result<(usize, String), String> { + let bytes = fs::read(path).map_err(|error| error.to_string())?; + if bytes.contains(&0) { + return Ok((0, String::new())); + } + let data = String::from_utf8_lossy(&bytes); + let lines = data.lines().collect::>(); + let query = (!options.case_sensitive).then(|| options.query.to_lowercase()); + let found = lines + .iter() + .enumerate() + .filter_map(|(index, line)| { + let matched = query.as_ref().map_or_else( + || line.contains(options.query), + |query| line.to_lowercase().contains(query), + ); + matched.then_some(index) + }) + .take(limit) + .collect::>(); + let mut output = String::new(); + let mut last = None; + for match_index in &found { + let start = match_index.saturating_sub(options.context); + let end = match_index + .saturating_add(options.context + 1) + .min(lines.len()); + for (index, line) in lines.iter().enumerate().take(end).skip(start) { + if last.is_none_or(|last| index > last) { + output.push_str(&format!(" {} {line}\n", index + 1)); + last = Some(index); + } + } + } + Ok((found.len(), output)) +} + +fn regex_search_file( + path: &Path, + options: &SearchOptions<'_>, + limit: usize, +) -> Result<(usize, String), String> { + let mut command = Command::new("/usr/bin/grep"); + command.arg("-n").arg("-I").arg("-E"); + if !options.case_sensitive { + command.arg("-i"); + } + if options.context > 0 { + command.arg("-C").arg(options.context.to_string()); + } + command.arg("-m").arg(limit.to_string()); + command.arg("--").arg(options.query).arg(path); + let output = command.output().map_err(|error| error.to_string())?; + if output.status.code() == Some(1) { + return Ok((0, String::new())); + } + if !output.status.success() { + return Err(format!( + "invalid regex: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let output = String::from_utf8_lossy(&output.stdout).into_owned(); + let count = output + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(number, _)| number.parse::().is_ok()) + }) + .count(); + Ok((count, output)) +} + +fn wildcard_match(pattern: &str, value: &str) -> bool { + let (pattern, value) = (pattern.as_bytes(), value.as_bytes()); + let (mut p, mut v, mut star, mut retry) = (0, 0, None, 0); + while v < value.len() { + if p < pattern.len() && (pattern[p] == b'?' || pattern[p] == value[v]) { + p += 1; + v += 1; + } else if p < pattern.len() && pattern[p] == b'*' { + star = Some(p); + p += 1; + retry = v; + } else if let Some(star) = star { + p = star + 1; + retry += 1; + v = retry; + } else { + return false; + } + } + while p < pattern.len() && pattern[p] == b'*' { + p += 1; + } + p == pattern.len() +} + +impl Drop for Tools { + fn drop(&mut self) { + for job in self.jobs.values_mut() { + stop_job(job); + let _ = fs::remove_file(&job.output); + } + for path in &self.temporary_files { + let _ = fs::remove_file(path); + } + } +} + +pub(crate) fn execute_async(tools: Arc>, calls: Vec) -> ActiveTools { + let cancel = Arc::new(AtomicBool::new(false)); + let worker_cancel = Arc::clone(&cancel); + let (sender, results) = mpsc::channel(); + thread::Builder::new() + .name("agent-tools".into()) + .spawn(move || { + let mut output = String::new(); + let mut tools = tools + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + for (index, call) in calls.iter().enumerate() { + if worker_cancel.load(Ordering::Relaxed) { + output.push_str("Tool error: interrupted\n"); + break; + } + output.push_str(&format!("Tool result {} ({}):\n", index + 1, call.name)); + output.push_str(&tools.execute(call, &worker_cancel)); + if !output.ends_with('\n') { + output.push('\n'); + } + } + let _ = sender.send(output); + }) + .expect("agent tool worker must start"); + ActiveTools { results, cancel } +} + +pub(crate) fn error_async(error: String) -> ActiveTools { + let cancel = Arc::new(AtomicBool::new(false)); + let (sender, results) = mpsc::channel(); + let _ = sender.send(format!( + "Tool error: invalid tool call: {error}\nRetry using the exact tool syntax from the system prompt.\n" + )); + ActiveTools { results, cancel } +} + +pub(crate) fn parse_tool_calls( + model: ModelChoice, + text: &str, +) -> Result<(String, Vec), String> { + let (content, calls) = if model == ModelChoice::Glm52 { + parse_glm_calls(text)? + } else { + crate::server::parse_dsml_tool_calls(text)? + }; + calls + .into_iter() + .map(|(name, arguments)| match arguments { + Value::Object(arguments) => Ok(ToolCall { name, arguments }), + _ => Err(format!("tool {name} arguments are not an object")), + }) + .collect::, _>>() + .map(|calls| (content, calls)) +} + +pub(crate) fn system_prompt(model: ModelChoice, extra: &str) -> String { + let tools = if model == ModelChoice::Glm52 { + format!( + "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n# Tools\n\nYou are provided with function signatures within XML tags:\n\n{TOOL_SCHEMAS}\n\n\nFor a function call, output exactly: function-namekeyvalue\nTool calls are not allowed inside . Use read/search for focused context, edit with exact unique old text, and [upto] only between unique head and tail anchors. Use refresh_sec for long bash jobs and poll with bash_status or stop with bash_stop. Preserve the current system configuration unless the user explicitly asks otherwise." + ) + } else { + format!( + "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n## Tools\n\nInvoke native DSML tools exactly as:\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n\n\n\nTool calls are not allowed inside . String parameters use raw text and string=\"true\"; numbers and booleans use JSON text and string=\"false\". Read defaults to a bounded chunk; use more to continue and whole=true only when needed. Use write for new files or whole-file replacement. Use edit with path first and exact unique old text; old may contain one [upto] marker between unique head and tail anchors. For long bash commands pass refresh_sec, then use bash_status or bash_stop. The first web call asks permission to start visible Chrome.\n\n### Available Tool Schemas\n\n{TOOL_SCHEMAS}\n\n# Rules\n- Always use strict DSML syntax.\n- Use read/search to get anchors before editing.\n- Preserve the current system configuration unless explicitly asked otherwise." + ) + }; + if extra.trim().is_empty() { + tools + } else { + format!("{tools}\n\n{extra}") + } +} + +pub(crate) fn try_tool_result(active: &ActiveTools) -> Result, String> { + match active.results.try_recv() { + Ok(result) => Ok(Some(result)), + Err(TryRecvError::Empty) => Ok(None), + Err(TryRecvError::Disconnected) => { + Err("The agent tool worker stopped unexpectedly.".into()) + } + } +} + +pub(crate) fn visible_content(content: &str) -> &str { + [ + "<｜DSML｜tool_calls>", + "", + "", + "", + ] + .into_iter() + .filter_map(|marker| content.find(marker)) + .min() + .map_or(content, |end| content[..end].trim_end()) +} + +pub(crate) fn tool_summaries(model: ModelChoice, content: &str) -> Vec { + parse_tool_calls(model, content) + .map(|(_, calls)| { + calls + .into_iter() + .map(|call| { + let detail = ["path", "command", "query", "url"] + .into_iter() + .find_map(|name| string(&call, name)) + .map(|value| { + let mut value = value.replace('\n', " "); + if value.chars().count() > 120 { + value = value.chars().take(119).collect::() + "…"; + } + format!(" {value}") + }) + .unwrap_or_default(); + format!("🛠 {}{detail}", call.name) + }) + .collect() + }) + .unwrap_or_default() +} + +fn parse_glm_calls(text: &str) -> Result<(String, Vec<(String, Value)>), String> { + let scan = text + .rfind("") + .map_or(text, |position| &text[position + "".len()..]); + let Some(first) = scan.find("") else { + return Ok((text.to_owned(), Vec::new())); + }; + let visible_len = text.len() - scan.len() + first; + let mut rest = &scan[first..]; + let mut calls = Vec::new(); + while rest.starts_with("") { + rest = &rest["".len()..]; + let end = rest + .find("") + .ok_or_else(|| "incomplete GLM tool call".to_owned())?; + let body = &rest[..end]; + let name_end = body.find("").unwrap_or(body.len()); + let name = body[..name_end].trim(); + if name.is_empty() { + return Err("GLM tool call without function name".into()); + } + let mut arguments = Map::new(); + let mut args = &body[name_end..]; + while !args.is_empty() { + let key = between(&mut args, "", "")?; + let value = between(&mut args, "", "")?; + arguments.insert(key.to_owned(), Value::String(value.to_owned())); + } + calls.push((name.to_owned(), Value::Object(arguments))); + rest = rest[end + "".len()..].trim_start(); + } + Ok((text[..visible_len].trim_end().to_owned(), calls)) +} + +fn between<'a>(input: &mut &'a str, open: &str, close: &str) -> Result<&'a str, String> { + let body = input + .strip_prefix(open) + .ok_or_else(|| format!("expected {open}"))?; + let end = body + .find(close) + .ok_or_else(|| format!("expected {close}"))?; + *input = &body[end + close.len()..]; + Ok(&body[..end]) +} + +fn edit_span(data: &str, old: &str) -> Result<(usize, usize, bool), String> { + let markers = old.match_indices("[upto]").collect::>(); + if markers.len() > 1 { + return Err("old text contains more than one [upto] marker".into()); + } + if let Some((marker, _)) = markers.first() { + let head = &old[..*marker]; + let tail = old[*marker + "[upto]".len()..].trim_start_matches(['\r', '\n']); + if tail.trim().is_empty() { + return Err("old text after [upto] must include a unique tail anchor".into()); + } + let start = unique_match(data, head, "old head")?; + let after_head = start + head.len(); + let tail_offset = unique_match(&data[after_head..], tail, "old tail")?; + return Ok((start, after_head + tail_offset + tail.len(), true)); + } + let start = unique_match(data, old, "old text")?; + Ok((start, start + old.len(), false)) +} + +fn unique_match(data: &str, needle: &str, label: &str) -> Result { + if needle.is_empty() { + return Err(format!("{label} is empty")); + } + let matches = data + .match_indices(needle) + .map(|(index, _)| index) + .collect::>(); + match matches.as_slice() { + [] => Err(format!("{label} was not found")), + [index] => Ok(*index), + _ => Err(format!("{label} is ambiguous ({} matches)", matches.len())), + } +} + +fn stop_job(job: &mut BashJob) { + if job.child.try_wait().ok().flatten().is_some() { + return; + } + let pid = job.child.id(); + let _ = Command::new("/bin/kill") + .arg("-TERM") + .arg(format!("-{pid}")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let deadline = Instant::now() + Duration::from_secs(1); + while Instant::now() < deadline { + if job.child.try_wait().ok().flatten().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + let _ = Command::new("/bin/kill") + .arg("-KILL") + .arg(format!("-{pid}")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let _ = job.child.wait(); +} + +fn string<'a>(call: &'a ToolCall, name: &str) -> Option<&'a str> { + call.arguments.get(name).and_then(Value::as_str) +} + +fn required_string<'a>(call: &'a ToolCall, name: &str) -> Result<&'a str, String> { + string(call, name).ok_or_else(|| format!("{} requires {name}", call.name)) +} + +fn integer(call: &ToolCall, name: &str, default: usize, min: usize, max: usize) -> usize { + call.arguments + .get(name) + .and_then(|value| { + value + .as_u64() + .or_else(|| value.as_i64().map(|value| value.max(0) as u64)) + .or_else(|| { + value + .as_f64() + .filter(|value| value.is_finite()) + .map(|value| value.max(0.0) as u64) + }) + .or_else(|| { + value + .as_str() + .and_then(|value| value.parse::().ok()) + .filter(|value| value.is_finite()) + .map(|value| value.max(0.0) as u64) + }) + }) + .and_then(|value| usize::try_from(value).ok()) + .unwrap_or(default) + .clamp(min, max) +} + +fn boolean(call: &ToolCall, name: &str, default: bool) -> bool { + call.arguments + .get(name) + .and_then(|value| { + value.as_bool().or_else(|| { + value.as_str().and_then(|value| { + if value.eq_ignore_ascii_case("true") + || value.eq_ignore_ascii_case("yes") + || value == "1" + { + Some(true) + } else if value.eq_ignore_ascii_case("false") + || value.eq_ignore_ascii_case("no") + || value == "0" + { + Some(false) + } else { + None + } + }) + }) + }) + .unwrap_or(default) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prompts_and_parsers_expose_the_reference_tool_set() { + let prompt = system_prompt(ModelChoice::DeepSeekV4Flash, "extra"); + for name in [ + "google_search", + "visit_page", + "bash", + "bash_status", + "bash_stop", + "read", + "more", + "write", + "edit", + "search", + "list", + ] { + assert!(prompt.contains(&format!("\"name\":\"{name}\""))); + } + assert!(prompt.ends_with("extra")); + + let text = "donereadpathsrc/main.rs"; + let (visible, calls) = parse_tool_calls(ModelChoice::Glm52, text).unwrap(); + assert_eq!(visible, "done"); + assert_eq!(calls[0].name, "read"); + assert_eq!(calls[0].arguments["path"], "src/main.rs"); + + let dsml = "done<｜DSML｜tool_calls><｜DSML｜invoke name=\"read\"><｜DSML｜parameter name=\"path\" string=\"true\">src/main.rs"; + let (visible, calls) = parse_tool_calls(ModelChoice::DeepSeekV4Flash, dsml).unwrap(); + assert_eq!(visible, "done"); + assert_eq!(calls[0].arguments["path"], "src/main.rs"); + assert!( + parse_tool_calls( + ModelChoice::DeepSeekV4Flash, + "<｜DSML｜tool_calls><｜DSML｜invoke name=\"read\">" + ) + .is_err() + ); + } + + #[test] + fn anchored_edits_require_unique_head_and_tail() { + let data = "start\nold one\nold two\nfinish\nother\n"; + assert_eq!( + edit_span(data, "start\n[upto]\nfinish\n").unwrap(), + (0, 29, true) + ); + assert!(edit_span("same same", "same").is_err()); + assert!(edit_span(data, "start\n[upto]\n").is_err()); + } + + #[test] + fn project_boundary_rejects_parent_paths() { + let directory = std::env::temp_dir().join(format!( + "ds4-agent-test-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&directory).unwrap(); + let tools = Tools::new(&directory, 32_768).unwrap(); + assert!(tools.writable_path("inside.txt").is_ok()); + assert!(tools.writable_path("../outside.txt").is_err()); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn local_file_and_bash_tools_execute_in_the_project() { + let directory = std::env::temp_dir().join(format!( + "ds4-agent-tools-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&directory).unwrap(); + let mut tools = Tools::new(&directory, 4096).unwrap(); + let cancel = AtomicBool::new(false); + + let write = call("write", [("path", "note.txt"), ("content", "one\ntwo\n")]); + assert!(tools.execute(&write, &cancel).starts_with("Wrote 8 bytes")); + let read = call("read", [("path", "note.txt"), ("max_lines", "1")]); + assert!(tools.execute(&read, &cancel).contains("1 one")); + assert!(tools.execute(&call("more", []), &cancel).contains("2 two")); + let edit = call( + "edit", + [("path", "note.txt"), ("old", "two"), ("new", "three")], + ); + assert!(tools.execute(&edit, &cancel).starts_with("Edited note.txt")); + let search = call("search", [("query", "three"), ("glob", "*.txt")]); + assert!(tools.execute(&search, &cancel).contains("2 three")); + assert!( + tools + .execute(&call("list", [("path", ".")]), &cancel) + .contains("note.txt") + ); + let bash = call( + "bash", + [("command", "printf shell-ok"), ("refresh_sec", "1")], + ); + assert!(tools.execute(&bash, &cancel).contains("shell-ok")); + let running = call( + "bash", + [ + ("command", "printf started; sleep 5; printf finished"), + ("refresh_sec", "1"), + ], + ); + assert!(tools.execute(&running, &cancel).contains("status=running")); + let stopped = tools.execute(&call("bash_stop", [("job", "2")]), &cancel); + assert!(!stopped.contains("status=running")); + assert_eq!( + fs::read_to_string(directory.join("note.txt")).unwrap(), + "one\nthree\n" + ); + fs::remove_dir_all(directory).unwrap(); + } + + fn call(name: &str, arguments: [(&str, &str); N]) -> ToolCall { + ToolCall { + name: name.to_owned(), + arguments: arguments + .into_iter() + .map(|(name, value)| (name.to_owned(), Value::String(value.to_owned()))) + .collect(), + } + } +} diff --git a/src/app.rs b/src/app.rs index 0a9ddbc..5580986 100644 --- a/src/app.rs +++ b/src/app.rs @@ -32,7 +32,7 @@ use std::fs; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{self, TryRecvError}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use std::thread; use std::time::{Duration, Instant}; @@ -84,6 +84,10 @@ pub(crate) struct App { #[cfg(target_os = "macos")] active_generation: Option, #[cfg(target_os = "macos")] + agent_tools: Option<(i32, Arc>)>, + #[cfg(target_os = "macos")] + active_tools: Option, + #[cfg(target_os = "macos")] active_titling: Option, #[cfg(target_os = "macos")] runtime_preferences: Arc>, @@ -249,6 +253,10 @@ impl App { #[cfg(target_os = "macos")] active_generation: None, #[cfg(target_os = "macos")] + agent_tools: None, + #[cfg(target_os = "macos")] + active_tools: None, + #[cfg(target_os = "macos")] active_titling: None, #[cfg(target_os = "macos")] runtime_preferences, @@ -326,6 +334,10 @@ impl App { #[cfg(target_os = "macos")] active_generation: None, #[cfg(target_os = "macos")] + agent_tools: None, + #[cfg(target_os = "macos")] + active_tools: None, + #[cfg(target_os = "macos")] active_titling: None, #[cfg(target_os = "macos")] runtime_preferences, @@ -643,12 +655,15 @@ impl App { self.start_generation(); return scroll_chat_to_end(); } - Message::StopGeneration => - { + Message::StopGeneration => { #[cfg(target_os = "macos")] if let Some(active) = &self.active_generation { active.cancel.store(true, Ordering::Relaxed); } + #[cfg(target_os = "macos")] + if let Some(active) = &self.active_tools { + active.cancel.store(true, Ordering::Relaxed); + } } Message::GenerationTick => { #[cfg(target_os = "macos")] @@ -1007,6 +1022,10 @@ impl Drop for App { if let Some(active) = &self.active_generation { active.cancel.store(true, Ordering::Relaxed); } + #[cfg(target_os = "macos")] + if let Some(active) = &self.active_tools { + active.cancel.store(true, Ordering::Relaxed); + } } } @@ -1130,6 +1149,7 @@ mod tests { let mut message = ChatMessage { id: 1, user: false, + tool: false, reasoning: Some(String::new()), reasoning_complete: false, reasoning_open: true, diff --git a/src/app/generation.rs b/src/app/generation.rs index 0927d13..a87e76d 100644 --- a/src/app/generation.rs +++ b/src/app/generation.rs @@ -22,6 +22,7 @@ pub(super) struct TitleRequest { pub(crate) struct ChatMessage { pub(super) id: i32, pub(super) user: bool, + pub(super) tool: bool, pub(super) reasoning: Option, pub(super) reasoning_complete: bool, pub(super) reasoning_open: bool, @@ -40,11 +41,12 @@ impl ChatMessage { } pub(super) fn refresh_markdown(&mut self) { - if !self.user { + if !self.user && !self.tool { + let visible = crate::agent::visible_content(&self.content); let content = if self.reasoning.is_some() { - self.content.trim_start() + visible.trim_start() } else { - &self.content + visible }; self.markdown = markdown::parse(content).collect(); } @@ -56,6 +58,7 @@ impl From for ChatMessage { let mut message = Self { id: message.id, user: message.user, + tool: message.tool, reasoning: message.reasoning, reasoning_complete: message.reasoning_complete, reasoning_open: false, @@ -88,13 +91,15 @@ impl App { crate::settings::effective_settings(model, &generation, &runtime, &models_path()) }) }); - let effective = match effective { + let mut effective = match effective { Ok(settings) => settings, Err(error) => { self.error = Some(error); return; } }; + effective.turn.system_prompt = + crate::agent::system_prompt(model, &effective.turn.system_prompt); let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct; #[cfg(target_os = "macos")] let mut messages = self @@ -102,6 +107,7 @@ impl App { .iter() .map(|message| ChatTurn { user: message.user, + tool: message.tool, skip_previous_eos: false, reasoning: message.reasoning.clone(), reasoning_complete: message.reasoning_complete, @@ -111,6 +117,7 @@ impl App { #[cfg(target_os = "macos")] messages.push(ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -189,6 +196,31 @@ impl App { } pub(super) fn poll_generation(&mut self) -> bool { + #[cfg(target_os = "macos")] + if let Some(active) = &self.active_tools { + match crate::agent::try_tool_result(active) { + Ok(Some(result)) => { + let cancelled = active.cancel.load(Ordering::Relaxed); + self.active_tools = None; + if cancelled { + self.generating = false; + return false; + } + if let Err(error) = self.continue_after_tool_result(&result) { + self.generating = false; + self.error = Some(error); + } + return true; + } + Ok(None) => return false, + Err(error) => { + self.active_tools = None; + self.generating = false; + self.error = Some(error); + return false; + } + } + } #[cfg(target_os = "macos")] let Some(active) = &mut self.active_generation else { self.generating = false; @@ -221,9 +253,32 @@ impl App { context_changed = true; } Ok(GenerationEvent::Finished(result)) => { - self.generating = false; - if let Err(error) = result { - self.error = Some(error); + match result { + Ok(_) => { + let model = ModelChoice::from_id(&self.preferences.selected_model) + .unwrap_or_default(); + let content = self + .conversation + .last() + .map(|message| message.content.clone()) + .unwrap_or_default(); + match crate::agent::parse_tool_calls(model, &content) { + Ok((_, calls)) if !calls.is_empty() => { + if let Err(error) = self.start_agent_tools(calls) { + self.generating = false; + self.error = Some(error); + } + } + Ok(_) => self.generating = false, + Err(error) => { + self.active_tools = Some(crate::agent::error_async(error)); + } + } + } + Err(error) => { + self.generating = false; + self.error = Some(error); + } } self.active_generation = None; break; @@ -286,6 +341,83 @@ impl App { false } + #[cfg(target_os = "macos")] + fn start_agent_tools(&mut self, calls: Vec) -> Result<(), String> { + let session_id = self + .selected_session + .ok_or_else(|| "The active session is unavailable.".to_owned())?; + if self.agent_tools.as_ref().map(|(id, _)| *id) != Some(session_id) { + let project_id = self + .selected_project + .ok_or_else(|| "The active project is unavailable.".to_owned())?; + let root = self + .projects + .iter() + .find(|project| project.project.id == project_id) + .map(|project| PathBuf::from(&project.project.path)) + .ok_or_else(|| "The active project is unavailable.".to_owned())?; + let tools = crate::agent::Tools::new(&root, self.preferences.context_tokens)?; + self.agent_tools = Some((session_id, Arc::new(Mutex::new(tools)))); + } + let tools = Arc::clone(&self.agent_tools.as_ref().unwrap().1); + self.active_tools = Some(crate::agent::execute_async(tools, calls)); + Ok(()) + } + + #[cfg(target_os = "macos")] + fn continue_after_tool_result(&mut self, result: &str) -> Result<(), String> { + let session_id = self + .selected_session + .ok_or_else(|| "The active session is unavailable.".to_owned())?; + let model = ModelChoice::from_id(&self.preferences.selected_model) + .ok_or_else(|| "The selected model is not supported.".to_owned())?; + let generation = self.preferences.generation()?; + let runtime = self.preferences.runtime()?; + let mut effective = + crate::settings::effective_settings(model, &generation, &runtime, &models_path())?; + effective.turn.system_prompt = + crate::agent::system_prompt(model, &effective.turn.system_prompt); + let assistant_reasoning = effective.turn.reasoning_mode != ReasoningMode::Direct; + let saved = self + .database + .as_mut() + .ok_or_else(|| "The project database is unavailable.".to_owned())? + .continue_tool_turn(session_id, result, assistant_reasoning) + .map_err(|error| format!("Could not save the tool turn: {error}"))?; + self.conversation.push(ChatMessage::from(saved.0)); + let messages = self + .conversation + .iter() + .map(|message| ChatTurn { + user: message.user, + tool: message.tool, + skip_previous_eos: false, + reasoning: message.reasoning.clone(), + reasoning_complete: message.reasoning_complete, + content: message.content.clone(), + }) + .collect(); + let mut assistant = ChatMessage::from(saved.1); + assistant.reasoning_open = assistant_reasoning; + self.conversation.push(assistant); + let idle_timeout = + Duration::from_secs(self.preferences.idle_timeout_minutes.max(1) as u64 * 60); + self.active_generation = Some( + self.generation_service + .as_ref() + .ok_or_else(|| "The model runtime is unavailable.".to_owned())? + .generate( + effective.engine, + effective.turn, + messages, + CheckpointTarget::Local(session_checkpoint_path(session_id)), + idle_timeout, + )?, + ); + self.tokens_per_second = None; + Ok(()) + } + /// Asks the model for a session title without opening a session for it: the /// turn runs against the shared transient KV cache and only its text is kept. /// @@ -318,6 +450,7 @@ impl App { .filter(|message| !message.content.trim().is_empty()) .map(|message| ChatTurn { user: message.user, + tool: message.tool, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -329,6 +462,7 @@ impl App { } messages.push(ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, diff --git a/src/app/view/chat.rs b/src/app/view/chat.rs index fd27db1..644326a 100644 --- a/src/app/view/chat.rs +++ b/src/app/view/chat.rs @@ -57,7 +57,13 @@ impl App { } else { let markdown_style = markdown::Style::from_palette(app_theme().palette()); for (index, message) in self.conversation.iter().enumerate() { - let label = if message.user { "You" } else { "DS4" }; + let label = if message.user { + "You" + } else if message.tool { + "Tool" + } else { + "DS4" + }; let active = self.generating && index + 1 == self.conversation.len(); let mut body = column![text(label).size(11)].spacing(5); if let Some(reasoning) = &message.reasoning { @@ -89,11 +95,11 @@ impl App { } } if !message.content.is_empty() { - if message.user || message.markdown.is_empty() { + if message.user || message.tool || message.markdown.is_empty() { let content = if message.reasoning.is_some() { - message.content.trim_start() + crate::agent::visible_content(&message.content).trim_start() } else { - &message.content + crate::agent::visible_content(&message.content) }; body = body.push(text(content).size(14)); } else { @@ -109,6 +115,14 @@ impl App { } else if active && message.reasoning.is_none() { body = body.push(text("Loading model…").size(14)); } + if !message.user + && !message.tool + && let Some(model) = ModelChoice::from_id(&self.preferences.selected_model) + { + for summary in crate::agent::tool_summaries(model, &message.content) { + body = body.push(text(summary).size(13).color(muted_text())); + } + } let user = message.user; messages = messages.push( container(body) diff --git a/src/database.rs b/src/database.rs index cf9471d..8efe38b 100644 --- a/src/database.rs +++ b/src/database.rs @@ -347,6 +347,7 @@ pub struct StoredMessage { pub id: i32, pub session_id: i32, pub user: bool, + pub tool: bool, pub reasoning: Option, pub reasoning_complete: bool, pub content: String, @@ -357,6 +358,7 @@ pub struct StoredMessage { struct NewMessage<'a> { session_id: i32, user: bool, + tool: bool, reasoning: Option<&'a str>, reasoning_complete: bool, content: &'a str, @@ -621,6 +623,7 @@ impl Database { .values(NewMessage { session_id, user: true, + tool: false, reasoning: None, reasoning_complete: true, content: prompt, @@ -631,6 +634,7 @@ impl Database { .values(NewMessage { session_id, user: false, + tool: false, reasoning: reasoning.then_some(""), reasoning_complete: !reasoning, content: "", @@ -642,6 +646,41 @@ impl Database { .map_err(|error: diesel::result::Error| error.to_string()) } + pub fn continue_tool_turn( + &mut self, + session_id: i32, + result: &str, + reasoning: bool, + ) -> Result<(StoredMessage, StoredMessage), String> { + self.connection + .transaction(|connection| { + let tool = diesel::insert_into(messages::table) + .values(NewMessage { + session_id, + user: false, + tool: true, + reasoning: None, + reasoning_complete: true, + content: result, + }) + .returning(StoredMessage::as_returning()) + .get_result(connection)?; + let assistant = diesel::insert_into(messages::table) + .values(NewMessage { + session_id, + user: false, + tool: false, + reasoning: reasoning.then_some(""), + reasoning_complete: !reasoning, + content: "", + }) + .returning(StoredMessage::as_returning()) + .get_result(connection)?; + Ok((tool, assistant)) + }) + .map_err(|error: diesel::result::Error| error.to_string()) + } + pub fn update_message( &mut self, id: i32, @@ -832,6 +871,9 @@ mod tests { database .update_message(assistant.id, Some("Reasoning"), true, "Answer") .unwrap(); + database + .continue_tool_turn(session.id, "Tool result", false) + .unwrap(); database .update_session_context(session.id, 1_234, 65_536, Some(12.5)) .unwrap(); @@ -843,12 +885,17 @@ mod tests { assert_eq!(projects[0].sessions[0].context_limit, 65_536); assert_eq!(projects[0].sessions[0].last_tokens_per_second, Some(12.5)); let messages = reopened.load_messages(session.id).unwrap(); - assert_eq!(messages.len(), 2); + assert_eq!(messages.len(), 4); assert!(messages[0].user); + assert!(!messages[0].tool); assert_eq!(messages[0].content, "Question"); assert_eq!(messages[1].reasoning.as_deref(), Some("Reasoning")); assert!(messages[1].reasoning_complete); assert_eq!(messages[1].content, "Answer"); + assert!(messages[2].tool); + assert_eq!(messages[2].content, "Tool result"); + assert!(!messages[3].user); + assert!(!messages[3].tool); reopened.delete_session(session.id).unwrap(); assert!(reopened.load_messages(session.id).unwrap().is_empty()); drop(reopened); diff --git a/src/engine.rs b/src/engine.rs index ad01fc1..1c2c8bb 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -319,6 +319,7 @@ pub(crate) struct Generator { #[derive(Clone)] pub(crate) struct ChatTurn { pub(crate) user: bool, + pub(crate) tool: bool, pub(crate) skip_previous_eos: bool, pub(crate) reasoning: Option, pub(crate) reasoning_complete: bool, @@ -571,6 +572,7 @@ impl Generator { let mut reasoning = settings.reasoning_mode != ReasoningMode::Direct; let mut generated = ChatTurn { user: false, + tool: false, skip_previous_eos: false, reasoning: reasoning.then(String::new), reasoning_complete: !reasoning, @@ -896,7 +898,7 @@ fn conversation_key(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn output.extend_from_slice(value.as_bytes()); } - let mut output = b"DS4Server chat checkpoint v2".to_vec(); + let mut output = b"DS4Server chat checkpoint v3".to_vec(); text(&mut output, system); output.push(match reasoning { ReasoningMode::Direct => 0, @@ -905,6 +907,7 @@ fn conversation_key(system: &str, reasoning: ReasoningMode, messages: &[ChatTurn }); for message in messages { output.push(u8::from(message.user)); + output.push(u8::from(message.tool)); output.push(u8::from(message.skip_previous_eos)); match &message.reasoning { Some(reasoning) => { @@ -1057,6 +1060,7 @@ mod sampling_tests { fn split_utf8_token_bytes_are_joined_before_decoding() { let mut generated = ChatTurn { user: false, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -1076,6 +1080,7 @@ mod sampling_tests { fn checkpoint_tag_covers_the_canonical_chat_state() { let mut messages = vec![ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -1102,6 +1107,7 @@ mod sampling_tests { let prefix = conversation_key("System", ReasoningMode::High, &messages); messages.push(ChatTurn { user: false, + tool: false, skip_previous_eos: false, reasoning: Some("because".into()), reasoning_complete: true, diff --git a/src/engine/tokenizer.rs b/src/engine/tokenizer.rs index 7b26cc7..9964381 100644 --- a/src/engine/tokenizer.rs +++ b/src/engine/tokenizer.rs @@ -195,6 +195,7 @@ impl Tokenizer { system_prompt, &[ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -235,6 +236,32 @@ impl Tokenizer { output.extend(self.tokenize_rendered(system_prompt)); } for message in messages { + if message.tool { + if self.family == ModelFamily::Glm { + output.push(self.observation); + output.extend(self.tokenize_rendered("")); + output.extend( + self.tokenize_rendered( + &message + .content + .replace("", "</tool_response>"), + ), + ); + output.extend(self.tokenize_rendered("")); + } else { + output.push(self.user); + output.extend(self.tokenize("")); + output.extend( + self.tokenize( + &message + .content + .replace("", "</tool_result>"), + ), + ); + output.extend(self.tokenize("")); + } + continue; + } output.push(if message.user { self.user } else { diff --git a/src/engine/validation.rs b/src/engine/validation.rs index 42c0eab..7b4daaa 100644 --- a/src/engine/validation.rs +++ b/src/engine/validation.rs @@ -904,6 +904,7 @@ mod tests { &[ ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -911,6 +912,7 @@ mod tests { }, ChatTurn { user: false, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -918,6 +920,7 @@ mod tests { }, ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -938,12 +941,33 @@ mod tests { model.render_continuation("Hello", ReasoningMode::Direct, true), [128_803, 19_923, 128_804, 128_822] ); + let tool_turn = ChatTurn { + user: false, + tool: true, + skip_previous_eos: false, + reasoning: None, + reasoning_complete: true, + content: "Hello".into(), + }; + let wrapped_user = ChatTurn { + user: true, + tool: false, + skip_previous_eos: false, + reasoning: None, + reasoning_complete: true, + content: "Hello".into(), + }; + assert_eq!( + model.render_conversation("", &[tool_turn], ReasoningMode::Direct), + model.render_conversation("", &[wrapped_user], ReasoningMode::Direct) + ); assert_eq!( model.render_conversation( "", &[ ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -951,6 +975,7 @@ mod tests { }, ChatTurn { user: false, + tool: false, skip_previous_eos: false, reasoning: Some("Hello".into()), reasoning_complete: true, @@ -958,6 +983,7 @@ mod tests { }, ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, diff --git a/src/main.rs b/src/main.rs index 6b67606..f5bb844 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +mod agent; mod app; mod database; mod engine; diff --git a/src/schema.rs b/src/schema.rs index de135a0..810f594 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -3,6 +3,7 @@ diesel::table! { id -> Integer, session_id -> Integer, user -> Bool, + tool -> Bool, reasoning -> Nullable, reasoning_complete -> Bool, content -> Text, diff --git a/src/server.rs b/src/server.rs index c844322..5ca2938 100644 --- a/src/server.rs +++ b/src/server.rs @@ -312,6 +312,49 @@ impl Drop for ConnectionSlot { } } +pub(crate) fn parse_dsml_tool_calls(text: &str) -> Result<(String, Vec<(String, Value)>), String> { + let has_tool_marker = TOOL_SYNTAXES + .iter() + .any(|syntax| text.contains(syntax.tool_start)); + let mut projector = ToolProjector::new(); + let events = projector.push(text, true, ""); + let mut content = String::new(); + let mut calls = Vec::<(String, String, bool)>::new(); + for event in events { + match event { + ToolProjectionEvent::Text(text) => content.push_str(&text), + ToolProjectionEvent::Start { index, name, .. } => { + if calls.len() == index { + calls.push((name, String::new(), false)); + } + } + ToolProjectionEvent::Arguments { index, fragment } => { + if let Some((_, arguments, _)) = calls.get_mut(index) { + arguments.push_str(&fragment); + } + } + ToolProjectionEvent::End { index } => { + if let Some((_, _, complete)) = calls.get_mut(index) { + *complete = true; + } + } + } + } + let calls = calls + .into_iter() + .filter(|(_, _, complete)| *complete) + .map(|(name, arguments, _)| { + serde_json::from_str(&arguments) + .map(|arguments| (name, arguments)) + .map_err(|error| format!("invalid DSML tool arguments: {error}")) + }) + .collect::, _>>()?; + if has_tool_marker && calls.is_empty() { + return Err("invalid or incomplete DSML tool call".into()); + } + Ok((content, calls)) +} + fn handle(mut stream: TcpStream, state: &State) { let _ = stream.set_write_timeout(Some(HTTP_IO_TIMEOUT)); let started = Instant::now(); diff --git a/src/server/tools.rs b/src/server/tools.rs index d1368de..035ebd6 100644 --- a/src/server/tools.rs +++ b/src/server/tools.rs @@ -132,7 +132,11 @@ impl ToolProjector { self.state = ToolProjectionState::Failed; break; }; - let id = random_tool_id(prefix); + let id = if prefix.is_empty() { + String::new() + } else { + random_tool_id(prefix) + }; self.ids.push(id.clone()); events.push(ToolProjectionEvent::Start { index: self.index, @@ -370,6 +374,7 @@ pub(super) fn render_messages( } "user" => turns.push(ChatTurn { user: true, + tool: false, skip_previous_eos: false, reasoning: None, reasoning_complete: true, @@ -388,6 +393,7 @@ pub(super) fn render_messages( } else { turns.push(ChatTurn { user: true, + tool: false, skip_previous_eos: protocol == Protocol::Responses, reasoning: None, reasoning_complete: true, @@ -403,6 +409,7 @@ pub(super) fn render_messages( let reasoning = content_text(&message.reasoning_content); turns.push(ChatTurn { user: false, + tool: false, skip_previous_eos: false, reasoning: (preserve_reasoning && !reasoning.is_empty()).then_some(reasoning), reasoning_complete: true,