Compare commits

...

126 Commits

Author SHA1 Message Date
da13bac755 fix: more work on background tasks 2026-07-24 11:43:11 +02:00
c187108f89 Localize category labels in blog menus. 2026-07-24 09:20:12 +02:00
7a3c5683e7 feat: reworked the category editing 2026-07-24 09:00:12 +02:00
d9085cf681 fix: still working on rendering speed 2026-07-24 07:10:57 +02:00
473c48937d fix: align rendering process with bds2 2026-07-24 06:22:45 +02:00
28119c15b7 fix: display fix for tasks with long titles 2026-07-23 23:07:41 +02:00
87318c8b22 fix: hopefully fixed the background task progress situation 2026-07-23 22:48:53 +02:00
ea90167a5a fix: fixed rendering from validate-apply 2026-07-23 21:51:12 +02:00
2844ed5a47 Add forced full-site rendering 2026-07-23 10:23:02 +02:00
d87e80f96b Match bDS2 zero-byte route validation 2026-07-23 09:55:19 +02:00
3e3b2d9698 Color-code Git commit sync status 2026-07-23 09:46:50 +02:00
632822fe1b Remove remote editing from the Git sidebar 2026-07-23 09:38:40 +02:00
08a5bd192c Preserve input focus across passive overlays 2026-07-23 09:28:16 +02:00
a01c86e383 Exit when the desktop window closes 2026-07-23 09:21:44 +02:00
4de0b6b0d1 fix: website validation 2026-07-23 08:54:07 +02:00
0d44c21b0e Localize sidebar dates and calendar months 2026-07-22 20:28:40 +02:00
7b8e539340 Match bDS2 canonical file serialization 2026-07-22 20:19:21 +02:00
fdf47da40a Restore project workspace state 2026-07-22 19:57:26 +02:00
0753b290ad Complete forced template deletion 2026-07-22 19:26:59 +02:00
fc039d813a Update the persistence schema specification 2026-07-22 19:17:50 +02:00
9087096c5d Localize engine progress messages 2026-07-22 19:09:10 +02:00
3051331b87 Match post tag names exactly 2026-07-22 18:58:29 +02:00
a1c04ff898 Default script entrypoints by kind 2026-07-22 18:54:27 +02:00
e9f2cdb25d Support HEIC media imports 2026-07-22 18:49:27 +02:00
bdfe57b66f Encode WebP thumbnails at quality 80 2026-07-22 18:33:20 +02:00
70503c8bc2 Enforce unique MCP proposal identities 2026-07-22 18:25:19 +02:00
2dcaf7f7f7 Limit automatic translation to manual saves 2026-07-22 18:11:09 +02:00
556d9c1ff4 Classify localized page routes as core 2026-07-22 18:06:06 +02:00
ba6a297d9b Use configured languages in generation validation 2026-07-22 18:02:04 +02:00
64e3847ef2 chore: fix/cleanup stuff related to previous unchecked commits 2026-07-22 17:54:33 +02:00
Bauer, Georg
a3f2c451e3 fix: small findings with regard to deletes from sidebar 2026-07-22 15:35:22 +02:00
Bauer, Georg
cc520d517a Add hover-revealed delete button to script sidebar rows, routed through the confirm-delete modal.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 15:20:47 +02:00
Bauer, Georg
5968cf47ba Drop the media translation delete button from the spec; scripting API remains the only delete path.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 15:15:47 +02:00
Bauer, Georg
2bf95bfcd0 Spec script deletion as confirm_delete_modal and cover it with a test.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 15:15:31 +02:00
Bauer, Georg
733aa9a6ff Formalize tier-1 comment-based specs into allium constructs.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 14:58:22 +02:00
Bauer, Georg
1dd8194fa8 chore: cleaned up trigger problems in allium specs 2026-07-22 14:33:42 +02:00
2fc683d513 Add portable project metadata diff coverage. 2026-07-21 23:54:37 +02:00
1410b97aeb Reopen published code only for content changes. 2026-07-21 23:25:11 +02:00
8a6274ef6a Normalize template and script slug changes. 2026-07-21 23:18:12 +02:00
e28d42d345 Cascade template slug changes to references. 2026-07-21 23:05:29 +02:00
b8014e0a10 Align legacy post snapshots with bDS2. 2026-07-21 22:54:06 +02:00
535303e8b5 Restore discarded posts from canonical files. 2026-07-21 22:45:59 +02:00
1cdc86b9cc Resync media sidecars after post deletion. 2026-07-21 22:38:53 +02:00
53ada2c705 Remove stale post files after path changes. 2026-07-21 22:31:01 +02:00
3920d6fc70 Keep post link graphs synchronized. 2026-07-21 22:24:28 +02:00
b0e9223158 Reopen canonical drafts after manual translation edits. 2026-07-21 22:13:36 +02:00
89a07c885a Preserve file-only content when archiving. 2026-07-21 22:04:03 +02:00
1c04087471 Add explicit post unarchive. 2026-07-21 21:49:06 +02:00
1e32c7e945 Rewrite published posts after template changes. 2026-07-21 21:32:28 +02:00
80f291b8ea Reopen published posts only for changes. 2026-07-21 21:25:16 +02:00
b3bf134980 Align checksums with bDS2. 2026-07-21 21:18:47 +02:00
8e78a5fc2b fix: syntax highlighting was broken in bundled app 2026-07-21 21:05:24 +02:00
8170076e9b Render built-in macros from Liquid templates. 2026-07-21 20:44:03 +02:00
3ac7a8bfc4 Enforce the supported Liquid subset. 2026-07-21 20:21:35 +02:00
557497d4c2 Persist status-bar airplane mode. 2026-07-21 20:11:33 +02:00
2f7d752e73 Fix online one-shot AI compatibility. 2026-07-21 20:03:26 +02:00
812de13f62 fix: clean up some problems from the implementations 2026-07-21 19:46:57 +02:00
Georg Bauer
71d998f68e Merge pull request #5 from rfc1437/codex/vollstandigen-test-der-losung-durchfuhren
Bypass proxy for local AI/chat endpoints and improve test API key handling
2026-07-21 19:06:28 +02:00
Georg Bauer
489d3646f8 Fix loopback test networking. 2026-07-21 19:02:07 +02:00
Georg Bauer
9d3a092359 Merge pull request #4 from rfc1437/codex/run-full-test-suite-after-fixing-internet
Fix workspace test build on latest Rust
2026-07-21 13:47:07 +02:00
Georg Bauer
1ecd7a32d2 Merge pull request #3 from rfc1437/codex/thread-data_dir-in-fts-index-paths
Index published post bodies from files
2026-07-21 13:46:47 +02:00
Georg Bauer
0b4c303c89 Merge pull request #2 from rfc1437/codex/uberprufe-dateianderungen-von-ruds
Preserve category archive titles in metadata writes
2026-07-21 13:46:26 +02:00
Georg Bauer
18d96a10b8 Fix search indexing data directory 2026-07-21 13:27:09 +02:00
Georg Bauer
184d6e4845 Fix workspace test build warnings 2026-07-21 12:42:48 +02:00
Georg Bauer
d50e538fc3 Index published post bodies from files 2026-07-21 11:21:03 +02:00
Georg Bauer
1e6c2a4e2c Preserve category archive titles. 2026-07-21 11:20:07 +02:00
Bauer, Georg
486a9f8019 Update default branch to main in AGENTS.md
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 10:20:34 +02:00
08ca5f14aa Merge pull request 'main' (#98) from main into master
Reviewed-on: #98
2026-07-21 08:13:16 +00:00
Georg Bauer
eb61a03235 Merge pull request #1 from rfc1437/codex/run-one-shot-activities-in-background-thread
Run one-shot AI actions asynchronously
2026-07-21 10:10:27 +02:00
Georg Bauer
fdd9ddde4a Run one-shot AI actions asynchronously 2026-07-21 10:09:07 +02:00
ac993d8743 Align slug transliteration with bDS2 2026-07-20 23:01:12 +02:00
b641f6492b Add semantic post tag suggestions 2026-07-20 22:52:03 +02:00
d1829f0e9b Schedule only missing post translations 2026-07-20 22:20:57 +02:00
b713f7fa07 Open quick actions in editor popovers 2026-07-20 21:57:14 +02:00
e6fa506b28 Add syntax highlighting to code editors 2026-07-20 21:39:40 +02:00
fe6bf3e1f4 Open Blogmarks through the standard tab flow 2026-07-20 21:13:21 +02:00
dc11155aac Fix blog descriptions in list page titles 2026-07-20 20:05:32 +02:00
3795db76b0 Fix calendar activity heatmap in preview and generation 2026-07-20 19:56:18 +02:00
5726b92c5d Fix canonical media links in preview and generation 2026-07-20 19:42:00 +02:00
713575dd99 Fix project macro rendering in preview and generation 2026-07-20 19:31:41 +02:00
5425d83ee1 Fix blogmark deep-link delivery and editor activation 2026-07-20 19:13:18 +02:00
3aa5dcbe55 Fix style swatches and live preview modes. 2026-07-20 18:45:27 +02:00
e531d32a77 Hide post gallery action in preview mode. 2026-07-20 18:18:53 +02:00
47d069dfac Update bDS2 rendering fixtures and routes. 2026-07-20 18:07:54 +02:00
49adfed68f Fix navigable blog previews. 2026-07-20 17:10:33 +02:00
29cdd016b8 Implement post editor image drops. 2026-07-20 16:19:14 +02:00
49447ef451 Implement the bDS2 sync scripting API. 2026-07-20 15:53:31 +02:00
b10212d47e Remove orphaned UI messages. 2026-07-20 15:32:19 +02:00
d922f5661f Localize the terminal interface. 2026-07-20 15:29:31 +02:00
25fe22a3d2 Remove dead TUI style settings. 2026-07-20 14:41:32 +02:00
fe7dd82f33 Implement the Pico CSS theme editor. 2026-07-20 14:28:47 +02:00
1d190d33c5 Update dependencies and add dependency audit gates. 2026-07-20 13:37:23 +02:00
df71f1de95 Load sidebar entries automatically in 200-item batches. 2026-07-20 13:21:14 +02:00
7887d02e0b chore: documentation added and adapted to RuDS 2026-07-20 13:02:34 +02:00
c82b790b82 chore: removed plan documents, we moved to issues now 2026-07-20 12:41:14 +02:00
91d3ae475e chore: clippy issues 2026-07-20 12:18:34 +02:00
c80b5d680d Persist discovered AI endpoint models in the database per endpoint.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-20 12:13:56 +02:00
2d6fcafc7e Separate online and airplane AI profiles. 2026-07-20 11:52:00 +02:00
3739da0984 Show effective settings in bds-cli config. 2026-07-20 09:28:26 +02:00
269e4b0e4a fix: reduced app size by a good margin 2026-07-20 08:44:17 +02:00
7ba2699b68 Restore native text editing commands 2026-07-19 22:23:32 +02:00
c56550f1ad Modernize application scrollbars 2026-07-19 21:56:39 +02:00
ca7b864261 Clarify activity bar tooltips 2026-07-19 21:46:19 +02:00
23981ce2ae Implement the terminal UI 2026-07-19 21:36:49 +02:00
45cb0cd502 Implement the headless SSH server 2026-07-19 20:35:02 +02:00
92a942d52f Implement the OPML menu editor 2026-07-19 19:37:00 +02:00
7d20139531 Add in-app documentation browsers 2026-07-19 18:57:46 +02:00
e8764262a3 Implement semantic embeddings and duplicate review 2026-07-19 18:06:47 +02:00
fd4fd744e0 Implement persistent native A2UI chat surfaces 2026-07-19 16:57:43 +02:00
ac611e3f7f Implement persistent conversational AI chat 2026-07-19 16:18:01 +02:00
fb5cae2131 Implement MCP automation and approvals. 2026-07-19 15:10:43 +02:00
fdfae200a0 Implement the shared automation CLI. 2026-07-19 14:25:15 +02:00
a2abb901cd Add the shared domain event bus. 2026-07-19 13:48:03 +02:00
429f471e4e Implement complete WordPress WXR import workflow. 2026-07-19 13:17:45 +02:00
9dab0ca57e fix: project reload on rebuild and syntax check now gives toast 2026-07-19 12:07:10 +02:00
422f71c8ad Implement the Git workflow 2026-07-19 11:53:02 +02:00
90a9002124 Group site generation by section 2026-07-19 10:27:12 +02:00
055dd5efc5 Implement post editor gallery image imports 2026-07-19 09:52:09 +02:00
9a72287fc6 feat: implement the bDS2-compatible core Lua API 2026-07-19 09:24:39 +02:00
33929fd04b feat: return normalized token usage from one-shot AI calls (closes #15)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-18 23:15:04 +02:00
97fd213817 fix: bring dashboard tag cloud and sections into bDS2 parity (closes #11)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-18 23:08:27 +02:00
4222c480cb feat: remove the post duplication feature (closes #10)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-18 22:55:37 +02:00
a2e2ab037d chore: cleaned up some specs around extensions and small changes to get code in line 2026-07-18 22:52:54 +02:00
5c1b333cce chore: reworked some specs and did small addition to database schema based on gaps in spec, also rechecked the core plan 2026-07-18 22:21:02 +02:00
943e5fa39b feat: media editor uses quick actions dropdown like bDS2 (closes #9)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-18 21:57:10 +02:00
eb8c2f786d fix: white text on manage-tags rows for readable contrast (#7)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-18 21:52:22 +02:00
293 changed files with 100005 additions and 29098 deletions

View File

@@ -2,3 +2,19 @@
bundle-macos = "packager --release --packages bds-ui --formats app --formats dmg"
bundle-windows = "packager --release --packages bds-ui --formats nsis"
bundle-linux = "packager --release --packages bds-ui --formats deb --formats appimage"
[target.'cfg(target_os = "macos")']
rustflags = [
"-C", "prefer-dynamic",
"-C", "link-arg=-mmacosx-version-min=26.0",
"-C", "link-arg=-Wl,-rpath,@executable_path/../Resources",
]
[target.'cfg(target_os = "linux")']
rustflags = [
"-C", "prefer-dynamic",
"-C", "link-arg=-Wl,-rpath,$ORIGIN/../lib/bds-ui",
]
[target.'cfg(target_os = "windows")']
rustflags = ["-C", "prefer-dynamic"]

View File

@@ -12,6 +12,15 @@ Invariants and behaviours in the allium spec should be covered by unit tests of
- Before implementing or changing UI, read and follow `docs/UI_STYLE_GUIDE.md`.
- Reuse the shared styling primitives described there so new sidebars and editor areas remain consistent with the post editor.
## Computer Use for the macOS app
- Build the real app bundle with `cargo bundle-macos`.
- Use Computer Use against the exact app path `target/release/Blogging Desktop Server.app` (resolved to its absolute path), then raise that window before interacting with it.
- Do not use `cargo run` for Computer Use: it starts RuDS as a child of the command runner and Computer Use cannot attach to it as a macOS app.
- Do not run `open` on `target/debug/bds-ui`; macOS hands the executable to Ghostty instead of launching RuDS as an app.
- Do not target only the `de.rfc1437.ruds` bundle identifier because both the installed app and the repository build may share it; always use the exact repository bundle path.
- Do not improvise with AppleScript, Dock automation, or raw screen-capture workarounds. Use the Computer Use skill and its `node_repl` runtime.
## Plan Mode
- Make the plan extremely concise. Sacrifice grammar for the sake of concision.
@@ -19,13 +28,14 @@ Invariants and behaviours in the allium spec should be covered by unit tests of
## Commits
- our default branch is origin/master
- our default branch is origin/main
- commit messages are short - one sentence. do not write long articles.
- pull requests are more verbose and especially give reasoning for changes
## Important facts
- update `README.md` whenever a user-visible feature is added, removed, or materially changed; keep it a compact overview with relevant pointers
- work is tracked as Gitea issues (`tea` CLI, repo `hugo/RuDS`); there are no plan documents — file or update issues when scope changes
- published posts don't have body in the database, the body content is only in the file
- functionality you implement have to be tied to UI
- UI you implement has to be tied to functionality
@@ -45,6 +55,7 @@ Invariants and behaviours in the allium spec should be covered by unit tests of
- there are no "pre-existing" problems - you own every problem, you fix every problem
- don't leave unused code in the codebase, remove it instead
- after implementing / changing things, run the build and run tests to verify all works
- run `cargo test --workspace` with permission for its loopback test servers on the first attempt; AI mock-server and preview-server tests bind localhost and otherwise fail under sandboxing, causing a pointless rerun
- do not reference external JavaScript or CSS on CDNs, always bring it into the project
- do not embedd CSS/JavaScript into HTML, always reference .css and .js files in the project assets
- always make sure you follow proper i18n best practices. no untranslated string constants.

124
CLI.md Normal file
View File

@@ -0,0 +1,124 @@
# RuDS Command Line, Headless Server, and Terminal UI
## In this article
- [What the CLI is](#what-the-cli-is)
- [Installing the CLI](#installing-the-cli)
- [Global flags and output](#global-flags-and-output)
- [Command reference](#command-reference)
- [Creating content from the command line](#creating-content-from-the-command-line)
- [Running the headless server](#running-the-headless-server)
- [Using the terminal UI](#using-the-terminal-ui)
- [How the CLI and the desktop app stay in sync](#how-the-cli-and-the-desktop-app-stay-in-sync)
## What the CLI is
`bds-cli` is the automation surface of RuDS. It runs the same engines against the same application database and the same projects as the desktop app — there is no separate state, no export/import step, and no second configuration. Anything the CLI changes shows up in an open desktop window automatically.
Typical uses: scripted rebuilds and site generation, publishing from CI or cron, creating posts from other tools, importing images in bulk, running utility Lua scripts, and starting the headless server or the terminal UI.
## Installing the CLI
Install the launcher from the desktop app under **Settings → Data**, or run `bds-cli install` from a packaged binary. Both place a small forwarding launcher in `~/.local/bin` that executes the packaged CLI next to the shared runtime. Make sure `~/.local/bin` is on your `PATH`.
## Global flags and output
| Flag | Effect |
|---|---|
| `--json` | Print a stable JSON result envelope instead of human-readable text |
| `--airplane` | Block network commands and route automatic AI work to the local (airplane) endpoint |
The JSON envelope always has the shape `{"ok": bool, "command": string, "message": string, "data": object, "progress": [string], "notices": [string]}`, which makes the CLI safe to drive from scripts and LLM agents. Errors exit with code 1 and a formatted message on stderr; unknown commands and invalid options do the same.
With `--airplane`, `upload`, `push`, and `pull` are refused, and AI-assisted steps (language detection, translation, image enrichment) use the configured airplane endpoint. When no local endpoint is configured, the CLI prints a notice and falls back to offline heuristics where they exist — the command-line equivalent of the desktop's airplane-mode toast.
## Command reference
All commands operate on the active project from the shared project registry unless stated otherwise.
| Command | What it does |
|---|---|
| `rebuild` | Rebuild the cache database from the active project's files. `--incremental` applies only detected differences and imports orphan files |
| `repair <part>` | Run one derived-data repair: `post-links`, `media-links`, `thumbnails`, `embeddings`, or `search` |
| `render` | Render the generated site. `--incremental` validates output and applies only differences; `--force` re-renders everything, ignoring content hashes |
| `upload` | Upload generated HTML, thumbnails, and media using the project publishing settings, waiting for the publish job |
| `push` | Push the project repository to origin |
| `pull` | Fast-forward pull, then reconcile the cache database with the pulled files |
| `post` | Create a post from flags or JSON on stdin (see below) |
| `media <file>` | Import one image with best-effort AI enrichment and translations. `--language` overrides detection |
| `gallery <images…>` | Create a post and import/link all supplied images through the shared gallery pipeline |
| `config get\|set\|list` | Read or update global application settings; `list` shows effective values with secrets redacted |
| `project list\|add\|switch` | List, add (`--name` optional), or switch projects in the shared registry |
| `lua <script> [args…]` | Run an enabled utility Lua script from the active project in the sandboxed runtime |
| `server` | Start the authenticated headless SSH server (see below) |
| `tui` | Start the interactive terminal UI locally |
| `install` | Install the forwarding launcher in `~/.local/bin` |
## Creating content from the command line
`post` accepts either flags or a JSON object on stdin:
```
bds-cli post --title "Release notes" --content "…" --tags releases,news
echo '{"title":"Release notes","content":"…","tags":["releases"]}' | bds-cli post --stdin
```
Available flags: `--title`, `--content`, `--excerpt`, `--author`, `--language`, `--template`, `--tags`, `--categories`, `--no-translate`, `--stdin`. When `--language` is omitted the language is auto-detected (AI endpoint when available, offline heuristic otherwise, with a printed notice). After creation the same automatic translation cascade as the desktop app is scheduled and awaited.
`gallery` takes the same post flags plus image paths; every image is imported, linked to the new post, AI-enriched, and translated like the desktop gallery workflow. `media` runs the same pipeline for a single image without creating a post.
## Running the headless server
`bds-cli server` runs RuDS without a window — same binary family, same database, same engines. Launching the desktop binary with `BDS_MODE=server` is equivalent.
| Flag | Effect |
|---|---|
| `--bind <ip>` | SSH listen address. Defaults to loopback; external access must be explicit |
| `--port <port>` | SSH listen port (default 2222) |
| `--database <path>` | Application database path override |
| `--data-dir <path>` | Private application data directory containing the SSH key material |
Only the SSH port is ever exposed. On first start the server generates a restrictive RSA host key and an empty `authorized_keys` file in an `ssh/` folder inside the private application data directory. Add one public key per line — standard OpenSSH format. `authorized_keys` is re-read on every authentication, so key changes need no restart. There are no passwords.
Clients connect in two ways:
- **Terminal**: `ssh -p 2222 user@server` opens the terminal UI directly in the SSH session. Each client gets its own session; all sessions share the server's data and UI language.
- **Desktop app**: **File → Connect to Server…** with `user@host` or `user@host:port`. The desktop uses its own generated Ed25519 identity and trust-on-first-use `known_hosts`, negotiates the native protocol over the same SSH connection, and shows the remote workspace in the window. **File → Disconnect from Server** returns to the local workspace.
All connected clients stay synchronized through ordered domain events: an edit in one session refreshes lists and closes deleted-entity tabs everywhere.
## Using the terminal UI
Start it locally with `bds-cli tui`, or get it automatically over `ssh` to a headless server. The status line at the bottom always shows the keys available in the current context.
### Views and navigation
| Key | View |
|---|---|
| `1` | Posts |
| `2` | Media |
| `3` | Templates |
| `4` | Scripts |
| `5` | Tags |
| `6` | Settings |
| `7` | Git |
`↑`/`↓` or `j`/`k` move the sidebar selection, `Enter` opens the selected entry, `Esc` goes back, `q` quits.
### Working with content
- Posts open in a syntax-highlighted, soft-wrapped editor. `n` creates a new post. `Ctrl+S` saves the draft, `Ctrl+P` saves and publishes, `Ctrl+E` toggles the rendered Markdown preview, `Ctrl+G` runs the one-shot AI post analysis (airplane-gated).
- Templates and scripts open with HTML+Liquid and Lua highlighting.
- Images open in the terminal image preview.
- `/` opens a live filter for the current view: plain words search text, `tag:x` and `category:x` filter like the desktop chips, and an ISO date (`yyyy-mm-dd`) narrows to that day. Tokens combine with AND; `Enter` keeps the filter, `Esc` clears it.
- `:` opens the command prompt with the parameterless Blog-menu commands (metadata diff, validate site, force render, rebuilds, reindex, translations, duplicates, upload, preview URL). `:?` shows the list as help. Completed metadata-diff and site-validation tasks open a report panel; `Enter` applies the whole report, `Esc` closes it.
- `p` opens the project switcher; `o` switches to a folder prompt with bash-style tab completion to open a folder as a new project (a full rebuild is queued automatically).
- In Tags: `n` create, `Enter` rename, `c` cycle color, `t` cycle template, `d` then `y` delete, `s` sync from posts; in Merge, `Space` marks and `m` merges.
- In Git: `c` commit (stage all), `u` pull (fast-forward only), `s` push, `Enter` jumps the diff to a file.
- Settings sections mirror the desktop editor; `Enter` edits a field in the status line, `Ctrl+S` saves the section.
## How the CLI and the desktop app stay in sync
Every CLI mutation writes a notification row into the shared database. A running desktop app watches for these, invalidates its caches, and refreshes the affected views within moments — no manual reload. Processed notifications are pruned automatically.
Because CLI, TUI, server, and desktop all use the same engines, invariants hold everywhere: published bodies live in files, metadata flushes to the filesystem, and rebuilds reconcile external changes.

4511
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -5,6 +5,9 @@ members = [
"crates/bds-editor",
"crates/bds-ui",
"crates/bds-cli",
"crates/bds-mcp",
"crates/bds-server",
"crates/bds-package-runtime",
]
[workspace.package]
@@ -19,15 +22,20 @@ uuid = { version = "1", features = ["v4", "serde"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"
chrono = { version = "0.4", features = ["serde"] }
chrono = { version = "0.4", features = ["serde", "unstable-locales"] }
sha2 = "0.10"
deunicode = "1"
md5 = "0.8.1"
unicode-normalization = "0.1"
thiserror = "2"
anyhow = "1"
clap = { version = "4", features = ["derive"] }
tokio = { version = "1", features = ["full"] }
axum = "0.8"
walkdir = "2"
futures = "0.3"
image = "0.25"
hpvcd = "0.3.1"
webp = { version = "0.3.1", default-features = false }
rust-stemmers = "1"
sys-locale = "0.3"
fluent-bundle = "0.16"
@@ -38,6 +46,8 @@ pulldown-cmark = "0.13"
liquid = "0.26"
liquid-core = { version = "0.26", features = ["derive"] }
quick-xml = "0.41"
htmd = "0.5.4"
regex = "1"
rayon = "1.10"
pagefind = "1.5.2"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
@@ -47,14 +57,21 @@ diesel_migrations = "2.3.2"
libsqlite3-sys = { version = "0.37.0", features = ["bundled"] }
mlua = { version = "0.12", features = ["lua54", "vendored", "serialize"] }
url = "2.5"
fastembed = { version = "5.17.3", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"] }
usearch = "2.26"
ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "api-24", "coreml", "directml"] }
russh = "0.62.2"
ratatui = "0.30.2"
ratatui-image = { version = "11.0.6", default-features = false, features = ["crossterm", "image-defaults"] }
tui-markdown = { version = "0.3.8", default-features = false }
# UI framework
iced = { version = "0.13", features = ["wgpu", "advanced", "image", "svg", "tokio"] }
iced = { version = "0.13", features = ["wgpu", "advanced", "canvas", "image", "svg", "tokio", "markdown"] }
# Editor widget
cosmic-text = "0.12"
ropey = "1"
syntect = "5"
syntect = { version = "5", default-features = false, features = ["default-fancy"] }
# Platform integration (cross-platform)
muda = "0.15"
@@ -63,3 +80,8 @@ rfd = "0.15"
# Internal crates
bds-core = { path = "crates/bds-core" }
bds-editor = { path = "crates/bds-editor" }
bds-mcp = { path = "crates/bds-mcp" }
bds-server = { path = "crates/bds-server" }
[profile.release]
strip = "symbols"

604
DOCUMENTATION.md Normal file
View File

@@ -0,0 +1,604 @@
# RuDS User Guide
## In this article
- [Who this guide is for](#who-this-guide-is-for)
- [How RuDS works](#how-ruds-works)
- [Getting started](#getting-started)
- [Understanding the interface](#understanding-the-interface)
- [Working with posts](#working-with-posts)
- [Working with pages](#working-with-pages)
- [Working with media](#working-with-media)
- [Working with translations](#working-with-translations)
- [Finding content and duplicates](#finding-content-and-duplicates)
- [Previewing your site](#previewing-your-site)
- [Editing site navigation](#editing-site-navigation)
- [Using macros](#using-macros)
- [Using scripting](#using-scripting)
- [Using the AI assistant](#using-the-ai-assistant)
- [Organizing with tags](#organizing-with-tags)
- [Using blogmarks](#using-blogmarks)
- [Importing from WordPress (WXR)](#importing-from-wordpress-wxr)
- [Using Git (Source Control)](#using-git-source-control)
- [Configuring settings](#configuring-settings)
- [Checking and repairing metadata](#checking-and-repairing-metadata)
- [Managing templates](#managing-templates)
- [Generating and publishing](#generating-and-publishing)
- [Typical editorial workflows](#typical-editorial-workflows)
- [Working fully offline](#working-fully-offline)
- [Connecting external AI agents (MCP)](#connecting-external-ai-agents-mcp)
- [Using the command line](#using-the-command-line)
- [Running a headless server and the terminal UI](#running-a-headless-server-and-the-terminal-ui)
- [Troubleshooting and recovery](#troubleshooting-and-recovery)
- [Team conventions](#team-conventions)
## Who this guide is for
This guide is for people who use RuDS day to day to create, edit, organize, translate, generate, and publish blog content. It is written for editors, content managers, and project owners who need reliable guidance on what each part of the application does and how to use it safely.
If you need implementation notes, project architecture, or development setup, use the repository README. This guide stays focused on end-user operation and editorial decisions.
### Key takeaways
- RuDS documentation should help with real editorial work, not only isolated clicks.
- Each chapter explains purpose first, then usage.
- Safe content handling and recoverability matter throughout the application.
[↑ Back to In this article](#in-this-article)
---
## How RuDS works
RuDS is a local-first writing and publishing workspace. You can draft, revise, structure, preview, and publish content on your machine without depending on constant internet access. Optional remote Git synchronization and AI-assisted workflows extend that model, but they do not replace it.
Three states matter in day-to-day work. A draft is your in-progress state. Publishing marks a local content state as published inside your project. A Git commit creates a recoverable snapshot that can be reviewed, synchronized, and restored. These actions are related, but they are not the same operation.
The recommended sequence remains simple: edit in draft, publish when the content is ready, then commit immediately. That is the safest pattern for protecting work and keeping project history understandable.
### Key takeaways
- RuDS is designed for local reliability first.
- Publish and commit are different actions and both matter.
- The safe default lifecycle is: Draft -> Publish -> Commit.
[↑ Back to In this article](#in-this-article)
---
## Getting started
Before you begin editorial work, confirm that the project context is correct. Open RuDS and select the right project. If this is a new project, create it and define its identity early, including project name and description.
Next, open Settings and verify the project data path and Public Base URL. The data path should match your backup strategy. The Public Base URL should be set early because sitemap and feed generation depend on it.
Finally, define language and author defaults. These defaults reduce repetitive edits and keep output consistent when multiple contributors work in the same project.
### Key takeaways
- Set project identity, data location, and Public Base URL at the beginning.
- Configure language and author defaults before regular editing starts.
- Early setup decisions reduce later cleanup.
[↑ Back to In this article](#in-this-article)
---
## Understanding the interface
The RuDS interface is organized around workflows rather than isolated forms. The Activity Bar on the left moves between the major areas: Posts, Pages, Media, Scripts, Templates, Tags, Chat, Import, Source Control, and Settings. The Sidebar changes with the active area and helps with filtering, selection, and navigation. The Editor area is where most work happens and supports tabbed editing for content, configuration, and analysis views. When no tab is open, the editor area shows the Dashboard with project statistics, a posting timeline, a tag cloud, and recent posts.
The status bar at the bottom shows the active project and holds the airplane-mode switch that decides whether online AI endpoints may be used. It also shows the applied site theme and, during AI chat, running token totals.
The bottom panel and status area matter during longer operations such as imports, rebuild actions, metadata scans, and media work. Toasts provide quick feedback. The Output panel provides deeper detail when something needs attention.
Tab behavior is optimized for quick scanning and focused editing. Single click often opens a transient tab. Double click or explicit actions pin a tab for longer work.
### Key takeaways
- Use the Activity Bar for section-level context switching.
- Use the Sidebar for finding and narrowing content.
- Pin tabs when you move from inspection to editing.
[↑ Back to In this article](#in-this-article)
---
## Working with posts
The Posts section is for chronological content such as articles, notes, and recurring updates. In most editorial teams, Posts are the primary outward-facing stream.
A post combines title, body content, category, tags, excerpt, and status. Titles establish topic. Body content carries the narrative. Categories provide broad structure. Tags support finer discovery. Status should be used intentionally so collaborative workflows stay clear.
A reliable post workflow is: draft to completion, review structure and metadata, preview the result, publish when editorially ready, then commit immediately.
When you want help refining post metadata, use Quick Actions in the post editor and review AI suggestions for title, summary, and slug. Treat this as editorial assistance, not an automatic rewrite.
The post editor's links panel shows outgoing links to other posts and backlinks from posts that reference the current one. Use it to check context before restructuring or unpublishing, and click a linked post to open it directly.
### Key takeaways
- Use Posts for date-oriented and regularly updated content.
- Categories and tags serve different purposes: broad grouping versus precise discovery.
- Publish only when editorially ready, then commit right away.
[↑ Back to In this article](#in-this-article)
---
## Working with pages
Pages are for durable, non-chronological content such as About, Contact, legal notices, and other structural information. Use Pages when content should stay stable in navigation and should not be interpreted as part of a time-based feed.
Because pages are revisited over longer periods, naming consistency matters. Keep titles and slugs predictable, avoid unnecessary structural churn, and follow your project navigation conventions.
The working pattern is similar to posts: draft, review, preview, publish, commit. The difference is editorial intent: pages prioritize clarity and long-term maintainability over release cadence.
### Key takeaways
- Use Pages for stable structural content.
- Keep titles and slugs consistent for maintainability.
- Apply the same safe lifecycle: Draft -> Publish -> Commit.
[↑ Back to In this article](#in-this-article)
---
## Working with media
The Media section is where you import, describe, and maintain assets used by posts and pages. It is not only a file list; it is also where accessibility and descriptive quality are enforced through metadata.
When importing media, add metadata while context is still fresh. Alt text should describe meaning for accessibility. Captions should support reader understanding. Media tags should help later retrieval and reuse.
From the post editor you can insert already-imported media at the cursor position, drag supported image files—including HEIC and HEIF photos—onto an open post editor, or use the Gallery workflow to import several images at once. A dropped image is imported into the media library, linked to the current post, and inserted at the current cursor with a host-absolute `/media/...` URL that remains valid in rendered HTML. Multiple dropped files are processed in order with progress in the Tasks panel. Optional AI enrichment and media-metadata translation continue in the background; when the active AI endpoint is unavailable, the import still completes and RuDS reports that enrichment was skipped.
The Gallery workflow imports its selected images into the media library, links them to the current post, runs the same optional AI enrichment for titles and alt text, and inserts the gallery block into the post.
### Key takeaways
- Media management includes metadata quality, not only file import.
- Add alt text and captions during import, not as a postponed task.
- Drag images onto an open post editor for a direct import, link, and cursor insertion.
- Commit content and related media in the same change when possible.
[↑ Back to In this article](#in-this-article)
---
## Working with translations
RuDS supports translating both posts and media metadata into multiple languages. Translations are stored separately from canonical content so localized variants do not drift into unrelated records.
### Post translations
Each post has a canonical language and can have translations for additional languages. Translations keep their own title, excerpt, and content, while canonical metadata such as category, tags, slug, and publish state stays centralized.
The post editor shows the current language, existing translations, and missing languages. Posts marked Do Not Translate are excluded from automatic translation and from alternate language trees during site generation.
Published translation body content follows the same filesystem rule as published posts: the body lives in the file, not in the database.
### Media translations
Media items can have translated title, alt text, and caption values per language. The binary asset stays shared; only descriptive text varies by language.
### Automatic translation cascade
When blog languages are configured, RuDS can fill missing translations for posts and linked media. Automatic translation respects airplane mode and the configured AI runtime. If an automatic action cannot run in the current AI mode, RuDS reports that through the UI instead of silently inventing a result.
### Validation report
Use **Blog → Validate Translations** to open a completeness report across all configured languages. It lists missing and outdated translations for posts and media so you can close gaps deliberately instead of discovering them after generation.
### Key takeaways
- Post translations store title, excerpt, and content separately from the canonical post.
- Media translations store translated descriptive text while the asset stays shared.
- Automatic translation keeps posts and linked media aligned across configured languages.
- Do Not Translate excludes content from multi-language workflows.
- The translation validation report shows missing languages before they become visible gaps.
[↑ Back to In this article](#in-this-article)
---
## Finding content and duplicates
The sidebar search narrows the current list as you type. Posts, pages, and media support full-text search with language-aware stemming, and the filter panel adds calendar, tag, and category filters that combine with the text query.
When semantic similarity is enabled in the project settings, sidebar search also ranks results by meaning, not only by matching words. The same capability powers related-post ranking in the links panel and tag suggestions in the post editor.
Use **Blog → Find Duplicate Posts** to review pairs of posts with very similar or identical content. The review view pages through candidate pairs, lets you open both posts, and lets you dismiss a pair once you have decided it is intentional. Dismissals are remembered.
### Key takeaways
- Sidebar search combines full-text matching with calendar, tag, and category filters.
- Semantic similarity, when enabled, improves search, related posts, and tag suggestions.
- The duplicate review helps consolidate overlapping content deliberately.
[↑ Back to In this article](#in-this-article)
---
## Previewing your site
Preview renders through the same templates, routes, and asset files as site generation, so navigation and host-absolute links behave like the generated site. The preview server runs only on your machine at `127.0.0.1:4123` and is never exposed to the network.
Switch a post editor from **Markdown** to **Preview** to render that post at its normal generated-site URL inside RuDS. Use **File → Open in Browser** for the complete navigable blog: with a post editor active it starts on that post, and otherwise it starts on the home page. **Blog → Preview Post** also opens the active post in the default browser.
The preview includes draft and published posts on normal site routes. Draft content comes from the database, including draft translations; published content comes from the Markdown files that generation will use. Media, styles, scripts, images, and Pagefind files are served from their project filesystem locations, with packaged defaults used only when a project asset is absent.
### Key takeaways
- Preview and generation share one renderer and URL structure; preview overlays draft database content.
- The preview server is local-only.
- Browser and embedded previews remain navigable across normal site routes.
[↑ Back to In this article](#in-this-article)
---
## Editing site navigation
The site menu is stored as an OPML file in your project (`meta/menu.opml`) and edited visually in the Menu Editor (**Blog → Edit Menu**). The Home entry always stays first; below it you can add page links, submenus, and category entries, including categories that do not exist yet.
Reorder entries by dragging or with the equivalent keyboard controls, indent or unindent to build submenus, and save when the structure is right. The saved menu feeds the same renderer used by preview and generation, so the navigation you build is exactly what appears on the published site.
### Key takeaways
- The menu is a filesystem artifact (`meta/menu.opml`) and therefore Git-reviewable.
- Home is protected and always first.
- Preview, generation, and the editor all use the same menu structure.
[↑ Back to In this article](#in-this-article)
---
## Using macros
Macros let you insert dynamic content blocks directly inside Markdown by using `[[macro_name ...]]` syntax. RuDS expands these macros during preview and generated output using local assets only.
Built-in macros include YouTube, Vimeo, gallery, photo archive, and tag cloud helpers. Use them when you want reusable rich blocks without dropping into raw HTML.
### Key takeaways
- Macros are inserted directly in Markdown and expanded during preview and publishing.
- Use macro parameters to control behavior without leaving the editor.
- Built-in macros remain the first choice for common embedded content blocks.
[↑ Back to In this article](#in-this-article)
---
## Using scripting
Scripts in RuDS are Lua files stored in your project's `scripts/` directory. Published scripts are written as `.lua` files with frontmatter metadata, so they stay portable and Git-reviewable.
Each script has a Kind (`macro`, `transform`, or `utility`) and an Entrypoint. Utility and transform scripts typically default to `main`. Macro scripts default to `render`.
### Transform scripts
Transform scripts run during blogmark import to normalize or enrich incoming post data before the post is created. The entrypoint receives a post table and can optionally receive a context table.
```lua
function main(post, context)
local title = (post.title or ""):gsub("^%s+", ""):gsub("%s+$", "")
if title ~= "" and not title:match("^%[Clipped%]") then
post.title = "[Clipped] " .. title
end
post.categories = { "Inbox", "Research" }
return post
end
```
`context.source` identifies the import source. `context.url` contains the original bookmarked URL when that information exists.
### Macro scripts
Macro scripts let you create custom `[[macro_name ...]]` blocks that expand during preview and generation. The entrypoint receives a context table and the current post table.
```lua
function render(context, post)
local params = context.params or {}
local title = (post and post.title) or "Unknown"
local label = params.label or ""
return {
html = "<p>" .. title .. ": " .. label .. "</p>"
}
end
```
Built-in macros take priority over custom Lua macros that reuse the same slug.
### API access
Lua scripts can call the application API through `bds`. The in-app API tab is rendered from the live Lua capability map, and [API.md](API.md) is generated from the same source.
```lua
local result = bds.posts.get("post-id")
```
### Key takeaways
- Scripts in RuDS are Lua files, not Python files.
- Published scripts are stored as `.lua` files with frontmatter metadata.
- `main` is the usual entrypoint for utility and transform scripts; `render` is the usual entrypoint for macros.
- The scripting API is documented with Lua examples and kept in sync with the live runtime.
[↑ Back to In this article](#in-this-article)
---
## Using the AI assistant
The AI assistant is integrated into RuDS to help with editorial tasks such as search, analysis, metadata suggestions, translation, and structured content inspection.
The assistant works on your project data. Depending on configuration, requests can run against the configured online endpoint or the airplane-mode endpoint. Automatic AI actions remain gated by airplane mode rules in the app, and RuDS surfaces status through toasts and the Output area instead of silently bypassing that policy.
The assistant can present results as text, tables, cards, charts, metrics, lists, forms, and tabbed views. Ask plainly for the result you need, or request a specific presentation when that helps your workflow.
### Key takeaways
- The assistant works with your project content and metadata.
- AI configuration can be online or airplane-mode based, depending on your setup.
- Automatic AI actions respect airplane mode and report availability through the UI.
- Ask for a table, chart, list, or form when a specific shape is useful.
[↑ Back to In this article](#in-this-article)
---
## Organizing with tags
Tags are your precision taxonomy tool. Over time, even well-managed projects accumulate near-duplicate tags, naming inconsistencies, and labels that no longer help readers or editors. Use the Tags area to keep taxonomy useful: the tag cloud shows usage, the manage section covers create, rename, color, and template assignment, and the merge section consolidates overlapping tags.
When semantic similarity is enabled, the post editor suggests existing tags that fit the content you are writing, which helps keep the taxonomy consistent instead of growing new variants.
After significant taxonomy cleanup, create a focused commit that captures the change clearly.
### Key takeaways
- Tags improve discovery only if naming stays consistent.
- Merge and rename operations should be deliberate and reviewed.
- Commit taxonomy changes in focused snapshots.
[↑ Back to In this article](#in-this-article)
---
## Using blogmarks
Blogmarks provide a quick way to save links from the browser directly into RuDS as new posts. Generate the bookmarklet from Settings, add it to your browser bar, and click it when you want to capture a page into the current project.
Transform scripts can normalize incoming blogmark posts before creation. Use them for title cleanup, default tags, or source-specific formatting.
### Key takeaways
- Blogmarks turn the browser into a one-click content capture tool.
- Generate the bookmarklet from Settings and add it to your browser bar.
- Use transform scripts to enrich incoming posts automatically.
[↑ Back to In this article](#in-this-article)
---
## Importing from WordPress (WXR)
The Import section supports structured migration from WordPress exports. Treat import as a staged workflow: analyze first, adjust mappings, then execute. For larger sites, iterative passes are usually safer than a single rigid import.
### Key takeaways
- Treat WXR import as analyze, adjust, execute.
- Iterative passes are safer than one large import.
- Validate representative output before committing migrated content.
[↑ Back to In this article](#in-this-article)
---
## Using Git (Source Control)
Source Control in RuDS is the foundation for reliable recovery and collaboration. Publishing marks local editorial state, but Git commits provide durable history.
In a normal cycle, synchronize first, complete editorial changes, publish when ready, commit with a specific message, then push when you want to share the result.
### Key takeaways
- Git provides recoverable history; publishing alone does not.
- A stable rhythm is: sync, edit, publish, commit, push.
- Specific commit messages improve teamwork and recovery.
[↑ Back to In this article](#in-this-article)
---
## Configuring settings
Settings define how the project behaves. Project settings control identity, paths, public URL context, and render languages. Editor settings shape day-to-day working defaults. AI settings are optional and should enhance, not define, your editorial workflow.
Maintenance actions such as rebuilds and diff scans are repair tools for specific situations, not part of routine editing.
### Key takeaways
- Settings affect long-term consistency across the project.
- Optional integrations should not replace the core workflow.
- Rebuild actions are corrective tools, not daily habits.
[↑ Back to In this article](#in-this-article)
---
## Checking and repairing metadata
Over time, metadata stored in the database and metadata stored on disk can drift apart, especially after external edits, merges, or file operations. The Metadata Diff tool detects these inconsistencies and lets you repair them without rebuilding everything.
The scan covers posts and their translations, media and their translated sidecars, scripts, templates, project metadata, and embedding state. Results are grouped by entity type, and field pills let you focus on one kind of difference at a time.
Use DB to File when the database is correct. Use File to DB when the filesystem is correct.
### Key takeaways
- Metadata Diff compares database records against files on disk.
- Field pills help you bulk-repair one difference type at a time.
- Use it after external changes, not as part of routine editing.
[↑ Back to In this article](#in-this-article)
---
## Managing templates
Templates control the Liquid layout used when RuDS generates HTML pages. Template kinds determine where they are used: `post`, `list`, `not-found`, and `partial`.
Templates are stored as files with frontmatter metadata in the project data directory, so they are portable and Git-reviewable.
### Key takeaways
- Templates define the generated HTML layout.
- Four template kinds cover page, list, not-found, and reusable partial rendering.
- Templates are filesystem-backed and Git-friendly.
[↑ Back to In this article](#in-this-article)
---
## Generating and publishing
Publishing in RuDS is a staged process: publish content locally, generate or validate-and-apply site changes, commit the result, then deploy when ready.
Full generation builds the entire static site and skips files whose tracked content hash is unchanged. Use **Blog → Force Render Site** (Command/Ctrl+Shift+R) when every generated file must be rewritten regardless of its stored hash. Site validation detects missing, extra, and updated routes so RuDS can re-render only what changed. This is the practical incremental workflow for most daily editorial changes.
When blog languages are configured, generation produces language-aware route trees, per-language feeds, and alternate language metadata.
### Key takeaways
- Full generation produces the complete site.
- Force Render Site rewrites the complete site and search index without relying on stored hashes.
- Validate and Apply is the efficient daily workflow for incremental publishing.
- Public Base URL must be set before generation.
- Commit generated output before deploying for recoverability.
[↑ Back to In this article](#in-this-article)
---
## Typical editorial workflows
Short link posts benefit from a lightweight workflow: create, add concise context, classify, preview once, publish, commit. Long-form articles benefit from a fuller cycle: draft thoroughly, add media, review metadata, preview carefully, publish, commit content and media together.
Across both patterns, the safety baseline stays the same: Draft -> Publish -> Commit.
### Key takeaways
- Use a lightweight workflow for short notes and links.
- Use a fuller workflow for long-form content with media.
- Keep the same safety baseline in both cases.
[↑ Back to In this article](#in-this-article)
---
## Working fully offline
RuDS is designed so core editorial work can continue without network access. You can create and revise content, manage metadata, preview locally, and publish within local project state while offline.
When AI is involved, airplane mode determines which automatic actions are allowed and which endpoint class is used. Keep local commits frequent even when you are not pushing to a remote.
### Key takeaways
- Core editing and publishing workflows work offline.
- Local commits still matter when no remote is available.
- Reconnect and synchronize in a controlled order.
[↑ Back to In this article](#in-this-article)
---
## Connecting external AI agents (MCP)
RuDS can act as an MCP (Model Context Protocol) server so external AI tools such as Claude Code or GitHub Copilot can read your project and propose changes. Enable it under **Settings → MCP**, which also shows the endpoint status and offers guarded one-click configuration for supported agents.
Reads are direct: agents can browse posts, media, tags, categories, and statistics, and search your content. Writes never happen silently — every write an agent requests becomes a pending proposal that you review and accept or reject in **Settings → MCP**. Only an accepted proposal is applied, exactly once, through the same engines the editors use.
The full list of resources and tools is documented in **Help → MCP Server Documentation**.
### Key takeaways
- MCP is opt-in and configured under Settings → MCP.
- Agents read directly but can only write through proposals you approve.
- See Help → MCP Server Documentation for the complete surface.
[↑ Back to In this article](#in-this-article)
---
## Using the command line
RuDS ships a command-line interface for automation: rebuilds, rendering, publishing, Git sync, post and media creation, utility scripts, and more. Install the `bds-cli` launcher from **Settings → Data**; it runs the same engines and the same database as the desktop app, and the app picks up CLI changes automatically.
The complete command reference — including how to start the headless server and the terminal UI — is available in **Help → CLI Documentation**.
### Key takeaways
- The CLI shares engines and data with the desktop app; there is no separate state.
- Install it from Settings → Data.
- See Help → CLI Documentation for the full reference.
[↑ Back to In this article](#in-this-article)
---
## Running a headless server and the terminal UI
RuDS can run without a window on a server (for example a Linux VPS) and be used from several places at once. Start it with `bds-cli server`, or launch the desktop binary with `BDS_MODE=server` — both run the same headless mode. All clients connect through one SSH port (default 2222); nothing else is exposed to the network.
Access is controlled by SSH keys, not passwords. On first start the server creates an `ssh/` folder next to its database (for example `~/Library/Application Support/bds/ssh/` on macOS) containing the host key and an empty `authorized_keys` file. Paste the public keys of every machine that may connect into `authorized_keys`, one per line — the same file format OpenSSH uses.
To work in a terminal, connect with plain `ssh -p 2222 user@server`: the terminal UI opens directly. You can browse posts, media, templates, scripts, and tags, create and edit posts, publish, preview images, manage settings and Git, and run the one-shot AI actions. The status line at the bottom always shows the available keys. The same terminal UI also runs locally via `bds-cli tui` — see **Help → CLI Documentation** for the complete reference.
To work in the desktop app against a remote server, use **File → Connect to Server…** and enter `user@host` (or `user@host:port`). The app connects with the SSH key from its own local `ssh/` folder and shows the remote workspace in the window; **File → Disconnect from Server** returns to the local workspace.
Everything you see stays synchronized: when a script, another client, or a pipeline changes content on the server, every connected terminal and window updates. The interface language is a server-side setting — changing it in any client changes it for all of them.
### Key takeaways
- `bds-cli server` (or `BDS_MODE=server`) runs the same app headless; only the SSH port is exposed.
- Access is public-key only: manage `authorized_keys` in the server's `ssh/` folder.
- `ssh` gives the terminal UI; **File → Connect to Server…** gives the desktop app; both use the same keys.
- All connected clients stay synchronized and share one interface language.
[↑ Back to In this article](#in-this-article)
---
## Troubleshooting and recovery
If content looks correct locally but is missing for collaborators, the usual cause is that changes were published but not committed and pushed. Check repository status, create a commit, then push to the expected remote.
If content lists or references become inconsistent after manual file changes, start with Metadata Diff. If broader inconsistency remains, use rebuild tools to realign database and filesystem state.
### Key takeaways
- Most missing remote content issues are commit or push gaps.
- Metadata Diff is the first repair tool after external file changes.
- Frequent meaningful commits are the strongest safety net.
[↑ Back to In this article](#in-this-article)
---
## Team conventions
Shared conventions reduce ambiguity and merge friction. Teams should agree on category definitions, tag naming rules, publish-readiness criteria, and commit message patterns.
A practical minimum rule is simple: any content considered published should be committed promptly.
### Key takeaways
- Explicit conventions improve speed and reduce avoidable conflict.
- Start with a small rule set and enforce it consistently.
- Minimum standard: published content should be committed promptly.
[↑ Back to In this article](#in-this-article)

99
MCP.md Normal file
View File

@@ -0,0 +1,99 @@
# RuDS MCP Server
## In this article
- [What the MCP server is](#what-the-mcp-server-is)
- [Enabling and connecting](#enabling-and-connecting)
- [Security model](#security-model)
- [Resources](#resources)
- [Tools](#tools)
- [The proposal workflow](#the-proposal-workflow)
- [Configuring coding agents](#configuring-coding-agents)
## What the MCP server is
RuDS exposes your active project to external AI tools through the Model Context Protocol (MCP). An agent such as Claude Code or GitHub Copilot can browse posts, media, tags, categories, and statistics, run searches, and draft changes — while every write stays under your explicit control in the desktop app.
Two transports serve the same surface:
- **stdio**: the packaged `bds-mcp` executable speaks MCP over stdin/stdout. This is what coding agents launch directly.
- **HTTP**: a stateless localhost-only endpoint at `http://127.0.0.1:4124/mcp` (POST), started by the desktop app when enabled in **Settings → MCP**, and by the headless server for its loopback clients.
Both operate on the shared application database and the active project.
## Enabling and connecting
Open **Settings → MCP** in the desktop app to:
- enable or disable the HTTP endpoint and see its status and address,
- review pending write proposals (see below),
- install or remove ready-made agent configuration for supported tools.
The stdio binary needs no enablement; running it is the connection.
## Security model
- The HTTP endpoint binds to `127.0.0.1` only and validates the `Host` and `Origin` headers against localhost, so browsers cannot be tricked into calling it cross-origin. Only `POST` and `OPTIONS` are accepted.
- Reads are direct and read-only. Writes never take effect on their own: every write tool creates an inert, persisted proposal that a human must approve in the desktop app.
- Agent configuration written by RuDS never contains secrets.
## Resources
Resources use the `bds://` scheme and are read-only.
| Resource | Content |
|---|---|
| `bds://project` | Active project identity and metadata |
| `bds://posts` | Blog posts (paginated via `bds://posts{?cursor}`) |
| `bds://media` | Media items (paginated via `bds://media{?cursor}`) |
| `bds://tags` | All tags |
| `bds://categories` | All categories |
| `bds://stats` | Blog statistics |
| `bds://posts/{id}/media` | Media linked to one post |
| `bds://media/{id}/image` | The media image itself |
Post detail includes full content and metadata plus backlinks and outgoing links; published bodies are read from the canonical files.
## Tools
Read and search tools return results directly:
| Tool | Purpose |
|---|---|
| `check_term` | Check whether a term is a category, a tag, or both, with post counts |
| `search_posts` | Full-text and filtered post search with pagination, backlinks, and outgoing links |
| `count_posts` | Count filtered posts grouped by year, month, tag, category, or status |
| `read_post_by_slug` | Read full post content and metadata, optionally in a translated language |
| `get_post_translations` | List every translation for a post |
| `get_media_translations` | List every translated metadata record for a media item |
Write tools create proposals instead of writing:
| Tool | Proposes |
|---|---|
| `draft_post` | A new post (title, content, excerpt, tags, categories, author, language) |
| `propose_post_metadata` | Metadata changes to an existing post |
| `propose_media_metadata` | Title, alt text, caption, or tag changes for a media item |
| `upsert_media_translation` | Translated media metadata for one language |
| `propose_script` | A Lua script (validated before proposing) |
| `propose_template` | A Liquid template (validated before proposing) |
## The proposal workflow
1. An agent calls a write tool. RuDS validates the payload and stores it as a pending proposal; nothing else happens.
2. The proposal appears under **Settings → MCP** in the desktop app with its full payload.
3. You accept or reject it. Acceptance applies the change exactly once through the same engines the editors use — files, search indexes, embeddings, and domain events all behave like a normal edit. Rejection discards it.
4. Pending proposals expire automatically after 30 minutes. Concurrent resolution is safe: a proposal can only be applied once, and the agent can query the outcome.
This keeps agents useful for drafting and bulk suggestions while a human stays the only path to persistent change.
## Configuring coding agents
**Settings → MCP** can install a guarded configuration for:
| Agent | What is written |
|---|---|
| Claude Code | An entry in `~/.claude.json` launching the packaged `bds-mcp` stdio binary |
| GitHub Copilot | The equivalent MCP server entry in Copilot's configuration file |
Installation is opt-in per agent, shows what will be written, never stores secrets, and can be removed the same way.

View File

@@ -2,39 +2,85 @@
RuDS is a native Rust blogging desktop application and the successor to bDS2. It manages local projects from authoring through preview, static-site generation, integrity checks, and publishing while preserving the existing bDS filesystem and SQLite formats.
The desktop is a single-window application: closing its window persists UI state and exits RuDS. Background notifications remain unobtrusive and preserve keyboard focus while editing.
The project is under active development. Core blogging workflows are broadly available; remaining core work and optional extensions are tracked separately.
## Available Features
- Native Iced desktop workspace with localized menus, tabs, sidebars, dialogs, tasks, and embedded Wry previews.
- Post and translation authoring with draft/published lifecycle, metadata, tags, categories, links, and media.
- Media import, thumbnails, metadata translations, filters, validation, and post assignment.
- Template and Lua script management using a custom Ropey/Syntect/Cosmic Text editor.
- SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search.
- Markdown/Liquid rendering with native macros, multilingual routes, feeds, sitemap, Pagefind, and incremental site generation.
- Local preview in the app or system browser.
- Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations using online or local OpenAI-compatible endpoints with airplane-mode gating.
- Native Iced desktop workspace with localized menus, tabs, anchored editor popovers, automatically paged post/media sidebars with locale-aware post dates, calendar months, and relative-dated entity lists, row deletion, dialogs, embedded Wry previews, a live Pico CSS theme editor, and per-project restart restoration of the active activity, shell visibility, and open editor tabs. Its shared task UI keeps queued work off blocking workers, shows every active local or remote task plus recent history, and supports cooperative cancellation without oversubscribing background work.
- Post and translation authoring with change-aware draft/published/archive lifecycle, file-backed change discard, canonical draft reopening after manual translation edits, non-disruptive automatic translation, desktop archive/unarchive actions, in-place published-frontmatter updates, metadata, tags, categories, live link/backlink graphs, media, and batch gallery-image import.
- Media import including HEIC/HEIF decoding, q80 WebP thumbnails (plus q85 AI JPEGs), metadata translations, filters, validation, post assignment, and sequential drag-and-drop insertion into post editors.
- WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping.
- Post, Liquid template, and Lua script editing with dedicated syntax highlighting, explicit syntax-check feedback, change-aware published/draft lifecycle, normalized collision-safe template/script slug changes, reference-safe template renames, and publish-time enforcement of the bDS2 Liquid tag/filter/operator subset, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms, including airplane-gated Git sync.
- SQLite and filesystem persistence with byte-canonical bDS2 frontmatter, media sidecars, metadata JSON, and OPML menus; rebuild; bidirectional metadata diff/repair including project categories and publishing preferences; stale post-path cleanup on republish; bDS2-compatible checksums and NFD slug generation; and FTS5 search.
- Optional on-device multilingual semantic search and similar-post tag suggestions backed by a persistent USearch index, plus always-available partial-name tag autocomplete and duplicate-post review in the desktop workspace.
- Read-only in-app browsers in the Help menu for the bundled global `DOCUMENTATION.md`, the generated Lua API reference with public types and runnable examples, the [CLI/server/TUI documentation](CLI.md), and the [MCP server documentation](MCP.md), with safe GFM rendering and confirmed external links.
- A localized Tags workspace manages tags and category settings; its category table includes the main language and every configured translation, whose titles are used by matching category archives and menu entries.
- A localized OPML menu editor manages pages, submenus, and category archives with protected Home ordering, keyboard-accessible tree controls, drag-and-drop, and bDS2-compatible persistence.
- Project-scoped typed domain events synchronize desktop views with shared-engine and future CLI mutations; persisted CLI notifications are consumed once, and the selected UI language is shared through settings.
- Headless `bds-cli` automation for rebuild/repair/render, publishing and Git sync, post/media/gallery creation, effective shared settings with secret-presence redaction, projects, utility Lua tasks, JSON I/O, airplane-mode AI routing, and guarded launcher installation from Settings → Data or `bds-cli install`.
- Local MCP automation over stdio or a localhost-only stateless HTTP endpoint, with project resources, read/search/count tools, uniquely identified inert write proposals, clean duplicate-pending rejection, explicit desktop approval, and opt-in Claude Code/Copilot configuration.
- A fully localized Ratatui terminal workspace, available locally through `bds-cli tui`/`BDS_MODE=tui` and remotely through authenticated SSH shell sessions, with shared post/template/script editing and publishing, project/search/command overlays, settings, tags, Git, reports, task progress, live multi-client locale updates, and airplane-mode AI gating.
- `bds-cli server` hosting the shared application engines over a loopback-by-default, public-key-only SSH service, with restrictive private key material, live authorization updates, terminal-session transport, CLI-change synchronization, ordered domain/task events, and native desktop remote-project selection.
- bDS2-compatible Markdown/Liquid rendering with built-in macros rendered from bundled Liquid templates in isolated scopes (customizable with `macros/*` partial-template slugs), category-controlled list title visibility, descriptive category archive titles, canonical multilingual and flat page routes for every configured blog language, recursive menus, calendar archives, feeds, a root hreflang sitemap, Pagefind, shared cached multicore full-site rendering, change-aware and forced full renders from the native Blog menu/CLI/TUI, and fast route/mtime-based incremental validation whose targeted repair refreshes affected aggregate pages through cancellable section task groups with bDS2-style per-URL progress.
- Navigable generated-route preview in the app or system browser, with draft database overlays and published filesystem content.
- Optional one-shot AI translation, description, analysis, taxonomy, and language-detection operations run in background tasks with editor-level waiting indicators, using provider-portable JSON-only requests through independent online and local OpenAI-compatible profiles. Each profile has secure credentials, persistently discovered chat/title/image model selections, explicit tool/vision overrides, chat testing, and restart-persistent status-bar airplane-mode routing.
- Persistent conversational AI with safe Markdown, streamed and cancellable responses, model/session/token tracking, bounded project-aware blog tools, and localized conversation management in the Chat workspace. Allowlisted render tools add persistent native cards, charts, forms, lists, metrics, mind maps, tables, and tabs without executing assistant-provided HTML or JavaScript.
- SSH-agent-based SCP or rsync publishing.
- Site, media, and translation validation plus Blogmark capture and Lua transforms.
- Integrated Git workflow for each blog project's current repository, with repository initialization, read-only origin discovery, Git LFS image tracking, status and diffs, branch/file history with bDS2-compatible sync-status colors, commits, cancellable fetch/pull/push, and post-pull filesystem reconciliation; network actions respect airplane mode.
- Site, media, and translation validation plus `ruds://new-post` Blogmark capture and Lua transforms; captures open directly in the post editor and defer automatic translation until an explicit manual save, which queues one task per still-missing language. Rendering tasks keep long generated URLs compact in progress messages. Publishing never starts automatic translation. bDS2 keeps its separate `bds2://` bookmarklet protocol.
RuDS uses no JavaScript application runtime and loads no CSS or JavaScript from CDNs. The preview is served by the Rust application and displayed by the operating-system webview.
The packaged Apple Silicon application requires macOS 26 or newer.
Packaged executables share native `bds-core` and `bds-server` dynamic libraries. ONNX Runtime is statically contained in `bds-core`; the package does not ship or download a separate ONNX library. The **Install CLI** action writes a small forwarding launcher to `~/.local/bin`, so the command continues to execute the packaged CLI beside the same runtime libraries instead of copying them.
Local macOS packages are ad-hoc signed without hardened runtime. A Developer ID and notarization remain optional release-channel steps for downloads that should pass Gatekeeper without a user override.
## Repository Map
- `crates/bds-core` — data, engines, rendering, AI, publishing, and Lua
- `crates/bds-editor` — reusable syntax-highlighting editor
- `crates/bds-ui` — desktop application and platform integration
- `crates/bds-cli`planned automation CLI
- `crates/bds-cli`headless automation CLI over the shared engines
- `crates/bds-mcp` — packaged stdio MCP transport over the shared MCP engine
- `crates/bds-server` — reusable headless host, SSH transport, remote protocol, and desktop client library
- `specs` — authoritative Allium behavior specifications
- `fixtures` — compatibility projects and generated-site fixtures
- `locales` — UI and native-menu translations
## Plans and References
## References
- [Core plan and current status](RUST_PLAN_CORE.md)
- [Extension plan and current status](RUST_PLAN_EXTENSION.md)
- [Specification index](SPECIFICATION_INDEX.md)
- [User guide](DOCUMENTATION.md)
- [CLI, headless server, and terminal UI](CLI.md)
- [MCP server](MCP.md)
- [UI style guide](docs/UI_STYLE_GUIDE.md)
- `../bDS2` — reference implementation when an Allium contract is ambiguous
Contributor workflow and project invariants are documented in [AGENTS.md](AGENTS.md).
## Development gates
Install the dependency audit tools once:
```sh
cargo install cargo-machete cargo-outdated
```
Before committing, verify dependency usage and freshness, then build and test the complete workspace:
```sh
cargo machete --with-metadata
cargo outdated --workspace --root-deps-only --exit-code 1
cargo build --workspace
cargo test --workspace
```
The full test suite starts loopback-only mock and preview servers, so localhost binding must be permitted.
## Headless server
Run `bds-cli server` for a dedicated headless process, or set `BDS_MODE=server` when launching `bds-ui`. The SSH listener defaults to `127.0.0.1:2222`; use `--bind`/`BDS_SSH_BIND` and `--port`/`BDS_SSH_PORT` to opt into another address. Startup prints the private `authorized_keys` path. The desktop creates its own private `id_ed25519.pub`; add that public key to the server file, then use **File → Connect to Server…** and select a remote project. Host keys are recorded on first connection and verified thereafter. Run `bds-cli tui` or set `BDS_MODE=tui` for the local terminal workspace; an authenticated SSH shell opens the same workspace against server-side data and locale. Press `:` for its command list and `:?` for help.

View File

@@ -1,155 +0,0 @@
# RuDS Core Plan
## Goal
RuDS is the native Rust replacement for bDS2. Core covers the complete everyday workflow: open an existing project, author content, preview and generate the site, publish it, and maintain database/filesystem integrity.
The behavioural contract is `specs/*.allium`. When a spec is ambiguous, `../bDS2` is the reference implementation. [RUST_PLAN_EXTENSION.md](RUST_PLAN_EXTENSION.md) contains optional and advanced surfaces.
Status in this document describes the current source code as of 2026-07-18. It deliberately does not track build runs, test runs, release gates, or implementation history.
## Non-Negotiable Constraints
- Preserve existing SQLite, Markdown/frontmatter, translation, media sidecar, template, menu, and generated-site formats.
- Published post bodies live in files, not the database.
- Use a native Rust desktop stack: Iced, muda, rfd, and Wry. No JavaScript application runtime or remote assets.
- Preview and generated output share the same Rust rendering pipeline.
- Python scripting is intentionally replaced by Lua; built-in macros remain native Rust.
- UI locale follows the operating system; rendered content language follows project settings.
- Online AI is optional and blocked in airplane mode; local models may remain available.
- Metadata writes, publishing, metadata diff, and rebuild must share one field mapping.
- User-visible functionality requires a UI entry point, and UI controls require working functionality.
- macOS commands must be wired through native menus and application lifecycle handling.
## Workspace
| Crate | Responsibility |
|---|---|
| `bds-core` | Models, SQLite, filesystem formats, engines, rendering, generation, AI, publishing, and Lua |
| `bds-editor` | Reusable Ropey/Syntect/Cosmic Text editor widget |
| `bds-ui` | Iced application, native menus/dialogs, platform lifecycle, and embedded preview |
| `bds-cli` | Extension-only headless automation surface; currently a stub |
## Current Core Status
### Project, persistence, and integrity — Mostly done
Available:
- Existing project discovery and opening, with the database stored in private OS application data.
- Diesel-backed SQLite schema and migrations.
- Posts, translations, media, media translations, post-media links, tags, templates, scripts, settings, menus, and publishing settings.
- Draft/published lifecycle with canonical content placement.
- YAML frontmatter, translation files, media sidecars, template/script files, and project metadata.
- FTS5 post and media search with language-aware stemming.
- Database rebuild from filesystem and directional metadata diff/repair.
- Structured site, media, and translation validation.
- Unbounded unique slug allocation.
Open:
- Watch project files for external changes and reconcile open editors, preview, and database state.
- Make the script file extension project-configurable instead of hard-coding `.lua`.
### Native desktop shell — Done
Available:
- Iced workspace with activity bar, sidebar, tabs, status bar, task/output panel, toasts, and modal flows.
- Native muda menus, localized labels, accelerators, and state-dependent command enablement.
- Native file/folder dialogs and recent-project handling.
- macOS open-file and URL lifecycle plumbing.
- Localized UI separate from project content language.
Open:
- No known large core block. New menu commands must continue to be wired through both localization layers and the native intercept.
### Authoring and editor — Mostly done
Available:
- Dashboard and editors for posts, translations, media, tags, templates, scripts, and settings.
- Post create, edit, duplicate, publish, unpublish, discard, and delete flows.
- Media import, replacement, metadata editing, translations, thumbnails, filters, and post assignment.
- Template and Lua script creation, editing, validation, publication, and deletion.
- Rope-based editing with syntax highlighting, selection, clipboard, undo/redo, word/line/page movement, line numbers, soft wrapping, mouse selection, and committed IME input.
- Automatic translation flows with airplane-mode gating and media translation propagation.
Open:
- Replace the post-links placeholder with the specified functional linked-post view.
- Add the specified batch workflow for images linked from post content.
### Rendering, preview, and generation — Mostly done
Available:
- Shared Markdown and Liquid rendering for preview and generated output.
- Template lookup, routes, language variants, published snapshots, custom assigns, and URL rewriting.
- Native gallery, YouTube, Vimeo, photo archive, and tag cloud macros.
- User-authored Lua macro invocation during rendering.
- Localhost-only Axum preview server, draft routes, embedded Wry preview, and external-browser preview.
- Complete site generation with pages, archives, feeds, sitemap, static assets, changed-file tracking, parallel page rendering, and Pagefind output.
- OPML/menu document loading and normalized Home-first menu output.
Open:
- Represent generation as the specified group of section tasks followed by a final search-index task, rather than one coarse application task.
- Keep closing concrete output differences found against bDS2; approved normalization differences belong in this document when discovered.
### One-shot AI — Mostly done
Available:
- Configurable online and airplane-mode OpenAI-compatible endpoints.
- Secure API-key storage through the operating-system keychain.
- Model catalog discovery and model selection.
- Post translation, media translation, image alt text, post analysis, taxonomy analysis, and language detection.
- Explicit offline gating and user-visible errors.
Open:
- Persist actual input, output, cache-read, and cache-write token usage where the schema provides those fields.
Interactive chat, tools, agents, and MCP belong to the extension plan.
### Publishing — Mostly done
Available:
- SCP and rsync publishing through system commands.
- SSH-agent-only authentication with no password prompts.
- Separate HTML, thumbnail, and media targets.
- `.meta` exclusion, changed-file skipping for SCP, progress reporting, cancellation, and UI commands.
Open:
- Run the three upload targets as parallel tasks instead of processing them sequentially.
- Integrate external-file watching with publish/integrity workflows.
### Lua scripting — Partly done
Available:
- Sandboxed vendored Lua 5.4 runtime with cancellation and execution limits.
- Application log, progress, and toast functions.
- User macro and transform execution, including Blogmark transforms.
- Lua script persistence, rebuild, metadata diff, validation, and editor support.
Open:
- Expose the specified `bds.*` API for post, media, tag, project, and other script-visible data.
- Generate and bundle the Lua API reference, canonical type reference, and macro/transform/utility examples under `docs/scripting/`.
- Add a documentation-sync check tied to the exposed API.
## Remaining Core Blocks
1. Complete the Lua host API and scripting documentation.
2. Add filesystem change watching and reconciliation.
3. Parallelize publishing targets.
4. Finish the linked-post and linked-image authoring workflows.
5. Add generation section-task grouping and AI token accounting.
6. Remove the hard-coded Lua script extension.
Core is feature-complete when these blocks are closed and the implementation continues to satisfy the Allium and bDS2 compatibility contracts.

View File

@@ -1,132 +0,0 @@
# RuDS Extension Plan
## Goal
Extensions add migration, collaboration, automation, alternate clients, and advanced discovery on top of the core blogging workflow. They must reuse core engines and formats, remain reachable through real UI or automation surfaces, and preserve airplane-mode and localization rules.
Status describes the current source code as of 2026-07-18. Core one-shot AI, publishing, rendering, and integrity work is tracked in [RUST_PLAN_CORE.md](RUST_PLAN_CORE.md).
## Current Extension Status
### Git and richer validation — Partly done
Done:
- Site, media, translation, and metadata-diff engines and UI views.
Open:
- `GitEngine`, repository status, history, diff view, commit, fetch, pull, push, and LFS command integration.
- Replace the Git sidebar and Git log placeholders with working flows.
### WordPress import — Open
Open:
- WXR parsing, import analysis, recoverable execution, saved import definitions, and import UI.
- The existing media importer is core functionality and does not implement this workflow.
### Conversational AI and agent tools — Open
Open:
- Conversation persistence and chat UI.
- Streaming OpenAI-compatible responses and tool-call parsing.
- Tools over posts, media, templates, search, and other core engines.
- Agent integrations such as Claude Code and Copilot where required by the specs.
- Replace the current Chat placeholders with the working feature.
Core endpoint settings, offline gating, key storage, model discovery, and six one-shot operations are already implemented.
### Embeddings, semantic search, and duplicates — Open
Existing source contains schema placeholders only.
Open:
- Embedding generation and persistence.
- Vector index and semantic search.
- Duplicate detection, dismissed-pair handling, metadata integrity, and UI.
- Replace the Find Duplicates placeholder.
### Translation QA and documentation UX — Partly done
Done:
- Translation validation engine and report view.
Open:
- In-app project documentation browser.
- Browsable Lua API documentation and examples.
- Replace Documentation and API Documentation placeholders.
The generated Lua documentation itself is a core requirement.
### Menu editor and deep links — Partly done
Done:
- Menu file parsing/rendering, Home-first normalization, macOS URL plumbing, and Blogmark deep-link parsing.
Open:
- OPML/menu editor UI.
- Remaining application deep-link parity flows.
- Replace the Menu Editor placeholder.
### CLI, MCP, and domain events — Open
Open:
- Implement `bds-cli`; its current binary is only a stub.
- Commands from `cli.allium` and `cli_sync.allium` using the same project, database, engines, and settings as the desktop app.
- MCP tools/resources and proposal-based writes from `mcp.allium`.
- Domain event bus from `events.allium` for desktop, CLI, TUI, and future remote clients.
- Replace the MCP settings placeholder.
### Blogmark and transform pipeline — Mostly done
Done:
- Blogmark deep-link parsing, content capture, post import, transform selection, and Lua transform execution.
Open:
- Complete the dedicated user-facing capture/review workflow where the current implicit flow does not cover the bDS2 behavior.
- Close remaining transform-chain differences found against the specs and bDS2.
### Headless server — Open
Open:
- Desktop/server/TUI boot-mode selection.
- Headless engine host and SSH transport.
- Private host-key and authorized-key management.
- Desktop connection flow for remote projects.
### Terminal UI — Open
Open:
- Terminal renderer over shared application workflows.
- Sidebar/editor navigation, editing, publishing, and live domain-event updates.
- Remote operation through the headless server.
### A2UI surfaces — Open
Open:
- A2UI component renderer, surface state/data flow, and AI-assistant integration.
## Suggested Order
1. Git workflow and WordPress import.
2. CLI, MCP, and domain events.
3. Conversational AI and agent tools.
4. Embeddings and duplicate detection.
5. Documentation and menu UX.
6. Headless server and TUI.
7. A2UI after conversational AI exists.
The order may change when an extension directly unlocks a concrete user workflow; it must not create a parallel data model or bypass core engines.

View File

@@ -39,6 +39,7 @@ Diese Datei dient als Index zu allen Allium-Spezifikationen und Schema-Inventari
| Datei | Scope | Status | Beschreibung |
|-------|-------|--------|--------------|
| `git.allium` | Extension A | ✅ Existiert | Git-Operationen, LFS, Reconciliation |
| `import.allium` | Extension B | ✨ Neu | WordPress-WXR-Analyse, Konfliktprüfung und Importausführung |
| `mcp.allium` | Extension G | ✅ Existiert | MCP-Server (Tools, Resources) |
| `ai.allium` | Core/Extension C | ✅ Existiert | AI One-Shot Tasks und Chat |
| `embedding.allium` | Extension D | ✅ Existiert | Semantic Similarity (HNSW) |
@@ -203,13 +204,6 @@ Diese Datei dient als Index zu allen Allium-Spezifikationen und Schema-Inventari
- Korrupte Bilddateien
- Orphan Media (nicht verlinkt)
## Verfügbare Rust-Plan-Dokumente
| Datei | Beschreibung |
|-------|--------------|
| `RUST_PLAN_CORE.md` | Core-Umfang mit aktuellem Implementierungsstand und offenen Punkten |
| `RUST_PLAN_EXTENSION.md` | Extensions mit aktuellem Implementierungsstand und offenen Punkten |
## Spezifikations-Abdeckungs-Analyse
### Vollständig Abgedeckt ✅

View File

@@ -0,0 +1,30 @@
<div
class="macro-gallery gallery-cols-{{ columns }}"
data-post-id="{{ post_id | escape }}"
data-columns="{{ columns }}"
data-lightbox="true"
>
<div class="gallery-container gallery-lightbox">
{%- if items.size > 0 -%}
{%- for item in items -%}
<a
class="gallery-item"
href="{{ item.media_path | escape }}"
data-lightbox="{{ item.group_name | escape }}"
data-title="{{ item.title | escape }}"
>
<img
src="{{ item.media_path | escape }}"
alt="{{ item.alt | escape }}"
loading="lazy"
/>
</a>
{%- endfor -%}
{%- else -%}
<div class="gallery-empty">{{ no_items_label | escape }}</div>
{%- endif -%}
</div>
{%- if caption -%}
<figcaption class="gallery-caption">{{ caption | escape }}</figcaption>
{%- endif -%}
</div>

View File

@@ -0,0 +1,33 @@
<div class="{{ root_classes }}"{% for attr in data_attrs %} {{ attr.name }}="{{ attr.value | escape }}"{% endfor %}>
<div class="photo-archive-container">
{%- if months.size > 0 -%}
{%- for month in months -%}
<div class="photo-archive-month-wrapper">
<div class="photo-archive-month">
<div class="photo-archive-month-label">
<span>{{ month.label | escape }}</span>
</div>
<div class="photo-archive-gallery">
{%- for item in month.items -%}
<a
class="photo-archive-item"
href="{{ item.media_path | escape }}"
data-lightbox="{{ item.group_name | escape }}"
data-title="{{ item.title | escape }}"
>
<img
src="{{ item.media_path | escape }}"
alt="{{ item.alt | escape }}"
loading="lazy"
/>
</a>
{%- endfor -%}
</div>
</div>
</div>
{%- endfor -%}
{%- else -%}
<div class="photo-archive-empty">{{ no_items_label | escape }}</div>
{%- endif -%}
</div>
</div>

View File

@@ -0,0 +1,19 @@
<div
class="macro-tag-cloud"
data-tag-cloud="true"
data-orientation="{{ orientation }}"
data-color-distribution="quantile"
data-color-easing="0.7"
data-color-theme="pico"{%- if words_json %} data-tag-cloud-words="{{ words_json }}" data-width="{{ width }}" data-height="{{ height }}"{%- endif -%}
>
{%- if words_json -%}
<svg
class="tag-cloud-canvas"
viewBox="0 0 {{ width }} {{ height }}"
preserveAspectRatio="xMidYMid meet"
aria-label="{{ aria_label | escape }}"
></svg>
{%- else -%}
<div class="tag-cloud-empty">{{ no_items_label | escape }}</div>
{%- endif -%}
</div>

View File

@@ -0,0 +1,9 @@
<div class="macro-vimeo">
<iframe
src="https://player.vimeo.com/video/{{ id | escape }}"
title="{{ title | escape }}"
frameborder="0"
allow="autoplay; fullscreen; picture-in-picture"
allowfullscreen
></iframe>
</div>

View File

@@ -0,0 +1,9 @@
<div class="macro-youtube">
<iframe
src="https://www.youtube.com/embed/{{ id | escape }}?rel=0"
title="{{ title | escape }}"
frameborder="0"
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
allowfullscreen
></iframe>
</div>

View File

@@ -15,7 +15,7 @@
</svg>
</button>
<div class="blog-search-panel" data-blog-search-panel hidden>
<div id="blog-search" data-blog-search-root data-search-placeholder="{{ 'render.search.placeholder' | i18n: language }}"></div>
<div id="blog-search" data-blog-search-root data-search-placeholder="{{ 'render.search.placeholder' | i18n: language }}" data-search-no-results="{{ 'render.search.noResults' | i18n: language }}"></div>
</div>
</div>
</nav>
@@ -35,7 +35,7 @@
</svg>
</button>
<div class="blog-search-panel" data-blog-search-panel hidden>
<div id="blog-search" data-blog-search-root data-search-placeholder="{{ 'render.search.placeholder' | i18n: language }}"></div>
<div id="blog-search" data-blog-search-root data-search-placeholder="{{ 'render.search.placeholder' | i18n: language }}" data-search-no-results="{{ 'render.search.noResults' | i18n: language }}"></div>
</div>
</div>
{% endif %}

View File

@@ -6,5 +6,12 @@ license.workspace = true
[dependencies]
bds-core = { workspace = true }
bds-server = { workspace = true }
anyhow = { workspace = true }
tokio = { workspace = true }
clap = { workspace = true }
dirs = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
[dev-dependencies]
tempfile = "3"

1654
crates/bds-cli/src/lib.rs Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,3 +1,104 @@
fn main() {
println!("bds-cli: headless automation surface (not yet implemented)");
use std::io::Read as _;
use std::process::ExitCode;
use clap::Parser;
fn main() -> ExitCode {
let cli = match bds_cli::Cli::try_parse() {
Ok(cli) => cli,
Err(error) => {
let success = matches!(
error.kind(),
clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion
);
let _ = error.print();
return if success {
ExitCode::SUCCESS
} else {
ExitCode::from(1)
};
}
};
if let bds_cli::Command::Server(args) = &cli.command {
let data_root = args
.data_dir
.clone()
.unwrap_or_else(bds_core::util::application_data_dir);
let database_path = args
.database
.clone()
.unwrap_or_else(|| data_root.join("bds.db"));
let config = match bds_server::ServerConfig::from_environment(database_path, data_root) {
Ok(mut config) => {
if let Some(bind) = args.bind {
config.bind = bind;
}
if let Some(port) = args.port {
config.port = port;
}
config
}
Err(error) => {
eprintln!("Error: {error:#}");
return ExitCode::from(1);
}
};
return run_headless(bds_server::boot::BootMode::Server, config);
}
if matches!(cli.command, bds_cli::Command::Tui) {
let data_root = bds_core::util::application_data_dir();
let config = match bds_server::ServerConfig::from_environment(
bds_core::util::application_database_path(),
data_root,
) {
Ok(config) => config,
Err(error) => {
eprintln!("Error: {error:#}");
return ExitCode::from(1);
}
};
return run_headless(bds_server::boot::BootMode::Tui, config);
}
let json = cli.json;
let needs_stdin = match &cli.command {
bds_cli::Command::Post(args) => args.stdin,
bds_cli::Command::Gallery(args) => args.post.stdin,
_ => false,
};
let mut context = bds_cli::RunContext::system();
if needs_stdin && let Err(error) = std::io::stdin().read_to_string(&mut context.stdin) {
eprintln!("Error: could not read stdin: {error}");
return ExitCode::from(1);
}
match bds_cli::run(cli, context) {
Ok(output) => {
println!("{output}");
ExitCode::SUCCESS
}
Err(error) => {
if json {
eprintln!(
"{}",
serde_json::json!({"ok": false, "error": format!("{error:#}")})
);
} else {
eprintln!("Error: {error:#}");
}
ExitCode::from(1)
}
}
}
fn run_headless(mode: bds_server::boot::BootMode, config: bds_server::ServerConfig) -> ExitCode {
match bds_server::run_headless(mode, config) {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("Error: {error:#}");
ExitCode::from(1)
}
}
}

View File

@@ -0,0 +1,188 @@
use std::process::Command;
fn cli(home: &std::path::Path, args: &[&str]) -> std::process::Output {
Command::new(env!("CARGO_BIN_EXE_bds-cli"))
.env("HOME", home)
.args(args)
.output()
.unwrap()
}
#[test]
fn process_exit_codes_help_and_shared_state_roundtrip() {
let root = tempfile::tempdir().unwrap();
let home = root.path().join("home");
let project = root.path().join("project");
std::fs::create_dir_all(&home).unwrap();
std::fs::create_dir_all(&project).unwrap();
let help = cli(&home, &["--help"]);
assert!(help.status.success());
assert!(String::from_utf8_lossy(&help.stdout).contains("rebuild"));
let invalid = cli(&home, &["not-a-command"]);
assert_eq!(invalid.status.code(), Some(1));
assert!(!invalid.stderr.is_empty());
let added = cli(
&home,
&[
"project",
"add",
project.to_str().unwrap(),
"--name",
"Process Blog",
],
);
assert!(
added.status.success(),
"{}",
String::from_utf8_lossy(&added.stderr)
);
assert!(
cli(&home, &["project", "switch", "process-blog"])
.status
.success()
);
let created = Command::new(env!("CARGO_BIN_EXE_bds-cli"))
.env("HOME", &home)
.args(["--json", "post", "--stdin", "--no-translate"])
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.and_then(|mut child| {
use std::io::Write as _;
child
.stdin
.take()
.unwrap()
.write_all(br#"{"title":"Process post","content":"Body","language":"en"}"#)?;
child.wait_with_output()
})
.unwrap();
assert!(
created.status.success(),
"{}",
String::from_utf8_lossy(&created.stderr)
);
let value: serde_json::Value = serde_json::from_slice(&created.stdout).unwrap();
assert_eq!(value["ok"], true);
assert_eq!(value["data"]["slug"], "process-post");
let get_missing = cli(&home, &["config", "get", "missing"]);
assert_eq!(get_missing.status.code(), Some(1));
}
#[cfg(unix)]
#[test]
fn process_upload_push_and_pull_dispatch_successfully() {
use std::os::unix::fs::PermissionsExt as _;
let root = tempfile::tempdir().unwrap();
let home = root.path().join("home");
let project = root.path().join("project");
let remote = root.path().join("remote.git");
let peer = root.path().join("peer");
let fake_bin = root.path().join("bin");
for directory in [&home, &project, &fake_bin] {
std::fs::create_dir_all(directory).unwrap();
}
assert!(
cli(
&home,
&[
"project",
"add",
project.to_str().unwrap(),
"--name",
"External Blog",
],
)
.status
.success()
);
assert!(
cli(&home, &["project", "switch", "external-blog"])
.status
.success()
);
std::fs::write(
project.join("meta/publishing.json"),
r#"{"sshHost":"example.test","sshUser":"deploy","sshRemotePath":"/srv/blog","sshMode":"rsync"}"#,
)
.unwrap();
let rsync = fake_bin.join("rsync");
std::fs::write(&rsync, "#!/bin/sh\nexit 0\n").unwrap();
let mut permissions = std::fs::metadata(&rsync).unwrap().permissions();
permissions.set_mode(0o755);
std::fs::set_permissions(&rsync, permissions).unwrap();
let upload = Command::new(env!("CARGO_BIN_EXE_bds-cli"))
.env("HOME", &home)
.env("PATH", &fake_bin)
.env("SSH_AUTH_SOCK", root.path().join("agent.sock"))
.arg("upload")
.output()
.unwrap();
assert!(
upload.status.success(),
"{}",
String::from_utf8_lossy(&upload.stderr)
);
git(&project, &["init", "-b", "main"]);
git(&project, &["config", "user.email", "cli@example.test"]);
git(&project, &["config", "user.name", "CLI Test"]);
git(&project, &["add", "."]);
git(&project, &["commit", "-m", "Initial"]);
git(root.path(), &["init", "--bare", remote.to_str().unwrap()]);
git(
&project,
&["remote", "add", "origin", remote.to_str().unwrap()],
);
git(&project, &["push", "-u", "origin", "main"]);
git(&remote, &["symbolic-ref", "HEAD", "refs/heads/main"]);
let pushed = cli(&home, &["push"]);
assert!(
pushed.status.success(),
"{}",
String::from_utf8_lossy(&pushed.stderr)
);
git(
root.path(),
&["clone", remote.to_str().unwrap(), peer.to_str().unwrap()],
);
git(&peer, &["config", "user.email", "peer@example.test"]);
git(&peer, &["config", "user.name", "Peer Test"]);
std::fs::write(peer.join("remote-change.txt"), "change").unwrap();
git(&peer, &["add", "remote-change.txt"]);
git(&peer, &["commit", "-m", "Remote change"]);
git(&peer, &["push"]);
let pulled = cli(&home, &["pull"]);
assert!(
pulled.status.success(),
"{}",
String::from_utf8_lossy(&pulled.stderr)
);
assert!(project.join("remote-change.txt").is_file());
}
#[cfg(unix)]
fn git(cwd: &std::path::Path, args: &[&str]) {
let output = Command::new("git")
.current_dir(cwd)
.args(args)
.output()
.unwrap();
assert!(
output.status.success(),
"git {} failed:\n{}",
args.join(" "),
String::from_utf8_lossy(&output.stderr)
);
}

View File

@@ -4,6 +4,9 @@ edition.workspace = true
version.workspace = true
license.workspace = true
[lib]
crate-type = ["dylib"]
[dependencies]
uuid = { workspace = true }
serde = { workspace = true }
@@ -11,10 +14,13 @@ serde_json = { workspace = true }
serde_yaml = { workspace = true }
chrono = { workspace = true }
sha2 = { workspace = true }
deunicode = { workspace = true }
md5 = { workspace = true }
unicode-normalization = { workspace = true }
thiserror = { workspace = true }
walkdir = { workspace = true }
image = { workspace = true }
hpvcd = { workspace = true }
webp = { workspace = true }
rust-stemmers = { workspace = true }
sys-locale = { workspace = true }
fluent-bundle = { workspace = true }
@@ -23,6 +29,8 @@ pulldown-cmark = { workspace = true }
liquid = { workspace = true }
liquid-core = { workspace = true }
quick-xml = { workspace = true }
htmd = { workspace = true }
regex = { workspace = true }
rayon = { workspace = true }
pagefind = { workspace = true }
reqwest = { workspace = true }
@@ -34,6 +42,14 @@ diesel_migrations = { workspace = true }
libsqlite3-sys = { workspace = true }
mlua = { workspace = true }
url = { workspace = true }
base64 = { workspace = true }
dirs = { workspace = true }
fastembed = { workspace = true }
usearch = { workspace = true }
ort = { workspace = true }
[dev-dependencies]
tempfile = "3"
[package.metadata.cargo-machete]
ignored = ["libsqlite3-sys"]

View File

@@ -0,0 +1,12 @@
use std::fs;
use std::path::Path;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
let docs = root.join("docs/scripting");
let manifest = bds_core::scripting::api_manifest();
fs::write(docs.join("API_REFERENCE.md"), manifest.render_reference())?;
fs::write(docs.join("TYPES.md"), manifest.render_types())?;
fs::write(docs.join("completions.json"), manifest.render_completions())?;
Ok(())
}

View File

@@ -0,0 +1,3 @@
DROP INDEX IF EXISTS db_notifications_prune_idx;
DROP INDEX IF EXISTS db_notifications_unseen_cli_idx;
ALTER TABLE db_notifications DROP COLUMN project_id;

View File

@@ -0,0 +1,7 @@
ALTER TABLE db_notifications ADD COLUMN project_id TEXT;
CREATE INDEX db_notifications_unseen_cli_idx
ON db_notifications (from_cli, seen_at, created_at);
CREATE INDEX db_notifications_prune_idx
ON db_notifications (seen_at, created_at);

View File

@@ -0,0 +1 @@
DROP TABLE mcp_proposals;

View File

@@ -0,0 +1,24 @@
CREATE TABLE mcp_proposals (
id TEXT PRIMARY KEY NOT NULL,
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
kind TEXT NOT NULL CHECK (kind IN (
'draft_post',
'propose_script',
'propose_template',
'propose_media_translation',
'propose_media_metadata',
'propose_post_metadata'
)),
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN (
'pending', 'executing', 'accepted', 'rejected', 'expired'
)),
entity_id TEXT,
data TEXT NOT NULL,
result TEXT,
created_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL,
resolved_at BIGINT
);
CREATE INDEX mcp_proposals_project_status_idx
ON mcp_proposals(project_id, status, created_at);

View File

@@ -0,0 +1 @@
DROP TABLE ai_endpoint_models;

View File

@@ -0,0 +1,11 @@
CREATE TABLE ai_endpoint_models (
kind TEXT NOT NULL,
model_id TEXT NOT NULL,
label TEXT NOT NULL,
context_window INTEGER,
max_output_tokens INTEGER,
supports_tools INTEGER NOT NULL DEFAULT 0,
supports_vision INTEGER NOT NULL DEFAULT 0,
updated_at BIGINT NOT NULL,
PRIMARY KEY (kind, model_id)
);

View File

@@ -0,0 +1,38 @@
DROP INDEX mcp_proposals_entity_idx;
CREATE TABLE mcp_proposals_old (
id TEXT PRIMARY KEY NOT NULL,
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
kind TEXT NOT NULL CHECK (kind IN (
'draft_post',
'propose_script',
'propose_template',
'propose_media_translation',
'propose_media_metadata',
'propose_post_metadata'
)),
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN (
'pending', 'executing', 'accepted', 'rejected', 'expired'
)),
entity_id TEXT,
data TEXT NOT NULL,
result TEXT,
created_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL,
resolved_at BIGINT
);
INSERT INTO mcp_proposals_old (
id, project_id, kind, status, entity_id, data, result,
created_at, expires_at, resolved_at
)
SELECT
id, project_id, kind, status, entity_id, data, result,
created_at, expires_at, resolved_at
FROM mcp_proposals;
DROP TABLE mcp_proposals;
ALTER TABLE mcp_proposals_old RENAME TO mcp_proposals;
CREATE INDEX mcp_proposals_project_status_idx
ON mcp_proposals(project_id, status, created_at);

View File

@@ -0,0 +1,55 @@
CREATE TABLE mcp_proposals_new (
id TEXT PRIMARY KEY NOT NULL,
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
kind TEXT NOT NULL CHECK (kind IN (
'draft_post',
'propose_script',
'propose_template',
'propose_media_translation',
'propose_media_metadata',
'propose_post_metadata'
)),
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN (
'pending', 'executing', 'accepted', 'rejected', 'expired'
)),
entity_id TEXT NOT NULL,
data TEXT NOT NULL,
result TEXT,
created_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL,
resolved_at BIGINT
);
INSERT INTO mcp_proposals_new (
id, project_id, kind, status, entity_id, data, result,
created_at, expires_at, resolved_at
)
SELECT
id, project_id, kind, status, COALESCE(entity_id, id), data, result,
created_at, expires_at, resolved_at
FROM mcp_proposals;
DROP TABLE mcp_proposals;
ALTER TABLE mcp_proposals_new RENAME TO mcp_proposals;
-- Keep the newest record if a pre-migration database already contains an
-- invalid duplicate. NULL-backed proposals were assigned their own IDs above
-- and therefore remain distinct.
DELETE FROM mcp_proposals AS duplicate
WHERE EXISTS (
SELECT 1
FROM mcp_proposals AS keeper
WHERE keeper.kind = duplicate.kind
AND keeper.entity_id = duplicate.entity_id
AND keeper.status = duplicate.status
AND (
keeper.created_at > duplicate.created_at
OR (keeper.created_at = duplicate.created_at AND keeper.id > duplicate.id)
)
);
CREATE INDEX mcp_proposals_project_status_idx
ON mcp_proposals(project_id, status, created_at);
CREATE UNIQUE INDEX mcp_proposals_entity_idx
ON mcp_proposals(kind, entity_id, status);

View File

@@ -199,9 +199,7 @@ CREATE TABLE IF NOT EXISTS chat_messages (
content TEXT,
tool_call_id TEXT,
tool_calls TEXT,
created_at INTEGER NOT NULL,
cache_read_tokens INTEGER,
cache_write_tokens INTEGER
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS ai_providers (
@@ -260,9 +258,12 @@ CREATE TABLE IF NOT EXISTS embedding_keys (
post_id TEXT NOT NULL,
project_id TEXT NOT NULL,
content_hash TEXT NOT NULL,
vector TEXT NOT NULL
vector BLOB NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS embedding_keys_project_post_idx
ON embedding_keys(project_id, post_id);
CREATE TABLE IF NOT EXISTS dismissed_duplicate_pairs (
id TEXT NOT NULL PRIMARY KEY,
project_id TEXT NOT NULL REFERENCES projects(id),

View File

@@ -0,0 +1,3 @@
-- This file should undo anything in `up.sql`
ALTER TABLE chat_messages DROP COLUMN token_usage_output;
ALTER TABLE chat_messages DROP COLUMN token_usage_input;

View File

@@ -0,0 +1,3 @@
-- Your SQL goes here
ALTER TABLE chat_messages ADD COLUMN token_usage_input INTEGER;
ALTER TABLE chat_messages ADD COLUMN token_usage_output INTEGER;

View File

@@ -0,0 +1,15 @@
-- This file should undo anything in `up.sql`
CREATE TABLE embedding_keys_old (
label INTEGER NOT NULL PRIMARY KEY,
post_id TEXT NOT NULL,
project_id TEXT NOT NULL,
content_hash TEXT NOT NULL,
vector TEXT NOT NULL
);
INSERT INTO embedding_keys_old (label, post_id, project_id, content_hash, vector)
SELECT label, post_id, project_id, content_hash, CAST(vector AS TEXT)
FROM embedding_keys;
DROP TABLE embedding_keys;
ALTER TABLE embedding_keys_old RENAME TO embedding_keys;

View File

@@ -0,0 +1,18 @@
-- Your SQL goes here
CREATE TABLE embedding_keys_new (
label INTEGER NOT NULL PRIMARY KEY,
post_id TEXT NOT NULL,
project_id TEXT NOT NULL,
content_hash TEXT NOT NULL,
vector BLOB NOT NULL
);
INSERT INTO embedding_keys_new (label, post_id, project_id, content_hash, vector)
SELECT label, post_id, project_id, content_hash, CAST(vector AS BLOB)
FROM embedding_keys;
DROP TABLE embedding_keys;
ALTER TABLE embedding_keys_new RENAME TO embedding_keys;
CREATE UNIQUE INDEX embedding_keys_project_post_idx
ON embedding_keys(project_id, post_id);

View File

@@ -0,0 +1,3 @@
-- This file should undo anything in `up.sql`
ALTER TABLE chat_messages DROP COLUMN cache_write_tokens;
ALTER TABLE chat_messages DROP COLUMN cache_read_tokens;

View File

@@ -0,0 +1,3 @@
-- Your SQL goes here
ALTER TABLE chat_messages ADD COLUMN cache_read_tokens INTEGER;
ALTER TABLE chat_messages ADD COLUMN cache_write_tokens INTEGER;

View File

@@ -0,0 +1,2 @@
-- This file should undo anything in `up.sql`
ALTER TABLE chat_conversations DROP COLUMN surface_state;

View File

@@ -0,0 +1,2 @@
-- Your SQL goes here
ALTER TABLE chat_conversations ADD COLUMN surface_state TEXT;

View File

@@ -3,6 +3,7 @@ use std::path::Path;
use diesel::connection::SimpleConnection;
use diesel::prelude::*;
use diesel::sql_types::Text;
use crate::db::migrations;
@@ -17,6 +18,12 @@ pub enum DatabaseError {
/// Shared synchronous Diesel connection used by the engine query API.
pub struct DbConnection(RefCell<SqliteConnection>);
#[derive(QueryableByName)]
struct DatabasePathRow {
#[diesel(sql_type = Text)]
file: String,
}
impl DbConnection {
pub fn with<T>(
&self,
@@ -45,6 +52,40 @@ impl DbConnection {
.borrow_mut()
.batch_execute("ROLLBACK TO bds_operation; RELEASE bds_operation")
}
#[cfg(test)]
pub(crate) fn reject_tag_template_updates_for_test(&self) -> diesel::QueryResult<()> {
self.0.borrow_mut().batch_execute(
"CREATE TRIGGER reject_template_tag_cascade \
BEFORE UPDATE OF post_template_slug ON tags \
BEGIN SELECT RAISE(ABORT, 'reject cascade'); END",
)
}
#[cfg(test)]
pub(crate) fn insert_legacy_null_mcp_proposal_for_test(&self) -> diesel::QueryResult<()> {
self.0.borrow_mut().batch_execute(
"INSERT INTO mcp_proposals \
(id, project_id, kind, status, entity_id, data, created_at, expires_at) \
VALUES ('null-entity', 'p1', 'draft_post', 'pending', NULL, '{}', 3, 99)",
)
}
/// Filesystem database path for sibling surfaces that must open their own
/// short-lived connection (gallery workers, preview servers, Lua hosts).
pub fn database_path(&self) -> diesel::QueryResult<std::path::PathBuf> {
self.with(|conn| {
diesel::sql_query("SELECT file FROM pragma_database_list WHERE name = 'main'")
.get_result::<DatabasePathRow>(conn)
.and_then(|row| {
if row.file.is_empty() {
Err(diesel::result::Error::NotFound)
} else {
Ok(row.file.into())
}
})
})
}
}
/// Database wrapper managing a SQLite connection.
@@ -101,4 +142,16 @@ mod tests {
.unwrap();
assert_eq!(result, 1);
}
#[test]
fn reports_the_disk_database_path() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("project.sqlite3");
let db = Database::open(&path).unwrap();
assert_eq!(
db.conn().database_path().unwrap().canonicalize().unwrap(),
path.canonicalize().unwrap()
);
}
}

View File

@@ -15,10 +15,11 @@ mod tests {
use super::*;
use crate::db::Database;
use crate::db::schema::{
ai_catalog_meta, ai_model_modalities, ai_models, ai_providers, chat_conversations,
chat_messages, db_notifications, dismissed_duplicate_pairs, embedding_keys,
generated_file_hashes, import_definitions, media, media_translations, post_links,
post_media, post_translations, posts, projects, scripts, settings, tags, templates,
ai_catalog_meta, ai_endpoint_models, ai_model_modalities, ai_models, ai_providers,
chat_conversations, chat_messages, db_notifications, dismissed_duplicate_pairs,
embedding_keys, generated_file_hashes, import_definitions, mcp_proposals, media,
media_translations, post_links, post_media, post_translations, posts, projects, scripts,
settings, tags, templates,
};
use diesel::prelude::*;
use diesel_migrations::MigrationHarness;
@@ -35,7 +36,132 @@ mod tests {
let applied = db
.conn()
.with_migrations(|conn| conn.applied_migrations().unwrap().len());
assert_eq!(applied, 1);
assert_eq!(applied, 9);
}
#[test]
fn mcp_proposal_upgrade_normalizes_entities_and_enforces_uniqueness() {
use crate::db::queries::{mcp_proposal, project};
use crate::model::{McpProposal, ProposalKind, ProposalStatus};
let db = Database::open_in_memory().unwrap();
db.conn().with_migrations(|conn| {
for _ in 0..8 {
conn.run_next_migration(MIGRATIONS).unwrap();
}
});
project::insert_project(db.conn(), &project::make_test_project("p1", "blog")).unwrap();
let targeted = |id: &str, created_at| McpProposal {
id: id.into(),
project_id: "p1".into(),
kind: ProposalKind::ProposePostMetadata,
status: ProposalStatus::Pending,
entity_id: "post-1".into(),
data: "{}".into(),
result: None,
created_at,
expires_at: 99,
resolved_at: None,
};
mcp_proposal::insert_proposal(db.conn(), &targeted("older", 1)).unwrap();
mcp_proposal::insert_proposal(db.conn(), &targeted("newer", 2)).unwrap();
db.conn()
.insert_legacy_null_mcp_proposal_for_test()
.unwrap();
run_migrations(db.conn()).unwrap();
let proposals = mcp_proposal::list_proposals(db.conn(), "p1").unwrap();
assert_eq!(proposals.len(), 2);
assert!(proposals.iter().any(|proposal| proposal.id == "newer"));
assert_eq!(
proposals
.iter()
.find(|proposal| proposal.id == "null-entity")
.unwrap()
.entity_id,
"null-entity"
);
assert!(matches!(
mcp_proposal::insert_proposal(db.conn(), &targeted("duplicate", 4)),
Err(diesel::result::Error::DatabaseError(
diesel::result::DatabaseErrorKind::UniqueViolation,
_
))
));
}
#[test]
fn mcp_proposal_enum_constraints_round_trip_and_reject_unknown_values() {
use crate::db::queries::{mcp_proposal, project};
use crate::model::{McpProposal, ProposalKind, ProposalStatus};
let db = migrated_database();
project::insert_project(db.conn(), &project::make_test_project("p1", "blog")).unwrap();
let kinds = [
ProposalKind::DraftPost,
ProposalKind::ProposeScript,
ProposalKind::ProposeTemplate,
ProposalKind::ProposeMediaTranslation,
ProposalKind::ProposeMediaMetadata,
ProposalKind::ProposePostMetadata,
];
let statuses = [
ProposalStatus::Pending,
ProposalStatus::Executing,
ProposalStatus::Accepted,
ProposalStatus::Rejected,
ProposalStatus::Expired,
];
for (index, kind) in kinds.into_iter().enumerate() {
let status = statuses[index % statuses.len()];
let id = format!("proposal-{index}");
let proposal = McpProposal {
id: id.clone(),
project_id: "p1".into(),
kind,
status,
entity_id: id.clone(),
data: "{}".into(),
result: None,
created_at: 1,
expires_at: 2,
resolved_at: None,
};
mcp_proposal::insert_proposal(db.conn(), &proposal).unwrap();
let stored = mcp_proposal::get_proposal(db.conn(), &id).unwrap();
assert_eq!(stored.kind, kind);
assert_eq!(stored.status, status);
}
for (id, kind, status) in [
("invalid-kind", "unknown", "pending"),
("invalid-status", "draft_post", "unknown"),
] {
let result = db.conn().with(|connection| {
diesel::insert_into(mcp_proposals::table)
.values((
mcp_proposals::id.eq(id),
mcp_proposals::project_id.eq("p1"),
mcp_proposals::kind.eq(kind),
mcp_proposals::status.eq(status),
mcp_proposals::entity_id.eq(id),
mcp_proposals::data.eq("{}"),
mcp_proposals::created_at.eq(1_i64),
mcp_proposals::expires_at.eq(2_i64),
))
.execute(connection)
});
assert!(matches!(
result,
Err(diesel::result::Error::DatabaseError(
diesel::result::DatabaseErrorKind::CheckViolation,
_
))
));
}
}
#[test]
@@ -72,9 +198,11 @@ mod tests {
ai_models::table,
ai_model_modalities::table,
ai_catalog_meta::table,
ai_endpoint_models::table,
embedding_keys::table,
dismissed_duplicate_pairs::table,
import_definitions::table,
mcp_proposals::table,
db_notifications::table,
);
}
@@ -82,7 +210,7 @@ mod tests {
#[test]
fn migrations_expose_current_ai_catalog_and_usage_columns() {
let db = migrated_database();
let (provider_ref, model_ref, cache_tokens) = db
let (provider_ref, model_ref, usage_tokens) = db
.conn()
.with(|conn| {
diesel::insert_into(ai_providers::table)
@@ -124,6 +252,8 @@ mod tests {
chat_messages::conversation_id.eq("conversation"),
chat_messages::role.eq("assistant"),
chat_messages::created_at.eq(1_i64),
chat_messages::token_usage_input.eq(Some(56)),
chat_messages::token_usage_output.eq(Some(78)),
chat_messages::cache_read_tokens.eq(Some(12)),
chat_messages::cache_write_tokens.eq(Some(34)),
))
@@ -138,16 +268,102 @@ mod tests {
.first::<Option<String>>(conn)?,
chat_messages::table
.select((
chat_messages::token_usage_input,
chat_messages::token_usage_output,
chat_messages::cache_read_tokens,
chat_messages::cache_write_tokens,
))
.first::<(Option<i32>, Option<i32>)>(conn)?,
.first::<(Option<i32>, Option<i32>, Option<i32>, Option<i32>)>(conn)?,
))
})
.unwrap();
assert_eq!(provider_ref.as_deref(), Some("provider-package"));
assert_eq!(model_ref.as_deref(), Some("model-package"));
assert_eq!(cache_tokens, (Some(12), Some(34)));
assert_eq!(usage_tokens, (Some(56), Some(78), Some(12), Some(34)));
}
#[test]
fn existing_chat_schema_is_upgraded_with_cache_token_columns() {
let db = Database::open_in_memory().unwrap();
db.conn().with_migrations(|conn| {
for _ in 0..4 {
conn.run_next_migration(MIGRATIONS).unwrap();
}
});
db.conn()
.with(|conn| {
diesel::insert_into(chat_conversations::table)
.values((
chat_conversations::id.eq("existing"),
chat_conversations::title.eq("Existing chat"),
chat_conversations::created_at.eq(1_i64),
chat_conversations::updated_at.eq(1_i64),
))
.execute(conn)?;
diesel::insert_into(chat_messages::table)
.values((
chat_messages::conversation_id.eq("existing"),
chat_messages::role.eq("assistant"),
chat_messages::created_at.eq(1_i64),
chat_messages::token_usage_input.eq(Some(8)),
chat_messages::token_usage_output.eq(Some(5)),
))
.execute(conn)
})
.unwrap();
run_migrations(db.conn()).unwrap();
let usage = db
.conn()
.with(|conn| {
chat_messages::table
.select((
chat_messages::token_usage_input,
chat_messages::token_usage_output,
chat_messages::cache_read_tokens,
chat_messages::cache_write_tokens,
))
.first::<(Option<i32>, Option<i32>, Option<i32>, Option<i32>)>(conn)
})
.unwrap();
assert_eq!(usage, (Some(8), Some(5), None, None));
}
#[test]
fn existing_conversations_are_preserved_when_surface_state_is_added() {
let db = Database::open_in_memory().unwrap();
db.conn().with_migrations(|conn| {
for _ in 0..5 {
conn.run_next_migration(MIGRATIONS).unwrap();
}
});
db.conn()
.with(|conn| {
diesel::insert_into(chat_conversations::table)
.values((
chat_conversations::id.eq("existing-surface-chat"),
chat_conversations::title.eq("Keep me"),
chat_conversations::created_at.eq(1_i64),
chat_conversations::updated_at.eq(1_i64),
))
.execute(conn)
})
.unwrap();
run_migrations(db.conn()).unwrap();
let (title, state) = db
.conn()
.with(|conn| {
chat_conversations::table
.filter(chat_conversations::id.eq("existing-surface-chat"))
.select((chat_conversations::title, chat_conversations::surface_state))
.first::<(String, Option<String>)>(conn)
})
.unwrap();
assert_eq!(title, "Keep me");
assert_eq!(state, None);
}
}

View File

@@ -0,0 +1,150 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::{chat_conversations, chat_messages};
use crate::model::{ChatConversation, ChatMessage, NewChatConversation, NewChatMessage};
pub fn insert_conversation(
conn: &DbConnection,
conversation: &NewChatConversation<'_>,
) -> QueryResult<ChatConversation> {
conn.with(|connection| {
diesel::insert_into(chat_conversations::table)
.values(conversation)
.execute(connection)?;
chat_conversations::table
.find(conversation.id)
.select(ChatConversation::as_select())
.first(connection)
})
}
pub fn get_conversation(conn: &DbConnection, id: &str) -> QueryResult<ChatConversation> {
conn.with(|connection| {
chat_conversations::table
.find(id)
.select(ChatConversation::as_select())
.first(connection)
})
}
pub fn list_conversations(conn: &DbConnection) -> QueryResult<Vec<ChatConversation>> {
conn.with(|connection| {
chat_conversations::table
.order((
chat_conversations::updated_at.desc(),
chat_conversations::id.desc(),
))
.select(ChatConversation::as_select())
.load(connection)
})
}
pub fn rename_conversation(
conn: &DbConnection,
id: &str,
title: &str,
updated_at: i64,
) -> QueryResult<ChatConversation> {
conn.with(|connection| {
diesel::update(chat_conversations::table.find(id))
.set((
chat_conversations::title.eq(title),
chat_conversations::updated_at.eq(updated_at),
))
.execute(connection)?;
chat_conversations::table
.find(id)
.select(ChatConversation::as_select())
.first(connection)
})
}
pub fn set_conversation_model(
conn: &DbConnection,
id: &str,
model: &str,
updated_at: i64,
) -> QueryResult<usize> {
conn.with(|connection| {
diesel::update(chat_conversations::table.find(id))
.set((
chat_conversations::model.eq(model),
chat_conversations::updated_at.eq(updated_at),
))
.execute(connection)
})
}
pub fn set_session_id(
conn: &DbConnection,
id: &str,
session_id: Option<&str>,
updated_at: i64,
) -> QueryResult<usize> {
conn.with(|connection| {
diesel::update(chat_conversations::table.find(id))
.set((
chat_conversations::copilot_session_id.eq(session_id),
chat_conversations::updated_at.eq(updated_at),
))
.execute(connection)
})
}
pub fn set_surface_state(
conn: &DbConnection,
id: &str,
state: &str,
updated_at: i64,
) -> QueryResult<usize> {
conn.with(|connection| {
diesel::update(chat_conversations::table.find(id))
.set((
chat_conversations::surface_state.eq(state),
chat_conversations::updated_at.eq(updated_at),
))
.execute(connection)
})
}
pub fn delete_conversation(conn: &DbConnection, id: &str) -> QueryResult<usize> {
conn.with(|connection| {
connection.transaction(|connection| {
diesel::delete(chat_messages::table.filter(chat_messages::conversation_id.eq(id)))
.execute(connection)?;
diesel::delete(chat_conversations::table.find(id)).execute(connection)
})
})
}
pub fn insert_message(
conn: &DbConnection,
message: &NewChatMessage<'_>,
updated_at: i64,
) -> QueryResult<ChatMessage> {
conn.with(|connection| {
connection.transaction(|connection| {
diesel::insert_into(chat_messages::table)
.values(message)
.execute(connection)?;
diesel::update(chat_conversations::table.find(message.conversation_id))
.set(chat_conversations::updated_at.eq(updated_at))
.execute(connection)?;
chat_messages::table
.order(chat_messages::id.desc())
.select(ChatMessage::as_select())
.first(connection)
})
})
}
pub fn list_messages(conn: &DbConnection, conversation_id: &str) -> QueryResult<Vec<ChatMessage>> {
conn.with(|connection| {
chat_messages::table
.filter(chat_messages::conversation_id.eq(conversation_id))
.order((chat_messages::created_at.asc(), chat_messages::id.asc()))
.select(ChatMessage::as_select())
.load(connection)
})
}

View File

@@ -0,0 +1,98 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::db_notifications;
use crate::model::{DbNotification, NotificationAction, NotificationEntity};
#[expect(
clippy::too_many_arguments,
reason = "fields mirror the persisted notification record"
)]
pub fn insert_notification(
conn: &DbConnection,
entity_type: &NotificationEntity,
entity_id: &str,
action: &NotificationAction,
from_cli: bool,
seen_at: Option<i64>,
created_at: i64,
project_id: Option<&str>,
) -> QueryResult<()> {
conn.with(|connection| {
diesel::insert_into(db_notifications::table)
.values((
db_notifications::entity_type.eq(entity_type),
db_notifications::entity_id.eq(entity_id),
db_notifications::action.eq(action),
db_notifications::from_cli.eq(i32::from(from_cli)),
db_notifications::seen_at.eq(seen_at),
db_notifications::created_at.eq(created_at),
db_notifications::project_id.eq(project_id),
))
.execute(connection)
.map(|_| ())
})
}
pub fn list_notifications(conn: &DbConnection) -> QueryResult<Vec<DbNotification>> {
conn.with(|connection| {
db_notifications::table
.order((
db_notifications::created_at.asc(),
db_notifications::id.asc(),
))
.select(DbNotification::as_select())
.load(connection)
})
}
pub fn list_unseen_cli_notifications(conn: &DbConnection) -> QueryResult<Vec<DbNotification>> {
conn.with(|connection| {
db_notifications::table
.filter(db_notifications::from_cli.eq(1))
.filter(db_notifications::seen_at.is_null())
.order((
db_notifications::created_at.asc(),
db_notifications::id.asc(),
))
.select(DbNotification::as_select())
.load(connection)
})
}
pub fn mark_notifications_seen(
conn: &DbConnection,
ids: &[i32],
seen_at: i64,
) -> QueryResult<usize> {
if ids.is_empty() {
return Ok(0);
}
conn.with(|connection| {
diesel::update(db_notifications::table.filter(db_notifications::id.eq_any(ids)))
.set(db_notifications::seen_at.eq(seen_at))
.execute(connection)
})
}
pub fn prune_processed(conn: &DbConnection, cutoff: i64) -> QueryResult<usize> {
conn.with(|connection| {
diesel::delete(
db_notifications::table
.filter(db_notifications::seen_at.is_not_null())
.filter(db_notifications::created_at.le(cutoff)),
)
.execute(connection)
})
}
pub fn prune_unprocessed(conn: &DbConnection, cutoff: i64) -> QueryResult<usize> {
conn.with(|connection| {
diesel::delete(
db_notifications::table
.filter(db_notifications::seen_at.is_null())
.filter(db_notifications::created_at.le(cutoff)),
)
.execute(connection)
})
}

View File

@@ -0,0 +1,283 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::{dismissed_duplicate_pairs, embedding_keys};
use crate::model::{DismissedDuplicatePair, EmbeddingKey};
pub fn get_key_for_post(
conn: &DbConnection,
project_id: &str,
post_id: &str,
) -> QueryResult<Option<EmbeddingKey>> {
conn.with(|c| {
embedding_keys::table
.filter(embedding_keys::project_id.eq(project_id))
.filter(embedding_keys::post_id.eq(post_id))
.select(EmbeddingKey::as_select())
.first(c)
.optional()
})
}
pub fn list_keys(conn: &DbConnection, project_id: &str) -> QueryResult<Vec<EmbeddingKey>> {
conn.with(|c| {
embedding_keys::table
.filter(embedding_keys::project_id.eq(project_id))
.order(embedding_keys::label.asc())
.select(EmbeddingKey::as_select())
.load(c)
})
}
pub fn max_label(conn: &DbConnection) -> QueryResult<i64> {
conn.with(|c| {
embedding_keys::table
.select(embedding_keys::label)
.order(embedding_keys::label.desc())
.first(c)
.optional()
.map(|label| label.unwrap_or(0))
})
}
pub fn upsert_key(conn: &DbConnection, key: &EmbeddingKey) -> QueryResult<()> {
conn.with(|c| {
diesel::insert_into(embedding_keys::table)
.values(key)
.on_conflict((embedding_keys::project_id, embedding_keys::post_id))
.do_update()
.set((
embedding_keys::content_hash.eq(&key.content_hash),
embedding_keys::vector.eq(&key.vector),
))
.execute(c)
.map(|_| ())
})
}
pub fn delete_key_for_post(
conn: &DbConnection,
project_id: &str,
post_id: &str,
) -> QueryResult<()> {
conn.with(|c| {
diesel::delete(
embedding_keys::table
.filter(embedding_keys::project_id.eq(project_id))
.filter(embedding_keys::post_id.eq(post_id)),
)
.execute(c)
.map(|_| ())
})
}
pub fn delete_stale_keys(
conn: &DbConnection,
project_id: &str,
live_post_ids: &[String],
) -> QueryResult<usize> {
conn.with(|c| {
let query = embedding_keys::table.filter(embedding_keys::project_id.eq(project_id));
if live_post_ids.is_empty() {
diesel::delete(query).execute(c)
} else {
diesel::delete(query.filter(embedding_keys::post_id.ne_all(live_post_ids))).execute(c)
}
})
}
pub fn insert_dismissed_pair(
conn: &DbConnection,
pair: &DismissedDuplicatePair,
) -> QueryResult<()> {
conn.with(|c| {
diesel::insert_into(dismissed_duplicate_pairs::table)
.values(pair)
.on_conflict((
dismissed_duplicate_pairs::project_id,
dismissed_duplicate_pairs::post_id_a,
dismissed_duplicate_pairs::post_id_b,
))
.do_nothing()
.execute(c)
.map(|_| ())
})
}
pub fn insert_dismissed_pairs(
conn: &DbConnection,
pairs: &[DismissedDuplicatePair],
) -> QueryResult<usize> {
if pairs.is_empty() {
return Ok(0);
}
conn.with(|c| {
diesel::insert_into(dismissed_duplicate_pairs::table)
.values(pairs)
.on_conflict((
dismissed_duplicate_pairs::project_id,
dismissed_duplicate_pairs::post_id_a,
dismissed_duplicate_pairs::post_id_b,
))
.do_nothing()
.execute(c)
})
}
pub fn list_dismissed_pairs(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<DismissedDuplicatePair>> {
conn.with(|c| {
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id))
.select(DismissedDuplicatePair::as_select())
.load(c)
})
}
pub fn delete_orphan_dismissals(
conn: &DbConnection,
project_id: &str,
live_post_ids: &[String],
) -> QueryResult<usize> {
conn.with(|c| {
let query = dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id));
if live_post_ids.is_empty() {
diesel::delete(query).execute(c)
} else {
diesel::delete(
query.filter(
dismissed_duplicate_pairs::post_id_a
.ne_all(live_post_ids)
.or(dismissed_duplicate_pairs::post_id_b.ne_all(live_post_ids)),
),
)
.execute(c)
}
})
}
pub fn delete_dismissals_for_post(
conn: &DbConnection,
project_id: &str,
post_id: &str,
) -> QueryResult<usize> {
conn.with(|c| {
diesel::delete(
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id))
.filter(
dismissed_duplicate_pairs::post_id_a
.eq(post_id)
.or(dismissed_duplicate_pairs::post_id_b.eq(post_id)),
),
)
.execute(c)
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::model::{Post, PostStatus, Project};
fn seeded() -> (Database, String, String) {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
let project_id = "embedding-project".to_string();
crate::db::queries::project::insert_project(
db.conn(),
&Project {
id: project_id.clone(),
name: "Embedding".into(),
slug: "embedding".into(),
description: None,
data_path: Some("/tmp/embedding".into()),
is_active: true,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
let post_id = "embedding-post".to_string();
crate::db::queries::post::insert_post(
db.conn(),
&Post {
id: post_id.clone(),
project_id: project_id.clone(),
title: "Post".into(),
slug: "post".into(),
excerpt: None,
content: Some("Body".into()),
status: PostStatus::Draft,
author: None,
language: Some("en".into()),
do_not_translate: false,
template_slug: None,
file_path: "posts/post.md".into(),
checksum: None,
tags: vec![],
categories: vec![],
published_title: None,
published_content: None,
published_tags: None,
published_categories: None,
published_excerpt: None,
created_at: 1,
updated_at: 1,
published_at: None,
},
)
.unwrap();
(db, project_id, post_id)
}
#[test]
fn embedding_vector_round_trips_as_blob_and_dismissals_are_canonical() {
let (db, project_id, post_id) = seeded();
let key = EmbeddingKey {
label: 1,
post_id: post_id.clone(),
project_id: project_id.clone(),
content_hash: "hash".into(),
vector: vec![0, 1, 255],
};
upsert_key(db.conn(), &key).unwrap();
assert_eq!(
get_key_for_post(db.conn(), &project_id, &post_id).unwrap(),
Some(key)
);
let replacement = EmbeddingKey {
label: 2,
post_id: post_id.clone(),
project_id: project_id.clone(),
content_hash: "new-hash".into(),
vector: vec![3, 2, 1],
};
upsert_key(db.conn(), &replacement).unwrap();
let updated = get_key_for_post(db.conn(), &project_id, &post_id)
.unwrap()
.unwrap();
assert_eq!(updated.label, 1, "a post keeps its stable HNSW label");
assert_eq!(updated.content_hash, "new-hash");
assert_eq!(updated.vector, vec![3, 2, 1]);
let pair = DismissedDuplicatePair {
id: "dismissal".into(),
project_id: project_id.clone(),
post_id_a: "a".into(),
post_id_b: "b".into(),
dismissed_at: 1,
};
insert_dismissed_pair(db.conn(), &pair).unwrap();
insert_dismissed_pair(db.conn(), &pair).unwrap();
assert_eq!(
list_dismissed_pairs(db.conn(), &project_id).unwrap(),
vec![pair]
);
}
}

View File

@@ -18,6 +18,18 @@ pub fn get_generated_file_hash(
})
}
pub fn list_generated_file_hashes(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<GeneratedFileHash>> {
conn.with(|c| {
generated_file_hashes::table
.filter(generated_file_hashes::project_id.eq(project_id))
.select(GeneratedFileHash::as_select())
.load(c)
})
}
pub fn upsert_generated_file_hash(
conn: &DbConnection,
hash: &GeneratedFileHash,
@@ -39,6 +51,52 @@ pub fn upsert_generated_file_hash(
})
}
pub fn upsert_generated_file_hashes(
conn: &DbConnection,
hashes: &[GeneratedFileHash],
) -> QueryResult<()> {
conn.with(|c| {
hashes.chunks(200).try_for_each(|chunk| {
diesel::insert_into(generated_file_hashes::table)
.values(chunk)
.on_conflict((
generated_file_hashes::project_id,
generated_file_hashes::relative_path,
))
.do_update()
.set((
generated_file_hashes::content_hash.eq(diesel::upsert::excluded(
generated_file_hashes::content_hash,
)),
generated_file_hashes::updated_at
.eq(diesel::upsert::excluded(generated_file_hashes::updated_at)),
))
.execute(c)
.map(|_| ())
})
})
}
pub fn touch_generated_file_hashes(
conn: &DbConnection,
project_id: &str,
relative_paths: &[String],
updated_at: i64,
) -> QueryResult<usize> {
conn.with(|c| {
relative_paths.chunks(500).try_fold(0, |updated, paths| {
diesel::update(
generated_file_hashes::table
.filter(generated_file_hashes::project_id.eq(project_id))
.filter(generated_file_hashes::relative_path.eq_any(paths)),
)
.set(generated_file_hashes::updated_at.eq(updated_at))
.execute(c)
.map(|count| updated + count)
})
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -79,4 +137,36 @@ mod tests {
assert_eq!(stored.content_hash, "def");
assert_eq!(stored.updated_at, 99);
}
#[test]
fn batch_upsert_inserts_and_updates_hashes() {
let db = setup();
let mut hashes = vec![
GeneratedFileHash {
project_id: "p1".into(),
relative_path: "index.html".into(),
content_hash: "one".into(),
updated_at: 1,
},
GeneratedFileHash {
project_id: "p1".into(),
relative_path: "rss.xml".into(),
content_hash: "two".into(),
updated_at: 1,
},
];
upsert_generated_file_hashes(db.conn(), &hashes).unwrap();
hashes[0].content_hash = "changed".into();
hashes[0].updated_at = 2;
upsert_generated_file_hashes(db.conn(), &hashes[..1]).unwrap();
let stored = list_generated_file_hashes(db.conn(), "p1").unwrap();
assert_eq!(stored.len(), 2);
assert_eq!(
get_generated_file_hash(db.conn(), "p1", "index.html")
.unwrap()
.content_hash,
"changed"
);
}
}

View File

@@ -0,0 +1,62 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::import_definitions;
use crate::model::ImportDefinition;
pub fn insert_import_definition(
conn: &DbConnection,
definition: &ImportDefinition,
) -> QueryResult<()> {
conn.with(|connection| {
diesel::insert_into(import_definitions::table)
.values(definition.clone())
.execute(connection)
.map(|_| ())
})
}
pub fn get_import_definition(conn: &DbConnection, id: &str) -> QueryResult<ImportDefinition> {
conn.with(|connection| {
import_definitions::table
.filter(import_definitions::id.eq(id))
.select(ImportDefinition::as_select())
.first(connection)
})
}
pub fn list_import_definitions(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<ImportDefinition>> {
conn.with(|connection| {
import_definitions::table
.filter(import_definitions::project_id.eq(project_id))
.order((
import_definitions::updated_at.desc(),
import_definitions::created_at.desc(),
))
.select(ImportDefinition::as_select())
.load(connection)
})
}
pub fn update_import_definition(
conn: &DbConnection,
definition: &ImportDefinition,
) -> QueryResult<()> {
conn.with(|connection| {
diesel::update(import_definitions::table.filter(import_definitions::id.eq(&definition.id)))
.set(definition.clone())
.execute(connection)
.map(|_| ())
})
}
pub fn delete_import_definition(conn: &DbConnection, id: &str) -> QueryResult<()> {
conn.with(|connection| {
diesel::delete(import_definitions::table.filter(import_definitions::id.eq(id)))
.execute(connection)
.map(|_| ())
})
}

View File

@@ -0,0 +1,185 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::mcp_proposals;
use crate::model::{McpProposal, ProposalStatus};
pub fn insert_proposal(conn: &DbConnection, proposal: &McpProposal) -> QueryResult<()> {
conn.with(|connection| {
diesel::insert_into(mcp_proposals::table)
.values(proposal)
.execute(connection)
.map(|_| ())
})
}
pub fn get_proposal(conn: &DbConnection, id: &str) -> QueryResult<McpProposal> {
conn.with(|connection| {
mcp_proposals::table
.filter(mcp_proposals::id.eq(id))
.select(McpProposal::as_select())
.first(connection)
})
}
pub fn list_proposals(conn: &DbConnection, project_id: &str) -> QueryResult<Vec<McpProposal>> {
conn.with(|connection| {
mcp_proposals::table
.filter(mcp_proposals::project_id.eq(project_id))
.order(mcp_proposals::created_at.desc())
.select(McpProposal::as_select())
.load(connection)
})
}
pub fn list_pending_proposals(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<McpProposal>> {
conn.with(|connection| {
mcp_proposals::table
.filter(mcp_proposals::project_id.eq(project_id))
.filter(mcp_proposals::status.eq(ProposalStatus::Pending))
.order(mcp_proposals::created_at.asc())
.select(McpProposal::as_select())
.load(connection)
})
}
pub fn expire_pending(conn: &DbConnection, now: i64) -> QueryResult<usize> {
conn.with(|connection| {
let expiring = mcp_proposals::table
.filter(mcp_proposals::status.eq(ProposalStatus::Pending))
.filter(mcp_proposals::expires_at.le(now))
.select(McpProposal::as_select())
.load::<McpProposal>(connection)?;
for proposal in &expiring {
diesel::delete(
mcp_proposals::table
.filter(mcp_proposals::id.ne(&proposal.id))
.filter(mcp_proposals::kind.eq(proposal.kind))
.filter(mcp_proposals::entity_id.eq(&proposal.entity_id))
.filter(mcp_proposals::status.eq(ProposalStatus::Expired)),
)
.execute(connection)?;
}
diesel::update(
mcp_proposals::table
.filter(mcp_proposals::status.eq(ProposalStatus::Pending))
.filter(mcp_proposals::expires_at.le(now)),
)
.set((
mcp_proposals::status.eq(ProposalStatus::Expired),
mcp_proposals::resolved_at.eq(Some(now)),
mcp_proposals::result.eq(Some("{\"message\":\"expired\"}".to_string())),
))
.execute(connection)
})
}
pub fn delete_status_collision(
conn: &DbConnection,
proposal: &McpProposal,
status: ProposalStatus,
) -> QueryResult<usize> {
conn.with(|connection| {
diesel::delete(
mcp_proposals::table
.filter(mcp_proposals::id.ne(&proposal.id))
.filter(mcp_proposals::kind.eq(proposal.kind))
.filter(mcp_proposals::entity_id.eq(&proposal.entity_id))
.filter(mcp_proposals::status.eq(status)),
)
.execute(connection)
})
}
pub fn claim_pending(conn: &DbConnection, id: &str, now: i64) -> QueryResult<bool> {
conn.with(|connection| {
diesel::update(
mcp_proposals::table
.filter(mcp_proposals::id.eq(id))
.filter(mcp_proposals::status.eq(ProposalStatus::Pending))
.filter(mcp_proposals::expires_at.gt(now)),
)
.set(mcp_proposals::status.eq(ProposalStatus::Executing))
.execute(connection)
.map(|changed| changed == 1)
})
}
pub fn resolve_claimed(
conn: &DbConnection,
id: &str,
status: ProposalStatus,
result: &str,
resolved_at: i64,
) -> QueryResult<bool> {
debug_assert!(matches!(
status,
ProposalStatus::Accepted | ProposalStatus::Rejected
));
conn.with(|connection| {
diesel::update(
mcp_proposals::table
.filter(mcp_proposals::id.eq(id))
.filter(mcp_proposals::status.eq(ProposalStatus::Executing)),
)
.set((
mcp_proposals::status.eq(status),
mcp_proposals::result.eq(Some(result.to_string())),
mcp_proposals::resolved_at.eq(Some(resolved_at)),
))
.execute(connection)
.map(|changed| changed == 1)
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::db::queries::project::{insert_project, make_test_project};
use crate::model::ProposalKind;
fn setup() -> Database {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
insert_project(db.conn(), &make_test_project("p1", "blog")).unwrap();
db
}
fn proposal(id: &str, expires_at: i64) -> McpProposal {
McpProposal {
id: id.into(),
project_id: "p1".into(),
kind: ProposalKind::DraftPost,
status: ProposalStatus::Pending,
entity_id: id.into(),
data: "{}".into(),
result: None,
created_at: 1,
expires_at,
resolved_at: None,
}
}
#[test]
fn lifecycle_claims_once_and_expires_pending_rows() {
let db = setup();
insert_proposal(db.conn(), &proposal("p1", 10)).unwrap();
insert_proposal(db.conn(), &proposal("p2", 1)).unwrap();
assert_eq!(expire_pending(db.conn(), 5).unwrap(), 1);
assert!(claim_pending(db.conn(), "p1", 5).unwrap());
assert!(!claim_pending(db.conn(), "p1", 5).unwrap());
assert!(resolve_claimed(db.conn(), "p1", ProposalStatus::Accepted, "{}", 6).unwrap());
assert_eq!(
get_proposal(db.conn(), "p1").unwrap().status,
ProposalStatus::Accepted
);
assert_eq!(
get_proposal(db.conn(), "p2").unwrap().status,
ProposalStatus::Expired
);
}
}

View File

@@ -1,4 +1,9 @@
pub mod chat;
pub mod db_notification;
pub mod embedding;
pub mod generated_file_hash;
pub mod import_definition;
pub mod mcp_proposal;
pub mod media;
pub mod media_translation;
pub mod post;

View File

@@ -79,75 +79,6 @@ pub fn update_post_status(
})
}
pub fn clear_post_content(conn: &DbConnection, id: &str, updated_at: i64) -> QueryResult<()> {
conn.with(|c| {
diesel::update(posts::table.filter(posts::id.eq(id)))
.set((
posts::content.eq(None::<String>),
posts::updated_at.eq(updated_at),
))
.execute(c)
.map(|_| ())
})
}
pub fn set_post_file_path(
conn: &DbConnection,
id: &str,
file_path: &str,
updated_at: i64,
) -> QueryResult<()> {
conn.with(|c| {
diesel::update(posts::table.filter(posts::id.eq(id)))
.set((
posts::file_path.eq(file_path),
posts::updated_at.eq(updated_at),
))
.execute(c)
.map(|_| ())
})
}
pub fn set_post_checksum(conn: &DbConnection, id: &str, checksum: Option<&str>) -> QueryResult<()> {
conn.with(|c| {
diesel::update(posts::table.filter(posts::id.eq(id)))
.set(posts::checksum.eq(checksum))
.execute(c)
.map(|_| ())
})
}
#[expect(
clippy::too_many_arguments,
reason = "arguments mirror the published snapshot columns"
)]
pub fn set_published_snapshot(
conn: &DbConnection,
id: &str,
title: &str,
content: &str,
tags: &str,
categories: &str,
excerpt: Option<&str>,
published_at: i64,
updated_at: i64,
) -> QueryResult<()> {
conn.with(|c| {
diesel::update(posts::table.filter(posts::id.eq(id)))
.set((
posts::published_title.eq(title),
posts::published_content.eq(content),
posts::published_tags.eq(tags),
posts::published_categories.eq(categories),
posts::published_excerpt.eq(excerpt),
posts::published_at.eq(published_at),
posts::updated_at.eq(updated_at),
))
.execute(c)
.map(|_| ())
})
}
pub fn delete_post(conn: &DbConnection, id: &str) -> QueryResult<()> {
conn.with(|c| {
diesel::delete(posts::table.filter(posts::id.eq(id)))
@@ -545,46 +476,6 @@ mod tests {
assert!(get_post_by_id(db.conn(), "x1").is_err());
}
#[test]
fn clear_content_sets_null() {
let db = setup();
insert_post(db.conn(), &make_post("x1", "hello")).unwrap();
clear_post_content(db.conn(), "x1", 5000).unwrap();
let fetched = get_post_by_id(db.conn(), "x1").unwrap();
assert!(fetched.content.is_none());
}
#[test]
fn set_file_path_updates() {
let db = setup();
insert_post(db.conn(), &make_post("x1", "hello")).unwrap();
set_post_file_path(db.conn(), "x1", "posts/2024/01/hello.md", 5000).unwrap();
let fetched = get_post_by_id(db.conn(), "x1").unwrap();
assert_eq!(fetched.file_path, "posts/2024/01/hello.md");
}
#[test]
fn published_snapshot() {
let db = setup();
insert_post(db.conn(), &make_post("x1", "hello")).unwrap();
set_published_snapshot(
db.conn(),
"x1",
"Pub Title",
"Pub Body",
"[\"rust\"]",
"[\"tech\"]",
Some("Pub Excerpt"),
3000,
3000,
)
.unwrap();
let fetched = get_post_by_id(db.conn(), "x1").unwrap();
assert_eq!(fetched.published_title.as_deref(), Some("Pub Title"));
assert_eq!(fetched.published_content.as_deref(), Some("Pub Body"));
assert_eq!(fetched.published_at, Some(3000));
}
#[test]
fn delete_removes_post() {
let db = setup();

View File

@@ -1,7 +1,7 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::post_links;
use crate::db::schema::{post_links, posts};
use crate::model::PostLink;
pub fn insert_post_link(conn: &DbConnection, link: &PostLink) -> QueryResult<()> {
@@ -55,6 +55,17 @@ pub fn list_links_by_target(
})
}
pub fn list_links_by_project(conn: &DbConnection, project_id: &str) -> QueryResult<Vec<PostLink>> {
conn.with(|c| {
post_links::table
.inner_join(posts::table.on(posts::id.eq(post_links::source_post_id)))
.filter(posts::project_id.eq(project_id))
.order(post_links::created_at)
.select(PostLink::as_select())
.load(c)
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -100,6 +111,14 @@ mod tests {
assert_eq!(links.len(), 2);
}
#[test]
fn list_by_project() {
let db = setup();
insert_post_link(db.conn(), &make_link("l1", "a", "b")).unwrap();
insert_post_link(db.conn(), &make_link("l2", "b", "c")).unwrap();
assert_eq!(list_links_by_project(db.conn(), "p1").unwrap().len(), 2);
}
#[test]
fn delete_by_source() {
let db = setup();

View File

@@ -37,7 +37,7 @@ pub fn list_post_media_by_post(conn: &DbConnection, post_id: &str) -> QueryResul
conn.with(|c| {
post_media::table
.filter(post_media::post_id.eq(post_id))
.order(post_media::sort_order)
.order((post_media::sort_order.asc(), post_media::media_id.asc()))
.select(PostMedia::as_select())
.load(c)
})
@@ -56,6 +56,23 @@ pub fn list_post_media_by_media(
})
}
pub fn list_post_media_by_project(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<PostMedia>> {
conn.with(|c| {
post_media::table
.filter(post_media::project_id.eq(project_id))
.order((
post_media::post_id.asc(),
post_media::sort_order.asc(),
post_media::media_id.asc(),
))
.select(PostMedia::as_select())
.load(c)
})
}
pub fn update_sort_order(
conn: &DbConnection,
post_id: &str,
@@ -123,6 +140,20 @@ mod tests {
assert_eq!(list[0].post_id, "post1");
}
#[test]
fn list_by_project_preserves_post_order() {
let db = setup();
link_media(db.conn(), &make_pm("pm1", "m1", 1)).unwrap();
link_media(db.conn(), &make_pm("pm2", "m2", 0)).unwrap();
let list = list_post_media_by_project(db.conn(), "p1").unwrap();
assert_eq!(
list.iter()
.map(|link| link.media_id.as_str())
.collect::<Vec<_>>(),
["m2", "m1"]
);
}
#[test]
fn unlink_removes_association() {
let db = setup();

View File

@@ -54,6 +54,22 @@ pub fn list_post_translations_by_post(
})
}
pub fn list_post_translations_by_project(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<PostTranslation>> {
conn.with(|c| {
post_translations::table
.filter(post_translations::project_id.eq(project_id))
.order((
post_translations::translation_for,
post_translations::language,
))
.select(PostTranslation::as_select())
.load(c)
})
}
pub fn update_post_translation(conn: &DbConnection, t: &PostTranslation) -> QueryResult<()> {
if !t.status.is_valid_for_translation() {
return Err(diesel::result::Error::SerializationError(
@@ -151,6 +167,15 @@ mod tests {
assert_eq!(list[1].language, "fr");
}
#[test]
fn list_by_project() {
let db = setup();
insert_post_translation(db.conn(), &make_translation("t1", "de")).unwrap();
insert_post_translation(db.conn(), &make_translation("t2", "fr")).unwrap();
let list = list_post_translations_by_project(db.conn(), "p1").unwrap();
assert_eq!(list.len(), 2);
}
#[test]
fn update_translation() {
let db = setup();

View File

@@ -46,6 +46,12 @@ pub fn list_all_settings(conn: &DbConnection) -> QueryResult<Vec<Setting>> {
})
}
pub fn delete_settings_by_prefix(conn: &DbConnection, prefix: &str) -> QueryResult<usize> {
conn.with(|c| {
diesel::delete(settings::table.filter(settings::key.like(format!("{prefix}%")))).execute(c)
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -93,4 +99,18 @@ mod tests {
let db = setup();
assert!(get_setting_by_key(db.conn(), "nope").is_err());
}
#[test]
fn delete_by_prefix_is_project_scoped() {
let db = setup();
set_setting_value(db.conn(), "project:p1:categories", "[]", 1).unwrap();
set_setting_value(db.conn(), "project:p2:categories", "[]", 1).unwrap();
assert_eq!(
delete_settings_by_prefix(db.conn(), "project:p1:").unwrap(),
1
);
assert!(get_setting_by_key(db.conn(), "project:p1:categories").is_err());
assert!(get_setting_by_key(db.conn(), "project:p2:categories").is_ok());
}
}

View File

@@ -7,6 +7,19 @@ diesel::table! {
}
}
diesel::table! {
ai_endpoint_models (kind, model_id) {
kind -> Text,
model_id -> Text,
label -> Text,
context_window -> Nullable<Integer>,
max_output_tokens -> Nullable<Integer>,
supports_tools -> Integer,
supports_vision -> Integer,
updated_at -> BigInt,
}
}
diesel::table! {
ai_model_modalities (rowid) {
rowid -> Integer,
@@ -66,6 +79,7 @@ diesel::table! {
copilot_session_id -> Nullable<Text>,
created_at -> BigInt,
updated_at -> BigInt,
surface_state -> Nullable<Text>,
}
}
@@ -78,6 +92,8 @@ diesel::table! {
tool_call_id -> Nullable<Text>,
tool_calls -> Nullable<Text>,
created_at -> BigInt,
token_usage_input -> Nullable<Integer>,
token_usage_output -> Nullable<Integer>,
cache_read_tokens -> Nullable<Integer>,
cache_write_tokens -> Nullable<Integer>,
}
@@ -92,6 +108,7 @@ diesel::table! {
from_cli -> Integer,
seen_at -> Nullable<BigInt>,
created_at -> BigInt,
project_id -> Nullable<Text>,
}
}
@@ -111,7 +128,7 @@ diesel::table! {
post_id -> Text,
project_id -> Text,
content_hash -> Text,
vector -> Text,
vector -> Binary,
}
}
@@ -138,6 +155,21 @@ diesel::table! {
}
}
diesel::table! {
mcp_proposals (id) {
id -> Text,
project_id -> Text,
kind -> Text,
status -> Text,
entity_id -> Text,
data -> Text,
result -> Nullable<Text>,
created_at -> BigInt,
expires_at -> BigInt,
resolved_at -> Nullable<BigInt>,
}
}
diesel::table! {
media (id) {
id -> Text,
@@ -316,6 +348,7 @@ diesel::joinable!(chat_messages -> chat_conversations (conversation_id));
diesel::joinable!(dismissed_duplicate_pairs -> projects (project_id));
diesel::joinable!(generated_file_hashes -> projects (project_id));
diesel::joinable!(import_definitions -> projects (project_id));
diesel::joinable!(mcp_proposals -> projects (project_id));
diesel::joinable!(media -> projects (project_id));
diesel::joinable!(media_translations -> media (translation_for));
diesel::joinable!(media_translations -> projects (project_id));
@@ -331,6 +364,7 @@ diesel::joinable!(templates -> projects (project_id));
diesel::allow_tables_to_appear_in_same_query!(
ai_catalog_meta,
ai_endpoint_models,
ai_model_modalities,
ai_models,
ai_providers,
@@ -341,6 +375,7 @@ diesel::allow_tables_to_appear_in_same_query!(
embedding_keys,
generated_file_hashes,
import_definitions,
mcp_proposals,
media,
media_translations,
post_links,

View File

@@ -5,8 +5,8 @@ use diesel::sql_types::{Integer, Text};
use diesel::sqlite::{Sqlite, SqliteValue};
use crate::model::{
NotificationAction, NotificationEntity, PostStatus, ScriptKind, ScriptStatus, TemplateKind,
TemplateStatus,
ChatRole, NotificationAction, NotificationEntity, PostStatus, ProposalKind, ProposalStatus,
ScriptKind, ScriptStatus, TemplateKind, TemplateStatus,
};
#[derive(Debug, AsExpression, FromSqlRow)]
@@ -93,3 +93,6 @@ text_enum_sql!(ScriptKind);
text_enum_sql!(ScriptStatus);
text_enum_sql!(NotificationEntity);
text_enum_sql!(NotificationAction);
text_enum_sql!(ProposalKind);
text_enum_sql!(ProposalStatus);
text_enum_sql!(ChatRole);

File diff suppressed because it is too large Load Diff

View File

@@ -13,6 +13,7 @@ use crate::engine::ai::{
TranslationResult,
};
use crate::engine::{EngineError, EngineResult};
use crate::i18n::{UiLocale, translate, translate_with};
use crate::model::{Media, Post, PostStatus};
use crate::util::frontmatter::read_post_file;
@@ -26,6 +27,29 @@ pub struct FillMissingTranslationsReport {
pub errors: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FillMissingTranslationsProgress {
ScanningPublishedPosts,
TranslatingPost { title: String, language: String },
Complete,
}
impl FillMissingTranslationsProgress {
pub fn localized(&self, locale: UiLocale) -> String {
match self {
Self::ScanningPublishedPosts => {
translate(locale, "engine.progress.scanningPublishedPosts")
}
Self::TranslatingPost { title, language } => translate_with(
locale,
"engine.progress.translatingPost",
&[("title", title), ("language", language)],
),
Self::Complete => translate(locale, "engine.progress.translationBatchComplete"),
}
}
}
pub fn configured_languages(main_language: &str, blog_languages: &[String]) -> Vec<String> {
let mut seen = HashSet::new();
std::iter::once(main_language.to_string())
@@ -64,7 +88,7 @@ pub fn fill_missing_translations(
main_language: &str,
blog_languages: &[String],
offline_mode: bool,
mut on_progress: impl FnMut(f32, &str) -> bool,
mut on_progress: impl FnMut(f32, &FillMissingTranslationsProgress) -> bool,
) -> EngineResult<FillMissingTranslationsReport> {
fill_missing_translations_with(
conn,
@@ -90,7 +114,7 @@ fn fill_missing_translations_with(
blog_languages: &[String],
post_translator: &mut dyn FnMut(&Post, &str) -> EngineResult<TranslationResult>,
media_translator: &mut dyn FnMut(&Media, &str) -> EngineResult<MediaTranslationResult>,
on_progress: &mut dyn FnMut(f32, &str) -> bool,
on_progress: &mut dyn FnMut(f32, &FillMissingTranslationsProgress) -> bool,
) -> EngineResult<FillMissingTranslationsReport> {
let configured = configured_languages(main_language, blog_languages);
if configured.len() <= 1 {
@@ -100,7 +124,10 @@ fn fill_missing_translations_with(
});
}
let posts = qp::list_posts_by_project(conn, project_id)?;
if !on_progress(0.0, "Scanning published posts") {
if !on_progress(
0.0,
&FillMissingTranslationsProgress::ScanningPublishedPosts,
) {
return Err(EngineError::Validation("cancelled".to_string()));
}
let mut work = Vec::new();
@@ -108,7 +135,10 @@ fn fill_missing_translations_with(
.into_iter()
.filter(|post| post.status == PostStatus::Published && !post.do_not_translate)
{
if !on_progress(0.0, "Scanning published posts") {
if !on_progress(
0.0,
&FillMissingTranslationsProgress::ScanningPublishedPosts,
) {
return Err(EngineError::Validation("cancelled".to_string()));
}
for language in missing_languages(conn, &post, &configured)? {
@@ -126,7 +156,10 @@ fn fill_missing_translations_with(
for (index, (post, language)) in work.iter().enumerate() {
if !on_progress(
0.15 + (index as f32 / work.len() as f32) * 0.85,
&format!("{}{language}", post.title),
&FillMissingTranslationsProgress::TranslatingPost {
title: post.title.clone(),
language: language.clone(),
},
) {
return Err(EngineError::Validation("cancelled".to_string()));
}
@@ -151,7 +184,7 @@ fn fill_missing_translations_with(
}
}
}
if !on_progress(1.0, "Translation batch complete") {
if !on_progress(1.0, &FillMissingTranslationsProgress::Complete) {
return Err(EngineError::Validation("cancelled".to_string()));
}
Ok(report)
@@ -178,29 +211,83 @@ pub fn translate_missing_for_post(
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let result = translate_one_post(
merge_reactive_translation_result(
&mut report,
&post,
&language,
translate_one_post(
conn,
data_dir,
&post,
&language,
false,
&mut |post, language| translate_post_ai(conn, offline_mode, post, language),
&mut |media, language| translate_media_ai(conn, offline_mode, media, language),
),
);
}
Ok(report)
}
/// Translate one language from the current missing-language set. This is the
/// reactive editor path: already-created or unconfigured translations are a
/// silent no-op, and generated translations remain drafts.
pub fn translate_missing_language_for_post(
conn: &Connection,
data_dir: &Path,
post_id: &str,
configured_languages: &[String],
language: &str,
offline_mode: bool,
is_cancelled: impl Fn() -> bool,
) -> EngineResult<FillMissingTranslationsReport> {
let post = qp::get_post_by_id(conn, post_id)?;
let targets = missing_languages(conn, &post, configured_languages)?;
if !targets.iter().any(|target| target == language) {
return Ok(FillMissingTranslationsReport {
nothing_to_do: true,
..Default::default()
});
}
if is_cancelled() {
return Err(EngineError::Validation("cancelled".to_string()));
}
let mut report = FillMissingTranslationsReport::default();
merge_reactive_translation_result(
&mut report,
&post,
language,
translate_one_post(
conn,
data_dir,
&post,
&language,
language,
false,
&mut |post, language| translate_post_ai(conn, offline_mode, post, language),
&mut |media, language| translate_media_ai(conn, offline_mode, media, language),
);
match result {
Ok(media_count) => {
report.translated_posts += 1;
report.translated_media += media_count;
}
Err(error) => {
report.failed_count += 1;
report
.errors
.push(format!("{} ({language}): {error}", post.title));
}
),
);
Ok(report)
}
fn merge_reactive_translation_result(
report: &mut FillMissingTranslationsReport,
post: &Post,
language: &str,
result: EngineResult<usize>,
) {
match result {
Ok(media_count) => {
report.translated_posts += 1;
report.translated_media += media_count;
}
Err(error) => {
report.failed_count += 1;
report
.errors
.push(format!("{} ({language}): {error}", post.title));
}
}
Ok(report)
}
fn translate_one_post(
@@ -222,7 +309,7 @@ fn translate_one_post(
let mut input = post.clone();
input.content = Some(body);
let translated = post_translator(&input, language)?;
let translation = crate::engine::post::upsert_translation(
let translation = crate::engine::post::upsert_automatic_translation(
conn,
data_dir,
&post.id,
@@ -293,7 +380,7 @@ fn translate_post_ai(
}),
},
)? {
OneShotResponse::Translation(result) => Ok(result),
(OneShotResponse::Translation(result), _usage) => Ok(result),
_ => Err(EngineError::Parse(
"unexpected post translation response".into(),
)),
@@ -320,7 +407,7 @@ fn translate_media_ai(
}),
},
)? {
OneShotResponse::MediaTranslation(result) => Ok(result),
(OneShotResponse::MediaTranslation(result), _usage) => Ok(result),
_ => Err(EngineError::Parse(
"unexpected media translation response".into(),
)),
@@ -342,9 +429,29 @@ mod tests {
use crate::db::Database;
use crate::db::fts::ensure_fts_tables;
use crate::db::queries::project::{insert_project, make_test_project};
use crate::engine::post::{create_post, publish_post};
use crate::engine::post::{create_post, publish_post, upsert_translation};
use tempfile::TempDir;
#[test]
fn progress_events_use_the_selected_ui_locale() {
assert_eq!(
FillMissingTranslationsProgress::ScanningPublishedPosts.localized(UiLocale::De),
"Veröffentlichte Beiträge werden durchsucht…"
);
assert_eq!(
FillMissingTranslationsProgress::TranslatingPost {
title: "Hallo".into(),
language: "fr".into(),
}
.localized(UiLocale::Fr),
"Traduction de Hallo → fr"
);
assert_eq!(
FillMissingTranslationsProgress::Complete.localized(UiLocale::Es),
"Lote de traducciones completado"
);
}
#[test]
fn batch_translates_only_missing_languages_and_publishes() {
let db = Database::open_in_memory().unwrap();
@@ -388,6 +495,9 @@ mod tests {
assert_eq!(requested, vec!["de", "fr"]);
assert_eq!(report.translated_posts, 2);
let canonical = crate::db::queries::post::get_post_by_id(db.conn(), &post.id).unwrap();
assert_eq!(canonical.status, PostStatus::Published);
assert!(canonical.content.is_none());
for language in ["de", "fr"] {
let translation = post_translation::get_post_translation_by_post_and_language(
db.conn(),
@@ -401,6 +511,52 @@ mod tests {
}
}
#[test]
fn reactive_language_translation_is_a_no_op_when_translation_exists() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
ensure_fts_tables(db.conn()).unwrap();
insert_project(db.conn(), &make_test_project("p1", "blog")).unwrap();
let dir = TempDir::new().unwrap();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Hello",
Some("Body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
upsert_translation(
db.conn(),
dir.path(),
&post.id,
"de",
"Hallo",
None,
Some("Inhalt"),
)
.unwrap();
let report = translate_missing_language_for_post(
db.conn(),
dir.path(),
&post.id,
&["en".to_string(), "de".to_string()],
"de",
true,
|| false,
)
.unwrap();
assert!(report.nothing_to_do);
assert_eq!(report.translated_posts, 0);
}
#[test]
fn skips_do_not_translate_posts() {
let db = Database::open_in_memory().unwrap();

View File

@@ -1,5 +1,6 @@
use std::fs;
use std::path::Path;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
@@ -9,12 +10,13 @@ use crate::db::DbConnection as Connection;
use crate::db::queries::script as script_queries;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Post, Script, ScriptKind};
use crate::scripting::{self, ExecutionControl, ExecutionKind};
use crate::scripting::{self, CoreHost, ExecutionControl, ExecutionKind, HostApi, UnavailableHost};
const MAX_TITLE_LENGTH: usize = 200;
const MAX_URL_LENGTH: usize = 2_048;
const MAX_TOASTS_TOTAL: usize = 20;
const MAX_TOAST_LENGTH: usize = 300;
const BLOGMARK_SCHEME: &str = "ruds";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BlogmarkCandidate {
@@ -36,7 +38,7 @@ pub struct BlogmarkImportResult {
pub fn parse_deep_link(raw: &str) -> EngineResult<BlogmarkCandidate> {
let parsed =
Url::parse(raw).map_err(|_| EngineError::Validation("invalid blogmark URL".into()))?;
if parsed.scheme() != "bds2" {
if parsed.scheme() != BLOGMARK_SCHEME {
return Err(EngineError::Validation(
"unsupported blogmark scheme".into(),
));
@@ -66,11 +68,40 @@ pub fn parse_deep_link(raw: &str) -> EngineResult<BlogmarkCandidate> {
})
}
pub fn bookmarklet(project_id: &str) -> String {
let project_id =
url::form_urlencoded::byte_serialize(project_id.as_bytes()).collect::<String>();
format!(
"javascript:(()=>{{const t=encodeURIComponent(document.title||'');const u=encodeURIComponent(location.href||'');location.href='ruds://new-post?title='+t+'&url='+u+'&project_id={project_id}';}})();"
)
}
pub fn receive_deep_link(
conn: &Connection,
data_dir: &Path,
project_id: &str,
raw: &str,
) -> EngineResult<BlogmarkImportResult> {
let host = CoreHost::from_connection(conn, project_id, data_dir)
.map(|host| Arc::new(host) as Arc<dyn HostApi>)
.unwrap_or_else(|_| Arc::new(UnavailableHost));
receive_deep_link_with_host(
conn,
data_dir,
project_id,
raw,
&ExecutionControl::default(),
host,
)
}
pub fn receive_deep_link_with_host(
conn: &Connection,
data_dir: &Path,
project_id: &str,
raw: &str,
control: &ExecutionControl,
host: Arc<dyn HostApi>,
) -> EngineResult<BlogmarkImportResult> {
let mut candidate = parse_deep_link(raw)?;
if let Some(target) = &candidate.project_id
@@ -89,7 +120,10 @@ pub fn receive_deep_link(
));
}
let (mut candidate, toasts, transform_errors) =
run_transforms(conn, data_dir, project_id, candidate)?;
run_transforms(conn, data_dir, project_id, candidate, control, host)?;
if control.is_cancelled() {
return Err(EngineError::Validation("script cancelled".into()));
}
if candidate.categories.is_empty() {
let metadata = crate::engine::meta::read_project_json(data_dir)?;
if let Some(category) = metadata
@@ -124,6 +158,8 @@ fn run_transforms(
data_dir: &Path,
project_id: &str,
candidate: BlogmarkCandidate,
control: &ExecutionControl,
host: Arc<dyn HostApi>,
) -> EngineResult<(BlogmarkCandidate, Vec<String>, Vec<String>)> {
let mut transforms = script_queries::list_scripts_by_project(conn, project_id)?
.into_iter()
@@ -140,6 +176,9 @@ fn run_transforms(
let mut toasts = Vec::new();
let mut errors = Vec::new();
for script in transforms {
if control.is_cancelled() {
return Err(EngineError::Validation("script cancelled".into()));
}
if script.entrypoint.trim().is_empty() {
continue;
}
@@ -148,12 +187,13 @@ fn run_transforms(
"source": "blogmark",
"url": current.get("url").cloned().unwrap_or(Value::Null),
});
match scripting::execute_many(
match scripting::execute_many_with_host(
&source,
&script.entrypoint,
&[current.clone(), context],
ExecutionKind::Transform,
&ExecutionControl::default(),
control,
Arc::clone(&host),
) {
Ok(execution) => {
let (next, returned_toasts) = split_transform_result(execution.value, &current);
@@ -163,6 +203,9 @@ fn run_transforms(
execution.toasts.into_iter().chain(returned_toasts),
);
}
Err(_error) if control.is_cancelled() => {
return Err(EngineError::Validation("script cancelled".into()));
}
Err(error) => errors.push(format!("{}: {error}", script.slug)),
}
}
@@ -278,10 +321,26 @@ mod tests {
use crate::model::ScriptKind;
use tempfile::TempDir;
struct PostsHost;
impl HostApi for PostsHost {
fn call(
&self,
namespace: &str,
method: &str,
_arguments: Vec<Value>,
) -> Result<Value, String> {
match (namespace, method) {
("posts", "get_all") => Ok(json!([{"id":"one"}, {"id":"two"}])),
_ => Err("unsupported test capability".into()),
}
}
}
#[test]
fn parses_and_hardens_blogmark_links() {
let candidate = parse_deep_link(
"bds2://new-post?title=%00Hello&url=https%3A%2F%2Fuser%3Apass%40example.com%2Fa%23frag&tags=one%2Ctwo",
"ruds://new-post?title=%00Hello&url=https%3A%2F%2Fuser%3Apass%40example.com%2Fa%23frag&tags=one%2Ctwo",
)
.unwrap();
assert_eq!(candidate.title, "Hello");
@@ -292,13 +351,22 @@ mod tests {
#[test]
fn rejects_other_schemes_and_actions() {
assert!(parse_deep_link("bds://new-post?title=x").is_err());
assert!(parse_deep_link("bds2://other?title=x").is_err());
assert!(parse_deep_link("bds2://new-post?title=x").is_err());
assert!(parse_deep_link("ruds://other?title=x").is_err());
}
#[test]
fn bookmarklet_targets_only_ruds_and_the_selected_project() {
let value = bookmarklet("project & seven");
assert!(value.contains("ruds://new-post?"));
assert!(value.contains("project_id=project+%26+seven"));
assert!(!value.contains("bds2://"));
}
#[test]
fn invalid_source_url_never_reaches_candidate() {
let candidate =
parse_deep_link("bds2://new-post?title=x&url=javascript%3Aalert%281%29").unwrap();
parse_deep_link("ruds://new-post?title=x&url=javascript%3Aalert%281%29").unwrap();
assert!(candidate.url.is_none());
}
@@ -328,7 +396,7 @@ mod tests {
db.conn(),
directory.path(),
&project.id,
"bds2://new-post?title=Example&url=https%3A%2F%2Fexample.com",
"ruds://new-post?title=Example&url=https%3A%2F%2Fexample.com",
)
.unwrap();
@@ -340,4 +408,80 @@ mod tests {
assert_eq!(result.toasts, vec!["done"]);
assert!(result.transform_errors.is_empty());
}
#[test]
fn transforms_use_the_supplied_project_host() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
crate::db::fts::ensure_fts_tables(db.conn()).unwrap();
let directory = TempDir::new().unwrap();
let project = crate::engine::project::create_project(
db.conn(),
"Blog",
Some(directory.path().to_str().unwrap()),
)
.unwrap();
crate::engine::script::create_script(
db.conn(),
&project.id,
"Count posts",
ScriptKind::Transform,
"function main(data) data.title = tostring(#bds.posts.get_all()) .. ' posts'; return data end",
None,
)
.unwrap();
let result = receive_deep_link_with_host(
db.conn(),
directory.path(),
&project.id,
"ruds://new-post?title=Example",
&ExecutionControl::default(),
Arc::new(PostsHost),
)
.unwrap();
assert_eq!(result.post.title, "2 posts");
}
#[test]
fn cancellation_stops_transform_import_before_post_creation() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
crate::db::fts::ensure_fts_tables(db.conn()).unwrap();
let directory = TempDir::new().unwrap();
let project = crate::engine::project::create_project(
db.conn(),
"Blog",
Some(directory.path().to_str().unwrap()),
)
.unwrap();
crate::engine::script::create_script(
db.conn(),
&project.id,
"Cancelled",
ScriptKind::Transform,
"function main(data) return data end",
None,
)
.unwrap();
let control = ExecutionControl::default();
control.cancel();
assert!(
receive_deep_link_with_host(
db.conn(),
directory.path(),
&project.id,
"ruds://new-post?title=Example",
&control,
Arc::new(PostsHost),
)
.is_err()
);
assert_eq!(
crate::db::queries::post::count_posts_by_project(db.conn(), &project.id).unwrap(),
0
);
}
}

View File

@@ -6,7 +6,7 @@ use std::path::Path;
use crate::db::DbConnection as Connection;
use crate::db::queries::post as post_q;
use crate::engine::EngineResult;
use crate::engine::{EngineError, EngineResult};
use crate::model::PostStatus;
use crate::util::{atomic_write, timestamp};
@@ -18,6 +18,15 @@ pub fn regenerate_calendar(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
regenerate_calendar_with_progress(conn, data_dir, project_id, |_, _, _| true)
}
pub fn regenerate_calendar_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
mut on_item: impl FnMut(usize, usize, &str) -> bool,
) -> EngineResult<()> {
let posts = post_q::list_posts_by_project(conn, project_id)?;
@@ -25,7 +34,11 @@ pub fn regenerate_calendar(
let mut months: BTreeMap<String, u32> = BTreeMap::new();
let mut days: BTreeMap<String, u32> = BTreeMap::new();
for post in &posts {
let total = posts.len() + 1;
for (index, post) in posts.iter().enumerate() {
if !on_item(index + 1, total, &post.title) {
return Err(EngineError::Cancelled);
}
if post.status != PostStatus::Published {
continue;
}
@@ -41,6 +54,10 @@ pub fn regenerate_calendar(
*days.entry(day_key).or_insert(0) += 1;
}
if !on_item(total, total, "calendar.json") {
return Err(EngineError::Cancelled);
}
let calendar = serde_json::json!({
"years": years,
"months": months,
@@ -82,4 +99,20 @@ mod tests {
serde_json::from_str(&std::fs::read_to_string(&cal_path).unwrap()).unwrap();
assert!(data["years"].as_object().unwrap().is_empty());
}
#[test]
fn cancellation_before_write_leaves_no_calendar() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
let dir = tempfile::tempdir().unwrap();
let project =
engine::project::create_project(db.conn(), "Test", Some(dir.path().to_str().unwrap()))
.unwrap();
let result =
regenerate_calendar_with_progress(db.conn(), dir.path(), &project.id, |_, _, _| false);
assert!(matches!(result, Err(EngineError::Cancelled)));
assert!(!dir.path().join("html/calendar.json").exists());
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,583 @@
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::model::ChatMessage;
pub const RENDER_TOOL_NAMES: [&str; 8] = [
"render_card",
"render_chart",
"render_form",
"render_list",
"render_metric",
"render_mindmap",
"render_table",
"render_tabs",
];
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
pub struct ChatSurfaceState {
#[serde(default)]
pub surface_data: BTreeMap<String, BTreeMap<String, Value>>,
#[serde(default)]
pub surface_tabs: BTreeMap<String, usize>,
#[serde(default)]
pub dismissed_surfaces: BTreeSet<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SurfaceKind {
Card,
Chart,
Form,
List,
Metric,
Mindmap,
Table,
Tabs,
Text,
Json,
}
impl SurfaceKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Card => "card",
Self::Chart => "chart",
Self::Form => "form",
Self::List => "list",
Self::Metric => "metric",
Self::Mindmap => "mindmap",
Self::Table => "table",
Self::Tabs => "tabs",
Self::Text => "text",
Self::Json => "json",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChartType {
Bar,
StackedBar,
Line,
Area,
Pie,
Donut,
Heatmap,
}
impl ChartType {
pub const fn as_str(self) -> &'static str {
match self {
Self::Bar => "bar",
Self::StackedBar => "stacked-bar",
Self::Line => "line",
Self::Area => "area",
Self::Pie => "pie",
Self::Donut => "donut",
Self::Heatmap => "heatmap",
}
}
fn parse(value: Option<&str>) -> Self {
match value {
Some("stacked-bar") => Self::StackedBar,
Some("line") => Self::Line,
Some("area") => Self::Area,
Some("pie") => Self::Pie,
Some("donut") => Self::Donut,
Some("heatmap") => Self::Heatmap,
_ => Self::Bar,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FormInputType {
Text,
Textarea,
Select,
Checkbox,
Date,
Number,
}
impl FormInputType {
fn parse(value: Option<&str>) -> Self {
match value {
Some("textarea") => Self::Textarea,
Some("select") => Self::Select,
Some("checkbox") => Self::Checkbox,
Some("date") => Self::Date,
Some("number") => Self::Number,
_ => Self::Text,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct InlineSurface {
pub id: String,
pub kind: SurfaceKind,
pub title: Option<String>,
pub subtitle: Option<String>,
pub body: Option<String>,
pub actions: Vec<SurfaceAction>,
pub columns: Vec<String>,
pub rows: Vec<Vec<String>>,
pub chart_type: Option<ChartType>,
pub series: Vec<ChartSeries>,
pub max_value: Option<f64>,
pub label: Option<String>,
pub value: Option<String>,
pub items: Vec<String>,
pub nodes: Vec<MindmapNode>,
pub fields: Vec<FormField>,
pub submit_label: Option<String>,
pub submit_action: Option<String>,
pub tabs: Vec<TabPanel>,
pub selected_index: Option<usize>,
pub raw: Option<Value>,
}
impl InlineSurface {
fn empty(id: String, kind: SurfaceKind) -> Self {
Self {
id,
kind,
title: None,
subtitle: None,
body: None,
actions: vec![],
columns: vec![],
rows: vec![],
chart_type: None,
series: vec![],
max_value: None,
label: None,
value: None,
items: vec![],
nodes: vec![],
fields: vec![],
submit_label: None,
submit_action: None,
tabs: vec![],
selected_index: None,
raw: None,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct SurfaceAction {
pub label: String,
pub action: String,
pub payload: Value,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ChartSeries {
pub label: String,
pub value: f64,
pub segments: Vec<ChartSegment>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ChartSegment {
pub label: String,
pub value: f64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct MindmapNode {
pub id: Option<String>,
pub label: String,
pub children: Vec<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FormField {
pub key: String,
pub label: String,
pub input_type: FormInputType,
pub placeholder: Option<String>,
pub value: Value,
pub options: Vec<FieldOption>,
pub required: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FieldOption {
pub label: String,
pub value: String,
}
impl std::fmt::Display for FieldOption {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.label)
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct TabPanel {
pub label: String,
pub content: Vec<InlineSurface>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SurfaceNavigation {
pub destination: String,
pub entity_id: Option<String>,
}
pub fn build_message_surfaces(
message: &ChatMessage,
state: &ChatSurfaceState,
) -> Vec<InlineSurface> {
let Some(raw) = message.tool_calls.as_deref() else {
return vec![];
};
let Ok(calls) = serde_json::from_str::<Vec<Value>>(raw) else {
return vec![];
};
calls
.iter()
.enumerate()
.filter_map(|(index, call)| {
let function = call.get("function").unwrap_or(call);
let name = function.get("name")?.as_str()?;
let raw = function
.get("arguments")
.cloned()
.unwrap_or_else(|| json!({}));
let arguments = raw
.as_str()
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(raw);
build_render_surface(
name,
&arguments,
format!("{}-surface-{index}", message.id),
state,
)
})
.filter(|surface| !state.dismissed_surfaces.contains(&surface.id))
.collect()
}
pub fn build_render_surface(
name: &str,
arguments: &Value,
id: String,
state: &ChatSurfaceState,
) -> Option<InlineSurface> {
match name {
"render_card" => Some(card(arguments, id)),
"render_chart" => Some(chart(arguments, id)),
"render_form" => Some(form(arguments, id, state)),
"render_list" => Some(list(arguments, id)),
"render_metric" => Some(metric(arguments, id)),
"render_mindmap" => Some(mindmap(arguments, id)),
"render_table" => Some(table(arguments, id)),
"render_tabs" => Some(tabs(arguments, id, state)),
unknown if unknown.starts_with("render_") => {
let mut surface = InlineSurface::empty(id, SurfaceKind::Json);
surface.raw = Some(arguments.clone());
Some(surface)
}
_ => None,
}
}
pub fn merge_form_data(payload: Value, surface_id: &str, state: &ChatSurfaceState) -> Value {
let mut result = payload.as_object().cloned().unwrap_or_default();
if let Some(values) = state.surface_data.get(surface_id) {
result.insert(
"formData".into(),
Value::Object(values.clone().into_iter().collect()),
);
}
Value::Object(result)
}
pub fn resolve_surface_action(action: &str, payload: &Value) -> Result<SurfaceNavigation, String> {
let map = payload
.as_object()
.ok_or_else(|| "surface action payload must be an object".to_string())?;
let normalized = action.replace('_', "").to_ascii_lowercase();
let open = |destination: &str, keys: &[&str]| {
let id = keys
.iter()
.find_map(|key| map.get(*key).and_then(Value::as_str))
.filter(|id| !id.trim().is_empty())
.ok_or_else(|| format!("surface action {action} requires an identifier"))?;
Ok(SurfaceNavigation {
destination: destination.into(),
entity_id: Some(id.into()),
})
};
match normalized.as_str() {
"openpost" => open("posts", &["postId", "post_id", "id"]),
"openmedia" => open("media", &["mediaId", "media_id", "id"]),
"openchat" => open(
"chat",
&[
"conversationId",
"conversation_id",
"chatId",
"chat_id",
"id",
],
),
"opensettings" => Ok(navigation("settings")),
"switchview" | "setview" | "setactiveview" => {
let view = map
.get("view")
.or_else(|| map.get("destination"))
.and_then(Value::as_str)
.filter(|view| {
[
"posts",
"pages",
"media",
"templates",
"scripts",
"tags",
"chat",
"import",
"git",
"settings",
]
.contains(view)
})
.ok_or_else(|| format!("surface action {action} requires a valid view"))?;
Ok(navigation(view))
}
"togglesidebar" => Ok(navigation("toggle_sidebar")),
"togglepanel" | "openpanel" => Ok(navigation("toggle_panel")),
"toggleassistantsidebar" => Ok(navigation("toggle_assistant_sidebar")),
_ => Err(format!("unsupported surface action: {action}")),
}
}
fn navigation(destination: &str) -> SurfaceNavigation {
SurfaceNavigation {
destination: destination.into(),
entity_id: None,
}
}
fn card(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Card);
s.title = string(a, "title");
s.subtitle = string(a, "subtitle");
s.body = string(a, "body");
s.actions = array(a, "actions")
.iter()
.map(|v| SurfaceAction {
label: string(v, "label").unwrap_or_default(),
action: string(v, "action").unwrap_or_default(),
payload: v.get("payload").cloned().unwrap_or_else(|| json!({})),
})
.collect();
s
}
fn chart(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Chart);
s.title = string(a, "title");
s.chart_type = Some(ChartType::parse(
a.get("chartType")
.or_else(|| a.get("chart_type"))
.and_then(Value::as_str),
));
s.series = array(a, "series")
.iter()
.map(|v| ChartSeries {
label: string(v, "label").unwrap_or_default(),
value: numeric(v.get("value")),
segments: array(v, "segments")
.iter()
.map(|seg| ChartSegment {
label: string(seg, "label").unwrap_or_default(),
value: numeric(seg.get("value")),
})
.collect(),
})
.collect();
s.max_value = Some(s.series.iter().map(|v| v.value).fold(0.0_f64, f64::max));
s
}
fn form(a: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface {
let mut s = InlineSurface::empty(id.clone(), SurfaceKind::Form);
s.title = string(a, "title");
s.fields = array(a, "fields")
.iter()
.map(|v| {
let key = string(v, "key").unwrap_or_else(|| "field".into());
let default = v
.get("defaultValue")
.or_else(|| v.get("default_value"))
.cloned()
.unwrap_or(Value::Null);
FormField {
label: string(v, "label").unwrap_or_else(|| key.clone()),
input_type: FormInputType::parse(
v.get("inputType")
.or_else(|| v.get("input_type"))
.and_then(Value::as_str),
),
placeholder: string(v, "placeholder"),
value: state
.surface_data
.get(&id)
.and_then(|m| m.get(&key))
.cloned()
.unwrap_or(default),
options: array(v, "options")
.iter()
.map(|o| FieldOption {
label: string(o, "label").unwrap_or_default(),
value: string(o, "value").unwrap_or_default(),
})
.collect(),
required: v.get("required").and_then(Value::as_bool).unwrap_or(false),
key,
}
})
.collect();
s.submit_label = a
.get("submitLabel")
.or_else(|| a.get("submit_label"))
.and_then(Value::as_str)
.map(str::to_owned);
s.submit_action = a
.get("submitAction")
.or_else(|| a.get("submit_action"))
.and_then(Value::as_str)
.map(str::to_owned);
s
}
fn list(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::List);
s.title = string(a, "title");
s.items = strings(a.get("items"));
s
}
fn metric(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Metric);
s.label = string(a, "label");
s.value = scalar(a.get("value"));
s
}
fn mindmap(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Mindmap);
s.title = string(a, "title");
s.nodes = array(a, "nodes")
.iter()
.map(|v| MindmapNode {
id: string(v, "id"),
label: string(v, "label").unwrap_or_default(),
children: strings(v.get("children")),
})
.collect();
s
}
fn table(a: &Value, id: String) -> InlineSurface {
let mut s = InlineSurface::empty(id, SurfaceKind::Table);
s.title = string(a, "title");
s.columns = strings(a.get("columns"));
s.rows = array(a, "rows").iter().map(|v| strings(Some(v))).collect();
s
}
fn tabs(a: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface {
let mut s = InlineSurface::empty(id.clone(), SurfaceKind::Tabs);
s.title = string(a, "title");
s.tabs = array(a, "tabs")
.iter()
.enumerate()
.map(|(ti, tab)| TabPanel {
label: string(tab, "label").unwrap_or_default(),
content: array(tab, "content")
.iter()
.enumerate()
.map(|(ci, v)| nested(v, format!("{id}-tab-{ti}-{ci}"), state))
.collect(),
})
.collect();
s.selected_index = Some(
state
.surface_tabs
.get(&id)
.copied()
.unwrap_or(0)
.min(s.tabs.len().saturating_sub(1)),
);
s
}
fn nested(v: &Value, id: String, state: &ChatSurfaceState) -> InlineSurface {
let Some(object) = v.as_object() else {
let mut s = InlineSurface::empty(id, SurfaceKind::Text);
s.body = Some(scalar(Some(v)).unwrap_or_default());
return s;
};
match object.get("type").and_then(Value::as_str).unwrap_or("text") {
"card" => card(v, id),
"chart" => chart(v, id),
"form" => form(v, id, state),
"list" => list(v, id),
"metric" => metric(v, id),
"mindmap" => mindmap(v, id),
"table" => table(v, id),
"tabs" => tabs(v, id, state),
"text" => {
let mut s = InlineSurface::empty(id, SurfaceKind::Text);
s.body = string(v, "body").or_else(|| string(v, "text"));
s
}
_ => {
let mut s = InlineSurface::empty(id, SurfaceKind::Json);
s.raw = Some(Value::Object(object.clone()));
s
}
}
}
fn array<'a>(v: &'a Value, key: &str) -> &'a [Value] {
v.get(key)
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or_default()
}
fn string(v: &Value, key: &str) -> Option<String> {
scalar(v.get(key))
}
fn strings(v: Option<&Value>) -> Vec<String> {
v.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|v| scalar(Some(v)))
.collect()
}
fn scalar(v: Option<&Value>) -> Option<String> {
match v? {
Value::Null => None,
Value::String(v) => Some(v.clone()),
Value::Bool(v) => Some(v.to_string()),
Value::Number(v) => Some(v.to_string()),
v => serde_json::to_string(v).ok(),
}
}
fn numeric(v: Option<&Value>) -> f64 {
v.and_then(|v| {
v.as_f64()
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
})
.filter(|v| v.is_finite())
.unwrap_or(0.0)
}
pub fn render_tool_result(name: &str, arguments: &Value) -> Option<Value> {
let kind = name.strip_prefix("render_")?;
RENDER_TOOL_NAMES.contains(&name).then(|| {
let mut values = arguments.as_object().cloned().unwrap_or_default();
values.insert("type".into(), Value::String(kind.into()));
Value::Object(values)
})
}

View File

@@ -0,0 +1,978 @@
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::Path;
use base64::Engine as _;
use diesel::prelude::*;
use serde_json::{Value, json};
use crate::db::DbConnection as Connection;
use crate::db::queries::{media, post, post_link, post_media, script, template};
use crate::db::schema::ai_models;
use crate::engine::{EngineError, EngineResult, chat_surfaces};
use crate::util::frontmatter::read_post_file;
pub fn model_supports_tools(conn: &Connection, model: &str) -> EngineResult<bool> {
let catalog_value = conn.with(|connection| {
ai_models::table
.filter(ai_models::model_id.eq(model))
.select(ai_models::tool_call)
.first::<i32>(connection)
.optional()
})?;
Ok(catalog_value.map_or_else(
|| {
let model = model.to_ascii_lowercase();
model.contains("gpt")
|| model.contains("claude")
|| model.contains("tool")
|| model.contains("qwen")
|| model.contains("mistral")
},
|value| value != 0,
))
}
pub fn system_prompt(conn: &Connection, project_id: &str) -> EngineResult<String> {
let posts = post::list_posts_by_project(conn, project_id)?;
let media_count = media::count_media_by_project(conn, project_id)?;
let tags = posts
.iter()
.flat_map(|post| post.tags.iter())
.collect::<BTreeSet<_>>()
.len();
let categories = posts
.iter()
.flat_map(|post| post.categories.iter())
.collect::<BTreeSet<_>>()
.len();
let configured = crate::engine::settings::get(conn, "ai.system_prompt")?.unwrap_or_default();
let contract = format!(
"You are the conversational assistant for this blog project. Use tools when facts from the project are needed. Never invent project content or identifiers. There are {} posts, {media_count} media items, {tags} tags, and {categories} categories. Keep answers concise and use GitHub-flavored Markdown when useful. Use render tools for structured data, comparisons, and forms; their payloads are native UI data, never HTML or JavaScript.",
posts.len()
);
Ok(if configured.trim().is_empty() {
contract
} else {
format!("{}\n\n{contract}", configured.trim())
})
}
pub fn tool_specs() -> Vec<Value> {
vec![
spec(
"get_blog_stats",
"Return aggregate project statistics.",
json!({}),
),
spec(
"check_term",
"Check whether a term is used as a tag or category.",
json!({"term": {"type": "string"}}),
),
spec(
"search_posts",
"Search post titles, slugs, excerpts, bodies, tags, and categories.",
json!({"query": {"type": "string"}, "language": {"type": "string"}, "limit": {"type": "integer"}}),
),
spec(
"read_post",
"Read one post by id.",
json!({"post_id": {"type": "string"}}),
),
spec(
"read_post_by_slug",
"Read one post by slug.",
json!({"slug": {"type": "string"}}),
),
spec(
"list_posts",
"List posts, optionally filtering by status, tag, or category.",
json!({"status": {"type": "string"}, "tag": {"type": "string"}, "category": {"type": "string"}, "limit": {"type": "integer"}}),
),
spec(
"count_posts",
"Count posts and optionally group by status, tag, or category.",
json!({"group_by": {"type": "string", "enum": ["status", "tag", "category"]}}),
),
spec(
"update_post_metadata",
"Update title, excerpt, tags, or categories on a post.",
json!({"post_id": {"type": "string"}, "title": {"type": "string"}, "excerpt": {"type": ["string", "null"]}, "tags": {"type": "array", "items": {"type": "string"}}, "categories": {"type": "array", "items": {"type": "string"}}}),
),
spec(
"list_media",
"List project media.",
json!({"limit": {"type": "integer"}}),
),
spec(
"get_media",
"Get one media item.",
json!({"media_id": {"type": "string"}}),
),
spec(
"view_image",
"Return a local image thumbnail as a data URL for visual inspection.",
json!({"media_id": {"type": "string"}, "size": {"type": "string", "enum": ["small", "medium", "large"]}}),
),
spec(
"update_media_metadata",
"Update title, alt text, caption, or tags on media.",
json!({"media_id": {"type": "string"}, "title": {"type": ["string", "null"]}, "alt": {"type": ["string", "null"]}, "caption": {"type": ["string", "null"]}, "tags": {"type": "array", "items": {"type": "string"}}}),
),
spec("list_tags", "List tags and usage counts.", json!({})),
spec(
"list_categories",
"List categories and usage counts.",
json!({}),
),
spec(
"get_post_backlinks",
"Get posts that link to a post.",
json!({"post_id": {"type": "string"}}),
),
spec(
"get_post_outlinks",
"Get posts linked from a post.",
json!({"post_id": {"type": "string"}}),
),
spec(
"get_post_media",
"Get media linked to a post.",
json!({"post_id": {"type": "string"}}),
),
spec(
"get_media_posts",
"Get posts that use a media item.",
json!({"media_id": {"type": "string"}}),
),
spec("list_templates", "List templates.", json!({})),
spec(
"read_template",
"Read a template by id.",
json!({"template_id": {"type": "string"}}),
),
spec("list_scripts", "List scripts.", json!({})),
spec(
"read_script",
"Read a script by id.",
json!({"script_id": {"type": "string"}}),
),
spec(
"navigate",
"Open a project area or entity in the application.",
json!({
"action": {"type": "string", "enum": ["open_post", "open_media", "open_settings", "open_chat", "switch_view", "toggle_sidebar", "toggle_panel", "toggle_assistant_sidebar"]},
"destination": {"type": "string", "enum": ["posts", "pages", "media", "templates", "scripts", "tags", "chat", "import", "git", "settings"]},
"entity_id": {"type": "string"},
"value": {"type": "string"}
}),
),
spec(
"render_card",
"Render an information card with optional allow-listed actions.",
json!({
"title": {"type": "string"}, "subtitle": {"type": "string"}, "body": {"type": "string"},
"actions": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "action": {"type": "string"}, "payload": {"type": "object"}
}, "required": ["label", "action"]}}
}),
),
spec(
"render_chart",
"Render a native chart; heatmap and stacked-bar series use labelled segments.",
json!({
"chartType": {"type": "string", "enum": ["bar", "stacked-bar", "line", "area", "pie", "donut", "heatmap"]},
"chart_type": {"type": "string", "enum": ["bar", "stacked-bar", "line", "area", "pie", "donut", "heatmap"]},
"title": {"type": "string"},
"series": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "value": {"type": "number"},
"segments": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "value": {"type": "number"}
}, "required": ["label", "value"]}}
}, "required": ["label"]}}
}),
),
spec(
"render_form",
"Render a native form that submits its current values with an allow-listed action.",
json!({
"title": {"type": "string"},
"fields": {"type": "array", "items": {"type": "object", "properties": {
"key": {"type": "string"}, "label": {"type": "string"},
"inputType": {"type": "string", "enum": ["text", "textarea", "select", "checkbox", "date", "number"]},
"input_type": {"type": "string", "enum": ["text", "textarea", "select", "checkbox", "date", "number"]},
"placeholder": {"type": "string"}, "defaultValue": {}, "default_value": {}, "required": {"type": "boolean"},
"options": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "value": {"type": "string"}
}}}
}, "required": ["key", "label", "inputType"]}},
"submitLabel": {"type": "string"}, "submit_label": {"type": "string"},
"submitAction": {"type": "string"}, "submit_action": {"type": "string"}
}),
),
spec(
"render_list",
"Render a native list.",
json!({"title": {"type": "string"}, "items": {"type": "array", "items": {"type": "string"}}}),
),
spec(
"render_metric",
"Render a prominent metric.",
json!({"label": {"type": "string"}, "value": {"type": "string"}}),
),
spec(
"render_mindmap",
"Render a native hierarchical mind map.",
json!({"title": {"type": "string"}, "nodes": {"type": "array", "items": {"type": "object", "properties": {
"id": {"type": "string"}, "label": {"type": "string"}, "children": {"type": "array", "items": {"type": "string"}}
}, "required": ["label"]}}}),
),
spec(
"render_table",
"Render a native data table.",
json!({
"title": {"type": "string"}, "columns": {"type": "array", "items": {"type": "string"}},
"rows": {"type": "array", "items": {"type": "array", "items": {"type": "string"}}}
}),
),
spec(
"render_tabs",
"Render switchable tabs containing nested native surfaces or text.",
json!({"title": {"type": "string"}, "tabs": {"type": "array", "items": {"type": "object", "properties": {
"label": {"type": "string"}, "content": {"type": "array", "items": {"type": "object"}}
}, "required": ["label", "content"]}}}),
),
]
}
pub fn execute(
conn: &Connection,
data_dir: &Path,
project_id: &str,
name: &str,
arguments: &Value,
) -> EngineResult<Value> {
match name {
"blog_stats" | "get_blog_stats" => blog_stats(conn, project_id),
"check_term" => check_term(conn, project_id, required_str(arguments, "term")?),
"search_posts" => search_posts(conn, project_id, arguments),
"read_post" => {
let item = post::get_post_by_id(conn, required_id(arguments, "post_id", "postId")?)?;
ensure_project(&item.project_id, project_id)?;
post_detail(data_dir, item)
}
"read_post_by_slug" => {
let item = post::get_post_by_project_and_slug(
conn,
project_id,
required_str(arguments, "slug")?,
)?;
post_detail(data_dir, item)
}
"list_posts" => list_posts(conn, project_id, arguments),
"count_posts" => count_posts(conn, project_id, arguments),
"update_post_metadata" => update_post_metadata(conn, data_dir, project_id, arguments),
"list_media" => list_media(conn, project_id, arguments),
"get_media" => {
let item =
media::get_media_by_id(conn, required_id(arguments, "media_id", "mediaId")?)?;
ensure_project(&item.project_id, project_id)?;
Ok(json!({"success": true, "media": item}))
}
"view_image" => view_image(conn, data_dir, project_id, arguments),
"update_media_metadata" => update_media_metadata(conn, data_dir, project_id, arguments),
"list_tags" => counted_terms(conn, project_id, true),
"list_categories" => counted_terms(conn, project_id, false),
"get_post_backlinks" => post_links(conn, project_id, arguments, true),
"get_post_outlinks" => post_links(conn, project_id, arguments, false),
"get_post_media" => linked_media(conn, project_id, arguments),
"get_media_posts" => linked_posts(conn, project_id, arguments),
"list_templates" => Ok(json!({
"templates": template::list_templates_by_project(conn, project_id)?,
})),
"read_template" => {
let item = template::get_template_by_id(
conn,
required_id(arguments, "template_id", "templateId")?,
)?;
ensure_project(&item.project_id, project_id)?;
Ok(json!({"success": true, "template": item}))
}
"list_scripts" => Ok(json!({
"scripts": script::list_scripts_by_project(conn, project_id)?,
})),
"read_script" => {
let item =
script::get_script_by_id(conn, required_id(arguments, "script_id", "scriptId")?)?;
ensure_project(&item.project_id, project_id)?;
Ok(json!({"success": true, "script": item}))
}
"navigate" => navigate(arguments),
name if chat_surfaces::RENDER_TOOL_NAMES.contains(&name) => {
Ok(chat_surfaces::render_tool_result(name, arguments)
.expect("render tool allow-list and result builder must stay in sync"))
}
_ => Ok(json!({"success": false, "error": "unknown_tool", "name": name})),
}
}
fn blog_stats(conn: &Connection, project_id: &str) -> EngineResult<Value> {
let posts = post::list_posts_by_project(conn, project_id)?;
let media_count = media::count_media_by_project(conn, project_id)?;
let templates = template::list_templates_by_project(conn, project_id)?.len();
let scripts = script::list_scripts_by_project(conn, project_id)?.len();
let tags = posts
.iter()
.flat_map(|item| &item.tags)
.collect::<BTreeSet<_>>()
.len();
let categories = posts
.iter()
.flat_map(|item| &item.categories)
.collect::<BTreeSet<_>>()
.len();
Ok(json!({
"posts": posts.len(), "media": media_count, "templates": templates,
"scripts": scripts, "tags": tags, "categories": categories,
}))
}
fn check_term(conn: &Connection, project_id: &str, term: &str) -> EngineResult<Value> {
let term = term.to_lowercase();
let posts = post::list_posts_by_project(conn, project_id)?;
let tag_count = posts
.iter()
.filter(|item| item.tags.iter().any(|value| value.to_lowercase() == term))
.count();
let category_count = posts
.iter()
.filter(|item| {
item.categories
.iter()
.any(|value| value.to_lowercase() == term)
})
.count();
Ok(json!({"term": term, "tag_posts": tag_count, "category_posts": category_count}))
}
fn search_posts(conn: &Connection, project_id: &str, arguments: &Value) -> EngineResult<Value> {
let query = required_str(arguments, "query")?;
let language = arguments
.get("language")
.and_then(Value::as_str)
.unwrap_or("en");
let limit = limit(arguments);
let mut matches = Vec::new();
for id in crate::db::fts::search_posts(conn, query, language)? {
if let Ok(item) = post::get_post_by_id(conn, &id)
&& item.project_id == project_id
{
matches.push(post_summary(&item));
if matches.len() == limit {
break;
}
}
}
Ok(json!({"posts": matches, "count": matches.len()}))
}
fn list_posts(conn: &Connection, project_id: &str, arguments: &Value) -> EngineResult<Value> {
let status = arguments.get("status").and_then(Value::as_str);
let tag = arguments.get("tag").and_then(Value::as_str);
let category = arguments.get("category").and_then(Value::as_str);
let items = post::list_posts_by_project(conn, project_id)?
.into_iter()
.filter(|item| status.is_none_or(|value| item.status.as_str() == value))
.filter(|item| tag.is_none_or(|value| contains_case_insensitive(&item.tags, value)))
.filter(|item| {
category.is_none_or(|value| contains_case_insensitive(&item.categories, value))
})
.take(limit(arguments))
.map(|item| post_summary(&item))
.collect::<Vec<_>>();
Ok(json!({"posts": items, "count": items.len()}))
}
fn count_posts(conn: &Connection, project_id: &str, arguments: &Value) -> EngineResult<Value> {
let items = post::list_posts_by_project(conn, project_id)?;
let Some(group_by) = arguments
.get("group_by")
.or_else(|| arguments.get("groupBy"))
.and_then(Value::as_str)
else {
return Ok(json!({"total_posts": items.len()}));
};
let mut groups = BTreeMap::<String, usize>::new();
for item in &items {
let values: Vec<String> = match group_by {
"status" => vec![item.status.as_str().to_string()],
"tag" => item.tags.clone(),
"category" => item.categories.clone(),
_ => {
return Err(EngineError::Validation(format!(
"unsupported post grouping: {group_by}"
)));
}
};
for value in values {
*groups.entry(value).or_default() += 1;
}
}
Ok(json!({"total_posts": items.len(), "group_by": group_by, "groups": groups}))
}
fn update_post_metadata(
conn: &Connection,
data_dir: &Path,
project_id: &str,
arguments: &Value,
) -> EngineResult<Value> {
let id = required_id(arguments, "post_id", "postId")?;
let existing = post::get_post_by_id(conn, id)?;
ensure_project(&existing.project_id, project_id)?;
if !["title", "excerpt", "tags", "categories"]
.iter()
.any(|key| arguments.get(key).is_some())
{
return Err(EngineError::Validation(
"no post metadata updates provided".to_string(),
));
}
let excerpt = optional_nullable_str(arguments, "excerpt")?;
let item = crate::engine::post::update_post(
conn,
data_dir,
id,
arguments.get("title").and_then(Value::as_str),
None,
excerpt,
None,
optional_string_array(arguments, "tags")?,
optional_string_array(arguments, "categories")?,
None,
None,
None,
None,
)?;
Ok(json!({"success": true, "post": post_summary(&item)}))
}
fn list_media(conn: &Connection, project_id: &str, arguments: &Value) -> EngineResult<Value> {
let items = media::list_media_by_project(conn, project_id)?
.into_iter()
.take(limit(arguments))
.collect::<Vec<_>>();
Ok(json!({"media": items, "count": items.len()}))
}
fn update_media_metadata(
conn: &Connection,
data_dir: &Path,
project_id: &str,
arguments: &Value,
) -> EngineResult<Value> {
let id = required_id(arguments, "media_id", "mediaId")?;
let existing = media::get_media_by_id(conn, id)?;
ensure_project(&existing.project_id, project_id)?;
if !["title", "alt", "caption", "tags"]
.iter()
.any(|key| arguments.get(key).is_some())
{
return Err(EngineError::Validation(
"no media metadata updates provided".to_string(),
));
}
let item = crate::engine::media::update_media(
conn,
data_dir,
id,
optional_nullable_str(arguments, "title")?,
optional_nullable_str(arguments, "alt")?,
optional_nullable_str(arguments, "caption")?,
None,
None,
optional_string_array(arguments, "tags")?,
)?;
Ok(json!({"success": true, "media": item}))
}
fn view_image(
conn: &Connection,
data_dir: &Path,
project_id: &str,
arguments: &Value,
) -> EngineResult<Value> {
let item = media::get_media_by_id(conn, required_id(arguments, "media_id", "mediaId")?)?;
ensure_project(&item.project_id, project_id)?;
if !item.mime_type.starts_with("image/") {
return Ok(json!({"success": false, "error": "not_image", "mime_type": item.mime_type}));
}
let size = arguments
.get("size")
.and_then(Value::as_str)
.unwrap_or("medium");
if !["small", "medium", "large"].contains(&size) {
return Err(EngineError::Validation(format!(
"unsupported thumbnail size: {size}"
)));
}
let path = data_dir.join(crate::util::thumbnail_path(&item.id, size, "webp"));
if !path.is_file() {
return Ok(json!({"success": false, "error": "thumbnail_not_available"}));
}
let encoded = base64::engine::general_purpose::STANDARD.encode(fs::read(path)?);
Ok(json!({
"success": true,
"media": item,
"data_url": format!("data:image/webp;base64,{encoded}"),
}))
}
fn post_links(
conn: &Connection,
project_id: &str,
arguments: &Value,
incoming: bool,
) -> EngineResult<Value> {
let id = required_id(arguments, "post_id", "postId")?;
let source = post::get_post_by_id(conn, id)?;
ensure_project(&source.project_id, project_id)?;
let links = if incoming {
post_link::list_links_by_target(conn, id)?
} else {
post_link::list_links_by_source(conn, id)?
};
let mut items = Vec::with_capacity(links.len());
for link in links {
let linked_id = if incoming {
&link.source_post_id
} else {
&link.target_post_id
};
let linked = post::get_post_by_id(conn, linked_id)?;
ensure_project(&linked.project_id, project_id)?;
items.push(json!({
"post": post_summary(&linked),
"link_text": link.link_text,
}));
}
if incoming {
Ok(json!({"success": true, "post_id": id, "linked_by": items}))
} else {
Ok(json!({"success": true, "post_id": id, "links_to": items}))
}
}
fn linked_media(conn: &Connection, project_id: &str, arguments: &Value) -> EngineResult<Value> {
let id = required_id(arguments, "post_id", "postId")?;
let item = post::get_post_by_id(conn, id)?;
ensure_project(&item.project_id, project_id)?;
let mut items = Vec::new();
for link in post_media::list_post_media_by_post(conn, id)? {
ensure_project(&link.project_id, project_id)?;
let item = media::get_media_by_id(conn, &link.media_id)?;
ensure_project(&item.project_id, project_id)?;
items.push(json!({"media": item, "sort_order": link.sort_order}));
}
Ok(json!({"success": true, "post_id": id, "media": items}))
}
fn linked_posts(conn: &Connection, project_id: &str, arguments: &Value) -> EngineResult<Value> {
let id = required_id(arguments, "media_id", "mediaId")?;
let item = media::get_media_by_id(conn, id)?;
ensure_project(&item.project_id, project_id)?;
let mut items = Vec::new();
for link in post_media::list_post_media_by_media(conn, id)? {
ensure_project(&link.project_id, project_id)?;
let item = post::get_post_by_id(conn, &link.post_id)?;
ensure_project(&item.project_id, project_id)?;
items.push(json!({"post": post_summary(&item), "sort_order": link.sort_order}));
}
Ok(json!({"success": true, "media_id": id, "posts": items}))
}
fn counted_terms(conn: &Connection, project_id: &str, tags: bool) -> EngineResult<Value> {
let mut counts = BTreeMap::<String, usize>::new();
for item in post::list_posts_by_project(conn, project_id)? {
for term in if tags { &item.tags } else { &item.categories } {
*counts.entry(term.clone()).or_default() += 1;
}
}
let values = counts
.into_iter()
.map(|(name, count)| json!({"name": name, "count": count}))
.collect::<Vec<_>>();
Ok(if tags {
json!({"tags": values, "count": values.len()})
} else {
json!({"categories": values, "count": values.len()})
})
}
fn navigate(arguments: &Value) -> EngineResult<Value> {
let action = arguments.get("action").and_then(Value::as_str);
let value = arguments
.get("value")
.or_else(|| arguments.get("entity_id"))
.or_else(|| arguments.get("entityId"))
.and_then(Value::as_str);
let (destination, entity_id) = match action {
Some("open_post" | "openPost") => ("posts", required_navigation_value(value, "post")?),
Some("open_media" | "openMedia") => ("media", required_navigation_value(value, "media")?),
Some("open_chat" | "openChat") => ("chat", required_navigation_value(value, "chat")?),
Some("open_settings" | "openSettings") => ("settings", None),
Some("switch_view" | "switchView") => {
(required_navigation_value(value, "view")?.unwrap(), None)
}
Some("toggle_sidebar" | "toggleSidebar") => ("toggle_sidebar", None),
Some("toggle_panel" | "togglePanel") => ("toggle_panel", None),
Some("toggle_assistant_sidebar" | "toggleAssistantSidebar") => {
("toggle_assistant_sidebar", None)
}
Some(action) => {
return Err(EngineError::Validation(format!(
"unsupported navigation action: {action}"
)));
}
None => (required_str(arguments, "destination")?, value),
};
if ![
"posts",
"pages",
"media",
"templates",
"scripts",
"tags",
"chat",
"import",
"git",
"settings",
"toggle_sidebar",
"toggle_panel",
"toggle_assistant_sidebar",
]
.contains(&destination)
{
return Err(EngineError::Validation(format!(
"unsupported navigation destination: {destination}"
)));
}
Ok(json!({
"success": true,
"navigation": {
"destination": destination,
"entity_id": entity_id,
}
}))
}
fn post_detail(data_dir: &Path, item: crate::model::Post) -> EngineResult<Value> {
let body = post_body(data_dir, &item)?;
Ok(json!({"success": true, "post": item, "body": body}))
}
fn post_body(data_dir: &Path, item: &crate::model::Post) -> EngineResult<String> {
if let Some(content) = item.content.as_deref() {
return Ok(content.to_string());
}
if item.file_path.is_empty() {
return Ok(String::new());
}
let raw = fs::read_to_string(data_dir.join(&item.file_path))?;
let (_, body) = read_post_file(&raw)
.map_err(|error| EngineError::Parse(format!("invalid post file: {error}")))?;
Ok(body)
}
fn post_summary(item: &crate::model::Post) -> Value {
json!({
"id": item.id, "title": item.title, "slug": item.slug,
"excerpt": item.excerpt, "status": item.status, "tags": item.tags,
"categories": item.categories, "created_at": item.created_at,
"updated_at": item.updated_at,
})
}
fn spec(name: &str, description: &str, properties: Value) -> Value {
let required = properties
.as_object()
.into_iter()
.flat_map(|values| values.iter())
.filter(|(_, schema)| !schema.get("type").is_some_and(Value::is_array))
.filter(|(name, _)| {
matches!(
name.as_str(),
"term"
| "query"
| "post_id"
| "slug"
| "media_id"
| "template_id"
| "script_id"
| "destination"
)
})
.map(|(name, _)| Value::String(name.clone()))
.collect::<Vec<_>>();
let required = if name == "navigate" {
Vec::new()
} else {
required
};
json!({
"type": "function",
"function": {
"name": name,
"description": description,
"parameters": {
"type": "object",
"properties": properties,
"required": required,
"additionalProperties": false,
}
}
})
}
fn required_navigation_value<'a>(
value: Option<&'a str>,
target: &str,
) -> EngineResult<Option<&'a str>> {
value
.filter(|value| !value.trim().is_empty())
.map(Some)
.ok_or_else(|| {
EngineError::Validation(format!("navigation {target} identifier is required"))
})
}
fn required_str<'a>(arguments: &'a Value, key: &str) -> EngineResult<&'a str> {
arguments
.get(key)
.and_then(Value::as_str)
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| EngineError::Validation(format!("tool argument {key} is required")))
}
fn required_id<'a>(arguments: &'a Value, snake: &str, camel: &str) -> EngineResult<&'a str> {
arguments
.get(snake)
.or_else(|| arguments.get(camel))
.and_then(Value::as_str)
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| EngineError::Validation(format!("tool argument {snake} is required")))
}
fn ensure_project(actual: &str, expected: &str) -> EngineResult<()> {
if actual == expected {
Ok(())
} else {
Err(EngineError::NotFound("project entity".to_string()))
}
}
fn limit(arguments: &Value) -> usize {
arguments
.get("limit")
.and_then(Value::as_u64)
.unwrap_or(25)
.clamp(1, 100) as usize
}
fn contains_case_insensitive(values: &[String], needle: &str) -> bool {
values
.iter()
.any(|value| value.eq_ignore_ascii_case(needle))
}
fn optional_string_array(arguments: &Value, key: &str) -> EngineResult<Option<Vec<String>>> {
let Some(value) = arguments.get(key) else {
return Ok(None);
};
let values = value
.as_array()
.ok_or_else(|| EngineError::Validation(format!("tool argument {key} must be an array")))?
.iter()
.map(|value| {
value.as_str().map(str::to_string).ok_or_else(|| {
EngineError::Validation(format!("tool argument {key} must contain strings"))
})
})
.collect::<EngineResult<Vec<_>>>()?;
Ok(Some(values))
}
fn optional_nullable_str<'a>(
arguments: &'a Value,
key: &str,
) -> EngineResult<Option<Option<&'a str>>> {
match arguments.get(key) {
None => Ok(None),
Some(Value::Null) => Ok(Some(None)),
Some(Value::String(value)) => Ok(Some(Some(value))),
Some(_) => Err(EngineError::Validation(format!(
"tool argument {key} must be a string or null"
))),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::db::queries::media::{insert_media, make_test_media};
use crate::db::queries::post::{insert_post, make_test_post};
use crate::db::queries::post_link::insert_post_link;
use crate::db::queries::post_media::link_media;
use crate::db::queries::project::{insert_project, make_test_project};
use crate::model::{PostLink, PostMedia};
fn setup() -> Database {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
insert_project(db.conn(), &make_test_project("p1", "blog")).unwrap();
insert_post(db.conn(), &make_test_post("source", "p1", "source")).unwrap();
insert_post(db.conn(), &make_test_post("target", "p1", "target")).unwrap();
insert_media(db.conn(), &make_test_media("media1", "p1")).unwrap();
insert_post_link(
db.conn(),
&PostLink {
id: "link1".into(),
source_post_id: "source".into(),
target_post_id: "target".into(),
link_text: Some("read next".into()),
created_at: 1,
},
)
.unwrap();
link_media(
db.conn(),
&PostMedia {
id: "post-media1".into(),
project_id: "p1".into(),
post_id: "source".into(),
media_id: "media1".into(),
sort_order: 3,
created_at: 1,
},
)
.unwrap();
db
}
#[test]
fn relationship_tools_return_project_entities() {
let db = setup();
let dir = tempfile::tempdir().unwrap();
let outlinks = execute(
db.conn(),
dir.path(),
"p1",
"get_post_outlinks",
&json!({"post_id": "source"}),
)
.unwrap();
assert_eq!(outlinks["links_to"][0]["post"]["id"], "target");
assert_eq!(outlinks["links_to"][0]["link_text"], "read next");
let backlinks = execute(
db.conn(),
dir.path(),
"p1",
"get_post_backlinks",
&json!({"post_id": "target"}),
)
.unwrap();
assert_eq!(backlinks["linked_by"][0]["post"]["id"], "source");
let post_media = execute(
db.conn(),
dir.path(),
"p1",
"get_post_media",
&json!({"post_id": "source"}),
)
.unwrap();
assert_eq!(post_media["media"][0]["media"]["id"], "media1");
assert_eq!(post_media["media"][0]["sort_order"], 3);
let media_posts = execute(
db.conn(),
dir.path(),
"p1",
"get_media_posts",
&json!({"media_id": "media1"}),
)
.unwrap();
assert_eq!(media_posts["posts"][0]["post"]["id"], "source");
}
#[test]
fn view_image_is_bounded_to_generated_image_thumbnails() {
let db = setup();
let dir = tempfile::tempdir().unwrap();
let relative = crate::util::thumbnail_path("media1", "medium", "webp");
let path = dir.path().join(relative);
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(&path, b"thumbnail").unwrap();
let result = execute(
db.conn(),
dir.path(),
"p1",
"view_image",
&json!({"media_id": "media1", "size": "medium"}),
)
.unwrap();
assert_eq!(result["success"], true);
assert_eq!(result["data_url"], "data:image/webp;base64,dGh1bWJuYWls");
let invalid = execute(
db.conn(),
dir.path(),
"p1",
"view_image",
&json!({"media_id": "media1", "size": "original"}),
);
assert!(matches!(invalid, Err(EngineError::Validation(_))));
}
#[test]
fn structured_render_tools_are_advertised_and_return_inert_native_data() {
let specs = tool_specs();
let names = specs
.iter()
.filter_map(|spec| spec.pointer("/function/name").and_then(Value::as_str))
.collect::<BTreeSet<_>>();
for name in chat_surfaces::RENDER_TOOL_NAMES {
assert!(names.contains(name), "missing tool schema for {name}");
}
let chart = specs
.iter()
.find(|spec| {
spec.pointer("/function/name").and_then(Value::as_str) == Some("render_chart")
})
.unwrap();
assert!(
chart
.pointer("/function/parameters/properties/chartType")
.is_some()
);
assert!(
chart
.pointer("/function/parameters/properties/chart_type")
.is_some()
);
let db = setup();
let dir = tempfile::tempdir().unwrap();
let raw = json!({"title": "<script>alert(1)</script>", "body": "<b>data</b>"});
let result = execute(db.conn(), dir.path(), "p1", "render_card", &raw).unwrap();
assert_eq!(result["type"], "card");
assert_eq!(result["title"], raw["title"]);
assert_eq!(result["body"], raw["body"]);
assert!(result.get("html").is_none());
assert!(result.get("javascript").is_none());
}
}

View File

@@ -0,0 +1,112 @@
use std::path::{Path, PathBuf};
use crate::engine::{EngineError, EngineResult};
/// Install a recoverable launcher pointing at a packaged `bds-cli` binary.
/// Existing unrelated files are never overwritten.
pub fn install_launcher(executable: &Path, home_dir: &Path) -> EngineResult<PathBuf> {
if !executable.is_file() {
return Err(EngineError::Validation(format!(
"installing the CLI requires the packaged bds-cli executable (not found at {})",
executable.display()
)));
}
let bin_dir = home_dir.join(".local/bin");
std::fs::create_dir_all(&bin_dir)?;
let target = bin_dir.join(if cfg!(windows) {
"bds-cli.cmd"
} else {
"bds-cli"
});
let source = executable.canonicalize()?;
let launcher = launcher_contents(&source);
#[cfg(unix)]
if target.is_symlink() && target.canonicalize().ok().as_ref() == Some(&source) {
std::fs::remove_file(&target)?;
}
if target.exists() {
if std::fs::read(&target).ok().as_deref() != Some(launcher.as_bytes()) {
return Err(EngineError::Conflict(format!(
"refusing to overwrite existing launcher at {}",
target.display()
)));
}
return Ok(target);
}
std::fs::write(&target, launcher)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt as _;
let mut permissions = std::fs::metadata(&target)?.permissions();
permissions.set_mode(0o755);
std::fs::set_permissions(&target, permissions)?;
}
Ok(target)
}
#[cfg(unix)]
fn launcher_contents(executable: &Path) -> String {
let quoted = executable.to_string_lossy().replace('\'', "'\"'\"'");
format!("#!/bin/sh\nexec '{quoted}' \"$@\"\n")
}
#[cfg(windows)]
fn launcher_contents(executable: &Path) -> String {
let escaped = executable.to_string_lossy().replace('%', "%%");
format!("@echo off\r\n\"{escaped}\" %*\r\n")
}
/// Resolve the CLI shipped beside the desktop executable and install it.
pub fn install_packaged_launcher(home_dir: &Path) -> EngineResult<PathBuf> {
let app = std::env::current_exe()?;
let cli = app.with_file_name(if cfg!(windows) {
"bds-cli.exe"
} else {
"bds-cli"
});
install_launcher(&cli, home_dir)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn install_is_idempotent_and_never_overwrites_an_unrelated_file() {
let root = tempfile::tempdir().unwrap();
let executable = root.path().join("packaged-bds-cli");
std::fs::write(&executable, b"binary").unwrap();
let home = root.path().join("home");
let target = install_launcher(&executable, &home).unwrap();
assert!(!target.is_symlink());
let launcher = std::fs::read_to_string(&target).unwrap();
assert!(launcher.contains(executable.canonicalize().unwrap().to_str().unwrap()));
assert_eq!(install_launcher(&executable, &home).unwrap(), target);
std::fs::remove_file(&target).unwrap();
std::fs::write(&target, b"mine").unwrap();
assert!(install_launcher(&executable, &home).is_err());
assert_eq!(std::fs::read(&target).unwrap(), b"mine");
}
#[cfg(unix)]
#[test]
fn replaces_the_previous_installer_symlink_with_a_forwarding_launcher() {
let root = tempfile::tempdir().unwrap();
let executable = root.path().join("packaged-bds-cli");
std::fs::write(&executable, b"binary").unwrap();
let home = root.path().join("home");
let target = home.join(".local/bin/bds-cli");
std::fs::create_dir_all(target.parent().unwrap()).unwrap();
std::os::unix::fs::symlink(&executable, &target).unwrap();
assert_eq!(install_launcher(&executable, &home).unwrap(), target);
assert!(!target.is_symlink());
assert!(
std::fs::read_to_string(target)
.unwrap()
.starts_with("#!/bin/sh")
);
}
}

View File

@@ -0,0 +1,222 @@
use crate::db::DbConnection as Connection;
use crate::db::queries::db_notification as qn;
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::model::{DomainEntity, DomainEvent, NotificationAction};
use crate::util::now_unix_ms;
pub const PROCESSED_TTL_MS: i64 = 60 * 60 * 1_000;
pub const UNPROCESSED_TTL_MS: i64 = 24 * 60 * 60 * 1_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct PruneResult {
pub processed: usize,
pub unprocessed: usize,
}
/// Run a future CLI mutation through the same in-process event path, then
/// persist only the successfully published events for the desktop watcher.
pub fn run_cli_mutation<T>(
conn: &Connection,
operation: impl FnOnce() -> EngineResult<T>,
) -> EngineResult<T> {
let (result, events) = domain_events::capture_current_thread(operation)
.map_err(|message| EngineError::Validation(message.to_string()))?;
let mut unique = Vec::<DomainEvent>::new();
for event in events {
if let Some(seen) = unique
.iter_mut()
.find(|seen| same_notification(seen, &event))
{
merge_notification_action(seen, &event);
} else {
unique.push(event);
}
}
for event in &unique {
record_cli_event(conn, event)?;
}
result
}
fn merge_notification_action(existing: &mut DomainEvent, incoming: &DomainEvent) {
let (
DomainEvent::EntityChanged {
action: existing_action,
..
},
DomainEvent::EntityChanged {
action: incoming_action,
..
},
) = (existing, incoming)
else {
return;
};
if *incoming_action == NotificationAction::Deleted
|| *existing_action != NotificationAction::Created
{
*existing_action = incoming_action.clone();
}
}
fn same_notification(left: &DomainEvent, right: &DomainEvent) -> bool {
match (left, right) {
(
DomainEvent::EntityChanged {
project_id: left_project,
entity: left_entity,
entity_id: left_id,
..
},
DomainEvent::EntityChanged {
project_id: right_project,
entity: right_entity,
entity_id: right_id,
..
},
) => left_project == right_project && left_entity == right_entity && left_id == right_id,
(
DomainEvent::SettingsChanged {
project_id: left_project,
key: left_key,
},
DomainEvent::SettingsChanged {
project_id: right_project,
key: right_key,
},
) => left_project == right_project && left_key == right_key,
_ => false,
}
}
pub fn record_cli_event(conn: &Connection, event: &DomainEvent) -> EngineResult<()> {
record_cli_event_at(conn, event, now_unix_ms())
}
#[doc(hidden)]
pub fn record_cli_event_at(
conn: &Connection,
event: &DomainEvent,
created_at: i64,
) -> EngineResult<()> {
let (entity, entity_id, action, project_id) = match event {
DomainEvent::EntityChanged {
project_id,
entity,
entity_id,
action,
} => (
entity.clone(),
entity_id.as_str(),
action.clone(),
Some(project_id.as_str()),
),
DomainEvent::SettingsChanged { project_id, key } => (
DomainEntity::Setting,
key.as_str(),
NotificationAction::Updated,
project_id.as_deref(),
),
};
qn::insert_notification(
conn, &entity, entity_id, &action, true, None, created_at, project_id,
)?;
Ok(())
}
pub fn consume_cli_notifications(conn: &Connection) -> EngineResult<Vec<DomainEvent>> {
consume_cli_notifications_at(conn, now_unix_ms())
}
#[doc(hidden)]
pub fn consume_cli_notifications_at(
conn: &Connection,
seen_at: i64,
) -> EngineResult<Vec<DomainEvent>> {
conn.begin_savepoint()?;
let result = (|| {
let notifications = qn::list_unseen_cli_notifications(conn)?;
let ids = notifications.iter().map(|item| item.id).collect::<Vec<_>>();
qn::mark_notifications_seen(conn, &ids, seen_at)?;
let events = notifications
.into_iter()
.filter_map(|notification| match notification.entity_type {
DomainEntity::Setting => Some(DomainEvent::SettingsChanged {
project_id: notification.project_id,
key: notification.entity_id,
}),
entity => notification
.project_id
.or_else(|| legacy_project_id(conn, &entity, &notification.entity_id))
.map(|project_id| DomainEvent::EntityChanged {
project_id,
entity,
entity_id: notification.entity_id,
action: notification.action,
}),
})
.collect();
Ok(events)
})();
match result {
Ok(events) => {
conn.release_savepoint()?;
Ok(events)
}
Err(error) => {
let _ = conn.rollback_savepoint();
Err(error)
}
}
}
fn legacy_project_id(conn: &Connection, entity: &DomainEntity, entity_id: &str) -> Option<String> {
let resolved = match entity {
DomainEntity::Post => crate::db::queries::post::get_post_by_id(conn, entity_id)
.ok()
.map(|item| item.project_id),
DomainEntity::Media => crate::db::queries::media::get_media_by_id(conn, entity_id)
.ok()
.map(|item| item.project_id),
DomainEntity::Tag => crate::db::queries::tag::get_tag_by_id(conn, entity_id)
.ok()
.map(|item| item.project_id),
DomainEntity::Template => crate::db::queries::template::get_template_by_id(conn, entity_id)
.ok()
.map(|item| item.project_id),
DomainEntity::Script => crate::db::queries::script::get_script_by_id(conn, entity_id)
.ok()
.map(|item| item.project_id),
DomainEntity::Project => Some(entity_id.to_string()),
DomainEntity::Setting => None,
};
resolved.or_else(|| {
crate::db::queries::project::get_active_project(conn)
.ok()
.map(|project| project.id)
})
}
pub fn prune_notifications(conn: &Connection) -> EngineResult<PruneResult> {
prune_notifications_at(conn, now_unix_ms())
}
#[doc(hidden)]
pub fn prune_notifications_at(conn: &Connection, now: i64) -> EngineResult<PruneResult> {
Ok(PruneResult {
processed: qn::prune_processed(conn, now - PROCESSED_TTL_MS)?,
unprocessed: qn::prune_unprocessed(conn, now - UNPROCESSED_TTL_MS)?,
})
}
/// Desktop watcher poll: consume once, publish through the shared bus, then
/// apply both retention windows.
pub fn poll_notifications(conn: &Connection) -> EngineResult<usize> {
let events = consume_cli_notifications(conn)?;
let count = events.len();
for event in events {
domain_events::publish(event);
}
prune_notifications(conn)?;
Ok(count)
}

View File

@@ -0,0 +1,175 @@
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::sync::mpsc::{self, Receiver, Sender};
use std::sync::{Arc, Mutex, OnceLock};
use crate::model::{DomainEntity, DomainEvent, NotificationAction};
#[derive(Default)]
struct BusState {
next_id: u64,
subscribers: BTreeMap<u64, Sender<DomainEvent>>,
}
#[derive(Default)]
struct BusInner {
state: Mutex<BusState>,
publish_lock: Mutex<()>,
}
/// Minimal deterministic in-process event bus. Delivery is synchronous and
/// ordered, while every subscriber owns an independent unbounded queue.
#[derive(Clone, Default)]
pub struct EventBus {
inner: Arc<BusInner>,
}
impl EventBus {
pub fn subscribe(&self) -> EventSubscription {
let (sender, receiver) = mpsc::channel();
let _publish = lock(&self.inner.publish_lock);
let mut state = lock(&self.inner.state);
let id = state.next_id;
state.next_id = state.next_id.wrapping_add(1);
state.subscribers.insert(id, sender);
EventSubscription {
id: Some(id),
inner: Arc::clone(&self.inner),
receiver,
}
}
pub fn publish(&self, event: DomainEvent) {
let _publish = lock(&self.inner.publish_lock);
let subscribers = lock(&self.inner.state)
.subscribers
.iter()
.map(|(id, sender)| (*id, sender.clone()))
.collect::<Vec<_>>();
let disconnected = subscribers
.into_iter()
.filter_map(|(id, sender)| sender.send(event.clone()).is_err().then_some(id))
.collect::<Vec<_>>();
if !disconnected.is_empty() {
let mut state = lock(&self.inner.state);
for id in disconnected {
state.subscribers.remove(&id);
}
}
}
}
pub struct EventSubscription {
id: Option<u64>,
inner: Arc<BusInner>,
receiver: Receiver<DomainEvent>,
}
impl EventSubscription {
pub fn drain(&self) -> Vec<DomainEvent> {
self.receiver.try_iter().collect()
}
pub fn unsubscribe(mut self) {
self.detach();
}
fn detach(&mut self) {
if let Some(id) = self.id.take() {
let _publish = lock(&self.inner.publish_lock);
lock(&self.inner.state).subscribers.remove(&id);
}
}
}
impl Drop for EventSubscription {
fn drop(&mut self) {
self.detach();
}
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn global_bus() -> &'static EventBus {
static BUS: OnceLock<EventBus> = OnceLock::new();
BUS.get_or_init(EventBus::default)
}
thread_local! {
static CAPTURED_EVENTS: RefCell<Option<Vec<DomainEvent>>> = const { RefCell::new(None) };
}
pub fn subscribe() -> EventSubscription {
global_bus().subscribe()
}
pub fn publish(event: DomainEvent) {
CAPTURED_EVENTS.with(|captured| {
if let Some(events) = captured.borrow_mut().as_mut() {
events.push(event.clone());
}
});
global_bus().publish(event);
}
pub fn entity_changed(
project_id: &str,
entity: DomainEntity,
entity_id: &str,
action: NotificationAction,
) {
publish(DomainEvent::EntityChanged {
project_id: project_id.to_string(),
entity,
entity_id: entity_id.to_string(),
action,
});
}
pub fn settings_changed(project_id: Option<&str>, key: &str) {
publish(DomainEvent::SettingsChanged {
project_id: project_id.map(str::to_string),
key: key.to_string(),
});
}
pub(crate) fn capture_current_thread<T>(
operation: impl FnOnce() -> T,
) -> Result<(T, Vec<DomainEvent>), &'static str> {
let started = CAPTURED_EVENTS.with(|captured| {
let mut captured = captured.borrow_mut();
if captured.is_some() {
false
} else {
*captured = Some(Vec::new());
true
}
});
if !started {
return Err("nested CLI event capture is not supported");
}
let mut reset = CaptureReset::default();
let result = operation();
let events = CAPTURED_EVENTS.with(|captured| captured.borrow_mut().take().unwrap_or_default());
reset.finished = true;
Ok((result, events))
}
#[derive(Default)]
struct CaptureReset {
finished: bool,
}
impl Drop for CaptureReset {
fn drop(&mut self) {
if !self.finished {
CAPTURED_EVENTS.with(|captured| {
captured.borrow_mut().take();
});
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -15,6 +15,8 @@ pub enum EngineError {
Conflict(String),
#[error("validation error: {0}")]
Validation(String),
#[error("operation cancelled")]
Cancelled,
}
impl From<crate::db::DatabaseError> for EngineError {

View File

@@ -0,0 +1,617 @@
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use base64::Engine as _;
use rayon::prelude::*;
use serde_json::json;
use crate::db::Database;
use crate::engine::{ai, media, post_media};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ImportedGalleryImage {
pub media_id: String,
pub title: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GalleryImportOutcome {
pub path: PathBuf,
pub result: Result<ImportedGalleryImage, String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GalleryImportReport {
pub selected_count: usize,
pub outcomes: Vec<GalleryImportOutcome>,
}
pub fn active_ai_endpoint_configured(conn: &crate::db::DbConnection, offline_mode: bool) -> bool {
ai::active_endpoint(conn, offline_mode)
.is_ok_and(|endpoint| !endpoint.url.trim().is_empty() && !endpoint.model.trim().is_empty())
}
pub fn translation_targets(
main_language: Option<&str>,
blog_languages: &[String],
source_language: &str,
) -> Vec<String> {
let mut seen = HashSet::new();
main_language
.into_iter()
.chain(blog_languages.iter().map(String::as_str))
.filter(|language| !language.is_empty() && *language != source_language)
.filter(|language| seen.insert((*language).to_string()))
.map(str::to_string)
.collect()
}
pub fn process_paths_concurrently<T, F>(
paths: Vec<PathBuf>,
concurrency: usize,
process: F,
) -> Result<Vec<T>, String>
where
T: Send,
F: Fn(usize, PathBuf) -> T + Send + Sync,
{
rayon::ThreadPoolBuilder::new()
.num_threads(concurrency.clamp(1, 8))
.build()
.map_err(|error| error.to_string())
.map(|pool| {
pool.install(|| {
paths
.into_par_iter()
.enumerate()
.map(|(index, path)| process(index, path))
.collect()
})
})
}
pub fn import_gallery_images(
db_path: &Path,
data_dir: &Path,
project_id: &str,
post_id: &str,
paths: Vec<PathBuf>,
source_language: &str,
offline_mode: bool,
) -> GalleryImportReport {
let selected_count = paths.len();
let metadata = crate::engine::meta::read_project_json(data_dir).ok();
let concurrency = metadata
.as_ref()
.map(|metadata| metadata.image_import_concurrency.clamp(1, 8) as usize)
.unwrap_or(4);
let targets = translation_targets(
metadata
.as_ref()
.and_then(|metadata| metadata.main_language.as_deref()),
metadata
.as_ref()
.map(|metadata| metadata.blog_languages.as_slice())
.unwrap_or_default(),
source_language,
);
let ai_available = Database::open(db_path)
.ok()
.is_some_and(|db| active_ai_endpoint_configured(db.conn(), offline_mode));
let first_sort_order = Database::open(db_path)
.ok()
.and_then(|db| post_media::list_media_for_post(db.conn(), post_id).ok())
.map(|media| media.len() as i32)
.unwrap_or(0);
let process_path = |index: usize, path: PathBuf| GalleryImportOutcome {
result: import_gallery_image(
db_path,
data_dir,
project_id,
post_id,
&path,
source_language,
first_sort_order + index as i32,
ai_available,
offline_mode,
&targets,
),
path,
};
let outcomes = match process_paths_concurrently(paths.clone(), concurrency, process_path) {
Ok(outcomes) => outcomes,
Err(error) => paths
.into_iter()
.map(|path| GalleryImportOutcome {
path,
result: Err(error.clone()),
})
.collect(),
};
GalleryImportReport {
selected_count,
outcomes,
}
}
#[expect(
clippy::too_many_arguments,
reason = "one worker receives the immutable batch context"
)]
fn import_gallery_image(
db_path: &Path,
data_dir: &Path,
project_id: &str,
post_id: &str,
path: &Path,
source_language: &str,
sort_order: i32,
ai_available: bool,
offline_mode: bool,
translation_targets: &[String],
) -> Result<ImportedGalleryImage, String> {
let db = Database::open(db_path).map_err(|error| error.to_string())?;
let imported = import_and_link_image(
db.conn(),
data_dir,
project_id,
post_id,
path,
source_language,
sort_order,
)
.map_err(|error| error.to_string())?;
let title = if ai_available {
enrich_imported_image(
db.conn(),
data_dir,
&imported,
offline_mode,
translation_targets,
)
.unwrap_or_else(|_| imported.original_name.clone())
} else {
imported.original_name.clone()
};
Ok(ImportedGalleryImage {
media_id: imported.id,
title,
})
}
pub fn import_and_link_image(
conn: &crate::db::DbConnection,
data_dir: &Path,
project_id: &str,
post_id: &str,
path: &Path,
source_language: &str,
sort_order: i32,
) -> crate::engine::EngineResult<crate::model::Media> {
let original_name = path
.file_name()
.map(|name| name.to_string_lossy().to_string())
.unwrap_or_else(|| "image".to_string());
let imported = media::import_media(
conn,
data_dir,
project_id,
path,
&original_name,
None,
None,
None,
None,
Some(source_language),
Vec::new(),
)?;
post_media::link_media_to_post(
conn,
data_dir,
project_id,
post_id,
&imported.id,
sort_order,
)?;
Ok(imported)
}
/// Apply the shared gallery AI enrichment and translation pipeline to one
/// already-imported image. Returns the generated title when AI was available.
pub fn enrich_imported_image(
conn: &crate::db::DbConnection,
data_dir: &Path,
imported: &crate::model::Media,
offline_mode: bool,
translation_targets: &[String],
) -> Result<String, String> {
let image_data_url = build_ai_image_data_url(
data_dir,
&imported.id,
&imported.file_path,
&imported.mime_type,
)?;
let response = ai::run_one_shot(
conn,
offline_mode,
&ai::OneShotRequest {
operation: ai::OneShotOperation::AnalyzeImage,
content: json!({
"title": imported.title,
"alt": imported.alt,
"caption": imported.caption,
"filename": imported.original_name,
"mime_type": imported.mime_type,
"image_data_url": image_data_url,
}),
},
)
.map_err(|error| error.to_string())?;
let ai::OneShotResponse::ImageAnalysis(analysis) = response.0 else {
return Err("AI returned an unexpected response".to_string());
};
media::update_media(
conn,
data_dir,
&imported.id,
Some(Some(&analysis.title)),
Some(Some(&analysis.alt)),
Some(Some(&analysis.caption)),
None,
None,
None,
)
.map_err(|error| error.to_string())?;
for target in translation_targets {
let Ok((ai::OneShotResponse::MediaTranslation(translation), _)) = ai::run_one_shot(
conn,
offline_mode,
&ai::OneShotRequest {
operation: ai::OneShotOperation::TranslateMedia {
target_language: target.clone(),
},
content: json!({
"title": analysis.title,
"alt": analysis.alt,
"caption": analysis.caption,
}),
},
) else {
continue;
};
let _ = media::upsert_media_translation(
conn,
data_dir,
&imported.id,
target,
Some(&translation.title),
Some(&translation.alt),
Some(&translation.caption),
);
}
Ok(if analysis.title.is_empty() {
imported.original_name.clone()
} else {
analysis.title
})
}
pub fn build_ai_image_data_url(
data_dir: &Path,
media_id: &str,
file_path: &str,
mime_type: &str,
) -> Result<String, String> {
if !mime_type.starts_with("image/") {
return Err("AI image analysis requires an image".to_string());
}
let source_path = data_dir.join(file_path.trim_start_matches('/'));
let thumbnail_path = data_dir.join(crate::util::thumbnail_path(media_id, "ai", "jpg"));
if !thumbnail_path.exists() {
crate::util::thumbnail::generate_all_thumbnails(
&source_path,
&data_dir.join("thumbnails"),
media_id,
)
.map_err(|error| format!("failed to generate AI thumbnail: {error}"))?;
}
let bytes = std::fs::read(&thumbnail_path)
.map_err(|error| format!("failed to read AI thumbnail: {error}"))?;
Ok(format!(
"data:image/jpeg;base64,{}",
base64::engine::general_purpose::STANDARD.encode(bytes)
))
}
#[cfg(test)]
mod tests {
use std::fs;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread;
use std::time::Duration;
use tempfile::TempDir;
use crate::db::queries::post::insert_post;
use crate::db::queries::post::make_test_post;
use crate::db::queries::post_media::{list_post_media_by_post, unlink_media};
use crate::db::queries::project::{insert_project, make_test_project};
use crate::db::{Database, fts::ensure_fts_tables};
use crate::engine::ai::{AiEndpointConfig, AiEndpointKind, save_endpoint};
use crate::engine::media::rebuild_media_from_filesystem;
use crate::model::metadata::ProjectMetadata;
use super::{import_gallery_images, process_paths_concurrently, translation_targets};
fn setup() -> (Database, TempDir) {
let dir = TempDir::new().unwrap();
let db = Database::open(&dir.path().join("bds.db")).unwrap();
db.migrate().unwrap();
ensure_fts_tables(db.conn()).unwrap();
insert_project(db.conn(), &make_test_project("p1", "Gallery")).unwrap();
insert_post(db.conn(), &make_test_post("post1", "p1", "gallery")).unwrap();
crate::engine::meta::write_project_json(
dir.path(),
&ProjectMetadata {
name: "Gallery".to_string(),
description: None,
public_url: None,
main_language: Some("en".to_string()),
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 2,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
blog_languages: vec![
"de".to_string(),
"en".to_string(),
"fr".to_string(),
"de".to_string(),
],
},
)
.unwrap();
(db, dir)
}
fn spawn_ai_server() -> String {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
thread::spawn(move || {
for stream in listener.incoming().take(3) {
let mut stream = stream.unwrap();
let mut request = Vec::new();
let mut buffer = [0_u8; 16_384];
loop {
let read = stream.read(&mut buffer).unwrap();
if read == 0 {
break;
}
request.extend_from_slice(&buffer[..read]);
let Some(header_end) = request.windows(4).position(|part| part == b"\r\n\r\n")
else {
continue;
};
let headers = String::from_utf8_lossy(&request[..header_end]);
let content_length = headers
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length: ")
.and_then(|value| value.parse::<usize>().ok())
})
.unwrap_or(0);
if request.len() >= header_end + 4 + content_length {
break;
}
}
let request = String::from_utf8_lossy(&request);
let content = if request.contains("Translate the media metadata into de") {
r#"{"title":"Berg","alt":"Ein Berg","caption":"Morgenlicht"}"#
} else if request.contains("Translate the media metadata into fr") {
r#"{"title":"Montagne","alt":"Une montagne","caption":"Lumière du matin"}"#
} else {
r#"{"title":"Mountain","alt":"A mountain","caption":"Morning light"}"#
};
let body = serde_json::json!({
"choices": [{"message": {"content": content}}]
})
.to_string();
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body,
);
stream.write_all(response.as_bytes()).unwrap();
}
});
format!("http://{address}")
}
#[test]
fn targets_are_unique_and_exclude_source_language() {
assert_eq!(
translation_targets(
Some("en"),
&["de".to_string(), "en".to_string(), "de".to_string()],
"fr",
),
vec!["en".to_string(), "de".to_string()]
);
}
#[test]
fn partial_failure_keeps_successful_image_linked_and_rebuildable_without_ai() {
let (db, dir) = setup();
let image = dir.path().join("photo.jpg");
fs::write(&image, b"jpeg data").unwrap();
let invalid = dir.path().join("notes.txt");
fs::write(&invalid, b"not an image").unwrap();
let report = import_gallery_images(
&dir.path().join("bds.db"),
dir.path(),
"p1",
"post1",
vec![image, invalid],
"en",
false,
);
assert_eq!(report.selected_count, 2);
assert!(report.outcomes[0].result.is_ok());
assert!(report.outcomes[1].result.is_err());
let links = list_post_media_by_post(db.conn(), "post1").unwrap();
assert_eq!(links.len(), 1);
let media_id = links[0].media_id.clone();
let sidecar = fs::read_to_string(
dir.path().join(
crate::db::queries::media::get_media_by_id(db.conn(), &media_id)
.unwrap()
.sidecar_path,
),
)
.unwrap();
assert!(sidecar.contains("linkedPostIds: [\"post1\"]"));
unlink_media(db.conn(), "post1", &media_id).unwrap();
rebuild_media_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(
list_post_media_by_post(db.conn(), "post1").unwrap().len(),
1
);
}
#[test]
fn configured_local_ai_enriches_metadata_and_unique_translation_targets() {
let (db, dir) = setup();
save_endpoint(
db.conn(),
&AiEndpointConfig {
kind: AiEndpointKind::Airplane,
url: spawn_ai_server(),
model: "local-vision".to_string(),
api_key: None,
},
)
.unwrap();
let image = dir.path().join("mountain.png");
image::DynamicImage::new_rgb8(8, 8).save(&image).unwrap();
let report = import_gallery_images(
&dir.path().join("bds.db"),
dir.path(),
"p1",
"post1",
vec![image],
"en",
true,
);
let imported = report.outcomes[0].result.as_ref().unwrap();
assert_eq!(imported.title, "Mountain");
let media =
crate::db::queries::media::get_media_by_id(db.conn(), &imported.media_id).unwrap();
assert_eq!(media.title.as_deref(), Some("Mountain"));
assert_eq!(media.alt.as_deref(), Some("A mountain"));
assert_eq!(media.caption.as_deref(), Some("Morning light"));
let translations = crate::db::queries::media_translation::list_media_translations_by_media(
db.conn(),
&media.id,
)
.unwrap();
assert_eq!(
translations
.iter()
.map(|translation| translation.language.as_str())
.collect::<Vec<_>>(),
vec!["de", "fr"]
);
for language in ["de", "fr"] {
assert!(
dir.path()
.join(crate::util::media_translation_sidecar_path(
&media.file_path,
language,
))
.is_file()
);
}
}
#[test]
fn analysis_failure_does_not_remove_the_post_link() {
let (db, dir) = setup();
save_endpoint(
db.conn(),
&AiEndpointConfig {
kind: AiEndpointKind::Airplane,
url: "http://127.0.0.1:9".to_string(),
model: "unavailable-local-model".to_string(),
api_key: None,
},
)
.unwrap();
let image = dir.path().join("linked.png");
image::DynamicImage::new_rgb8(8, 8).save(&image).unwrap();
let report = import_gallery_images(
&dir.path().join("bds.db"),
dir.path(),
"p1",
"post1",
vec![image],
"en",
true,
);
assert!(report.outcomes[0].result.is_ok());
assert_eq!(
list_post_media_by_post(db.conn(), "post1").unwrap().len(),
1
);
}
#[test]
fn concurrent_processor_clamps_worker_count_to_one_through_eight() {
let active = Arc::new(AtomicUsize::new(0));
let maximum = Arc::new(AtomicUsize::new(0));
let paths = (0..24)
.map(|index| PathBuf::from(format!("{index}.jpg")))
.collect();
let results = process_paths_concurrently(paths, 99, {
let active = Arc::clone(&active);
let maximum = Arc::clone(&maximum);
move |_index, path| {
let current = active.fetch_add(1, Ordering::SeqCst) + 1;
maximum.fetch_max(current, Ordering::SeqCst);
thread::sleep(Duration::from_millis(5));
active.fetch_sub(1, Ordering::SeqCst);
path
}
})
.unwrap();
assert_eq!(results.len(), 24);
assert!((1..=8).contains(&maximum.load(Ordering::SeqCst)));
assert_eq!(
process_paths_concurrently(vec![PathBuf::from("one.jpg")], 0, |_, path| path)
.unwrap()
.len(),
1
);
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,182 @@
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use crate::engine::{EngineError, EngineResult};
const SERVER_NAME: &str = "bDS";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum McpAgent {
ClaudeCode,
GithubCopilot,
}
impl McpAgent {
pub const fn all() -> [Self; 2] {
[Self::ClaudeCode, Self::GithubCopilot]
}
pub const fn label(self) -> &'static str {
match self {
Self::ClaudeCode => "Claude Code",
Self::GithubCopilot => "GitHub Copilot",
}
}
pub const fn as_str(self) -> &'static str {
match self {
Self::ClaudeCode => "claude_code",
Self::GithubCopilot => "github_copilot",
}
}
}
pub fn agent_config_path(agent: McpAgent, home_dir: &Path) -> PathBuf {
match agent {
McpAgent::ClaudeCode => home_dir.join(".claude.json"),
McpAgent::GithubCopilot => {
#[cfg(target_os = "macos")]
let path = home_dir.join("Library/Application Support/Code/User/mcp.json");
#[cfg(target_os = "windows")]
let path = home_dir.join("AppData/Roaming/Code/User/mcp.json");
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
let path = home_dir.join(".config/Code/User/mcp.json");
path
}
}
}
pub fn packaged_mcp_executable() -> EngineResult<PathBuf> {
let executable = std::env::current_exe()?;
let sibling = executable.with_file_name(if cfg!(windows) {
"bds-mcp.exe"
} else {
"bds-mcp"
});
if sibling.is_file() {
Ok(sibling)
} else {
Err(EngineError::NotFound(format!(
"packaged MCP executable {}",
sibling.display()
)))
}
}
pub fn is_agent_configured(agent: McpAgent, home_dir: &Path) -> bool {
read_config(&agent_config_path(agent, home_dir))
.ok()
.is_some_and(|config| {
server_map(&config, agent).is_some_and(|servers| servers.contains_key(SERVER_NAME))
})
}
pub fn install_agent_config(
agent: McpAgent,
home_dir: &Path,
executable: &Path,
) -> EngineResult<PathBuf> {
if !executable.is_file() {
return Err(EngineError::NotFound(format!(
"MCP executable {}",
executable.display()
)));
}
let path = agent_config_path(agent, home_dir);
let mut config = read_config(&path)?;
let key = server_key(agent);
let servers = config
.as_object_mut()
.ok_or_else(|| {
EngineError::Validation(format!("{} must contain a JSON object", path.display()))
})?
.entry(key)
.or_insert_with(|| Value::Object(Map::new()))
.as_object_mut()
.ok_or_else(|| {
EngineError::Validation(format!("{key} in {} must be an object", path.display()))
})?;
let executable = executable.to_string_lossy();
let server = match agent {
McpAgent::ClaudeCode => json!({"command": executable, "args": []}),
McpAgent::GithubCopilot => {
json!({"type": "stdio", "command": executable, "args": []})
}
};
servers.insert(SERVER_NAME.into(), server);
write_config(&path, &config)?;
Ok(path)
}
pub fn remove_agent_config(agent: McpAgent, home_dir: &Path) -> EngineResult<PathBuf> {
let path = agent_config_path(agent, home_dir);
let mut config = read_config(&path)?;
if let Some(servers) = config
.as_object_mut()
.and_then(|object| object.get_mut(server_key(agent)))
.and_then(Value::as_object_mut)
{
servers.remove(SERVER_NAME);
}
write_config(&path, &config)?;
Ok(path)
}
fn server_key(agent: McpAgent) -> &'static str {
match agent {
McpAgent::ClaudeCode => "mcpServers",
McpAgent::GithubCopilot => "servers",
}
}
fn server_map(config: &Value, agent: McpAgent) -> Option<&Map<String, Value>> {
config.get(server_key(agent)).and_then(Value::as_object)
}
fn read_config(path: &Path) -> EngineResult<Value> {
match std::fs::read_to_string(path) {
Ok(source) => serde_json::from_str(&source)
.map_err(|error| EngineError::Parse(format!("{}: {error}", path.display()))),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(json!({})),
Err(error) => Err(error.into()),
}
}
fn write_config(path: &Path, config: &Value) -> EngineResult<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let source = serde_json::to_string_pretty(config)?;
crate::util::atomic_write_str(path, &source)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn install_and_remove_preserve_unrelated_config_without_secrets() {
let root = tempfile::tempdir().unwrap();
let executable = root.path().join("bds-mcp");
std::fs::write(&executable, "binary").unwrap();
for agent in McpAgent::all() {
let path = agent_config_path(agent, root.path());
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(&path, r#"{"unrelated":{"token":"kept"}}"#).unwrap();
install_agent_config(agent, root.path(), &executable).unwrap();
assert!(is_agent_configured(agent, root.path()));
let source = std::fs::read_to_string(&path).unwrap();
assert!(source.contains("kept"));
assert!(!source.contains("api_key"));
remove_agent_config(agent, root.path()).unwrap();
assert!(!is_agent_configured(agent, root.path()));
assert!(std::fs::read_to_string(path).unwrap().contains("kept"));
}
}
}

View File

@@ -0,0 +1,234 @@
use std::net::{Ipv4Addr, SocketAddr};
use std::path::PathBuf;
use std::thread::JoinHandle;
use axum::Router;
use axum::extract::State;
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::post;
use serde_json::Value;
use crate::engine::{EngineError, EngineResult};
use super::McpContext;
use super::protocol::{MCP_PROTOCOL_VERSION, error, handle_rpc};
pub struct McpHttpServer {
address: SocketAddr,
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
thread: Option<JoinHandle<()>>,
}
impl McpHttpServer {
pub fn start(database_path: PathBuf, port: u16) -> EngineResult<Self> {
McpContext::new(database_path.clone()).prepare()?;
let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, port))?;
listener.set_nonblocking(true)?;
let address = listener.local_addr()?;
let (shutdown, shutdown_rx) = tokio::sync::oneshot::channel();
let thread = std::thread::Builder::new()
.name("bds-mcp-http".into())
.spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("MCP Tokio runtime");
runtime.block_on(async move {
let listener =
tokio::net::TcpListener::from_std(listener).expect("MCP loopback listener");
let context = McpContext::new(database_path);
let router = Router::new()
.route("/mcp", post(post_mcp).options(options_mcp))
.with_state(context);
let _ = axum::serve(listener, router)
.with_graceful_shutdown(async {
let _ = shutdown_rx.await;
})
.await;
});
})?;
Ok(Self {
address,
shutdown: Some(shutdown),
thread: Some(thread),
})
}
pub fn address(&self) -> SocketAddr {
self.address
}
pub fn endpoint(&self) -> String {
format!("http://{}/mcp", self.address)
}
pub fn stop(mut self) -> EngineResult<()> {
self.shutdown.take();
if let Some(thread) = self.thread.take() {
thread
.join()
.map_err(|_| EngineError::Parse("MCP server thread panicked".into()))?;
}
Ok(())
}
}
impl Drop for McpHttpServer {
fn drop(&mut self) {
self.shutdown.take();
if let Some(thread) = self.thread.take() {
let _ = thread.join();
}
}
}
async fn post_mcp(
State(context): State<McpContext>,
headers: HeaderMap,
body: axum::body::Bytes,
) -> Response {
if let Err((status, message)) = validate_http_headers(&headers) {
return with_cors((status, message).into_response(), &headers);
}
let request = match serde_json::from_slice::<Value>(&body) {
Ok(request) => request,
Err(_) => {
return with_cors(
(
StatusCode::BAD_REQUEST,
axum::Json(error(Value::Null, -32700, "Parse error")),
)
.into_response(),
&headers,
);
}
};
let response = match handle_rpc(&context, &request) {
Some(response) => (StatusCode::OK, axum::Json(response)).into_response(),
None => StatusCode::ACCEPTED.into_response(),
};
with_cors(response, &headers)
}
async fn options_mcp(headers: HeaderMap) -> Response {
if let Err((status, message)) = validate_origin_and_host(&headers) {
return with_cors((status, message).into_response(), &headers);
}
with_cors(StatusCode::NO_CONTENT.into_response(), &headers)
}
fn validate_http_headers(headers: &HeaderMap) -> Result<(), (StatusCode, &'static str)> {
validate_origin_and_host(headers)?;
if let Some(version) = headers
.get("mcp-protocol-version")
.and_then(|value| value.to_str().ok())
&& !["2025-03-26", MCP_PROTOCOL_VERSION].contains(&version)
{
return Err((StatusCode::BAD_REQUEST, "Unsupported MCP protocol version"));
}
if headers
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.is_none_or(|value| !value.starts_with("application/json"))
{
return Err((
StatusCode::UNSUPPORTED_MEDIA_TYPE,
"Expected application/json",
));
}
if headers
.get(header::ACCEPT)
.and_then(|value| value.to_str().ok())
.is_none_or(|value| {
!value.contains("application/json") || !value.contains("text/event-stream")
})
{
return Err((
StatusCode::NOT_ACCEPTABLE,
"Accept must include application/json and text/event-stream",
));
}
Ok(())
}
fn validate_origin_and_host(headers: &HeaderMap) -> Result<(), (StatusCode, &'static str)> {
let host = headers
.get(header::HOST)
.and_then(|value| value.to_str().ok())
.unwrap_or_default();
let host_name = host
.strip_prefix('[')
.and_then(|value| value.split_once(']').map(|(host, _)| host))
.unwrap_or_else(|| host.split(':').next().unwrap_or_default());
if !["localhost", "127.0.0.1", "::1"].contains(&host_name) {
return Err((StatusCode::FORBIDDEN, "Forbidden host"));
}
if let Some(origin) = headers
.get(header::ORIGIN)
.and_then(|value| value.to_str().ok())
{
let local = url::Url::parse(origin).ok().is_some_and(|origin| {
matches!(origin.host_str(), Some("localhost" | "127.0.0.1" | "::1"))
});
if !local {
return Err((StatusCode::FORBIDDEN, "Forbidden origin"));
}
}
Ok(())
}
fn with_cors(mut response: Response, request_headers: &HeaderMap) -> Response {
let headers = response.headers_mut();
let origin = request_headers
.get(header::ORIGIN)
.filter(|value| {
value.to_str().ok().is_some_and(|origin| {
url::Url::parse(origin).ok().is_some_and(|origin| {
matches!(origin.host_str(), Some("localhost" | "127.0.0.1" | "::1"))
})
})
})
.cloned()
.unwrap_or_else(|| HeaderValue::from_static("http://127.0.0.1"));
headers.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, origin);
headers.insert(
header::ACCESS_CONTROL_ALLOW_METHODS,
HeaderValue::from_static("POST, OPTIONS"),
);
headers.insert(
header::ACCESS_CONTROL_ALLOW_HEADERS,
HeaderValue::from_static("content-type, accept, origin, mcp-protocol-version"),
);
headers.insert(header::VARY, HeaderValue::from_static("Origin"));
response
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn header_validation_rejects_dns_rebinding_and_remote_origins() {
let mut headers = HeaderMap::new();
headers.insert(header::HOST, HeaderValue::from_static("attacker.example"));
assert!(validate_origin_and_host(&headers).is_err());
headers.insert(header::HOST, HeaderValue::from_static("127.0.0.1:4124"));
headers.insert(
header::ORIGIN,
HeaderValue::from_static("https://attacker.example"),
);
assert!(validate_origin_and_host(&headers).is_err());
headers.insert(
header::ORIGIN,
HeaderValue::from_static("http://localhost:3000"),
);
assert!(validate_origin_and_host(&headers).is_ok());
}
#[test]
fn router_only_allows_post_and_options() {
assert_eq!(axum::http::Method::POST.as_str(), "POST");
assert_eq!(axum::http::Method::OPTIONS.as_str(), "OPTIONS");
}
}

View File

@@ -0,0 +1,408 @@
mod agent_config;
mod http;
mod protocol;
mod resources;
mod tools;
use std::path::{Path, PathBuf};
use serde_json::Value;
use uuid::Uuid;
use crate::db::queries::{mcp_proposal as proposal_q, project as project_q};
use crate::db::{Database, DbConnection};
use crate::engine::{EngineError, EngineResult, cli_sync, domain_events};
use crate::model::DomainEvent;
use crate::util::now_unix_ms;
pub use crate::model::{McpProposal, ProposalKind, ProposalStatus};
pub use agent_config::{
McpAgent, agent_config_path, install_agent_config, is_agent_configured,
packaged_mcp_executable, remove_agent_config,
};
pub use http::McpHttpServer;
pub use protocol::{MCP_PROTOCOL_VERSION, handle_rpc};
pub use resources::ResourceContent;
pub const DEFAULT_HTTP_PORT: u16 = 4124;
pub const PROPOSAL_TTL_MS: i64 = 30 * 60 * 1_000;
pub const PROPOSALS_EVENT_KEY: &str = "mcp.proposals";
#[derive(Debug, Clone)]
pub struct McpContext {
database_path: PathBuf,
}
impl McpContext {
pub fn new(database_path: PathBuf) -> Self {
Self { database_path }
}
pub fn database_path(&self) -> &Path {
&self.database_path
}
/// Prepare shared storage once when a transport starts. Individual
/// stateless read requests never run migrations or repair derived state.
pub fn prepare(&self) -> EngineResult<()> {
let db = Database::open(&self.database_path)?;
db.migrate()
.map_err(|error| EngineError::Parse(error.to_string()))?;
crate::engine::search::prepare_search_index(db.conn())?;
Ok(())
}
pub fn list_resources(&self) -> Vec<Value> {
resources::list()
}
pub fn list_resource_templates(&self) -> Vec<Value> {
resources::templates()
}
pub fn read_resource(&self, uri: &str) -> EngineResult<resources::ResourceContent> {
let db = self.open_database()?;
resources::read(db.conn(), uri)
}
pub fn list_tools(&self) -> Vec<Value> {
tools::list()
}
pub fn call_tool(&self, name: &str, params: Value) -> EngineResult<Value> {
let db = self.open_database()?;
tools::call(db.conn(), name, params)
}
pub(crate) fn open_database(&self) -> EngineResult<Database> {
Ok(Database::open(&self.database_path)?)
}
}
pub fn get_proposal(conn: &DbConnection, proposal_id: &str) -> EngineResult<McpProposal> {
proposal_q::get_proposal(conn, proposal_id)
.map_err(|_| EngineError::NotFound(format!("MCP proposal {proposal_id}")))
}
pub fn list_proposals(conn: &DbConnection, project_id: &str) -> EngineResult<Vec<McpProposal>> {
expire_proposals(conn)?;
Ok(proposal_q::list_proposals(conn, project_id)?)
}
pub fn list_pending_proposals(
conn: &DbConnection,
project_id: &str,
) -> EngineResult<Vec<McpProposal>> {
expire_proposals(conn)?;
Ok(proposal_q::list_pending_proposals(conn, project_id)?)
}
pub fn expire_proposals(conn: &DbConnection) -> EngineResult<usize> {
conn.begin_savepoint()?;
let expired = match proposal_q::expire_pending(conn, now_unix_ms()) {
Ok(expired) => {
conn.release_savepoint()?;
expired
}
Err(error) => {
let _ = conn.rollback_savepoint();
return Err(error.into());
}
};
if expired > 0 {
notify_proposals_changed();
}
Ok(expired)
}
pub(crate) fn create_proposal(
conn: &DbConnection,
kind: ProposalKind,
project_id: &str,
entity_id: Option<&str>,
data: &Value,
) -> EngineResult<McpProposal> {
expire_proposals(conn)?;
let now = now_unix_ms();
let entity_id = entity_id
.map(str::to_string)
.unwrap_or_else(|| Uuid::new_v4().to_string());
let proposal = McpProposal {
id: Uuid::new_v4().to_string(),
project_id: project_id.to_string(),
kind,
status: ProposalStatus::Pending,
entity_id,
data: serde_json::to_string(data)?,
result: None,
created_at: now,
expires_at: now + PROPOSAL_TTL_MS,
resolved_at: None,
};
if let Err(error) = proposal_q::insert_proposal(conn, &proposal) {
if matches!(
error,
diesel::result::Error::DatabaseError(
diesel::result::DatabaseErrorKind::UniqueViolation,
_
)
) {
return Err(EngineError::Conflict(format!(
"proposal already pending for {} entity {}",
proposal.kind.as_str(),
proposal.entity_id
)));
}
return Err(error.into());
}
cli_sync::record_cli_event(
conn,
&DomainEvent::SettingsChanged {
project_id: None,
key: PROPOSALS_EVENT_KEY.to_string(),
},
)?;
Ok(proposal)
}
pub fn accept_proposal(
conn: &DbConnection,
data_dir: &Path,
proposal_id: &str,
) -> EngineResult<McpProposal> {
resolve_proposal(conn, data_dir, proposal_id, true)
}
pub fn reject_proposal(
conn: &DbConnection,
data_dir: &Path,
proposal_id: &str,
) -> EngineResult<McpProposal> {
resolve_proposal(conn, data_dir, proposal_id, false)
}
fn resolve_proposal(
conn: &DbConnection,
data_dir: &Path,
proposal_id: &str,
accept: bool,
) -> EngineResult<McpProposal> {
expire_proposals(conn)?;
conn.begin_savepoint()?;
let outcome = (|| {
let now = now_unix_ms();
if !proposal_q::claim_pending(conn, proposal_id, now)? {
let current = get_proposal(conn, proposal_id)?;
return Err(EngineError::Conflict(format!(
"MCP proposal {} is {}",
current.id,
current.status.as_str()
)));
}
let proposal = get_proposal(conn, proposal_id)?;
let result = if accept {
execute_proposal(conn, data_dir, &proposal)?
} else {
serde_json::json!({"message": "rejected"})
};
let status = if accept {
ProposalStatus::Accepted
} else {
ProposalStatus::Rejected
};
proposal_q::delete_status_collision(conn, &proposal, status)?;
if !proposal_q::resolve_claimed(
conn,
proposal_id,
status,
&serde_json::to_string(&result)?,
now_unix_ms(),
)? {
return Err(EngineError::Conflict(format!(
"MCP proposal {proposal_id} was resolved concurrently"
)));
}
get_proposal(conn, proposal_id)
})();
match outcome {
Ok(proposal) => {
conn.release_savepoint()?;
notify_proposals_changed();
Ok(proposal)
}
Err(error) => {
let _ = conn.rollback_savepoint();
Err(error)
}
}
}
fn execute_proposal(
conn: &DbConnection,
data_dir: &Path,
proposal: &McpProposal,
) -> EngineResult<Value> {
let data: Value = serde_json::from_str(&proposal.data)?;
match proposal.kind {
ProposalKind::DraftPost => {
let post = crate::engine::post::create_post(
conn,
data_dir,
&proposal.project_id,
required_string(&data, "title")?,
Some(required_string(&data, "content")?),
string_array(&data, "tags"),
string_array(&data, "categories"),
optional_string(&data, "author"),
optional_string(&data, "language"),
None,
)?;
let post = if let Some(excerpt) = optional_string(&data, "excerpt") {
crate::engine::post::update_post(
conn,
data_dir,
&post.id,
None,
None,
Some(Some(excerpt)),
None,
None,
None,
None,
None,
None,
None,
)?
} else {
post
};
let post = crate::engine::post::publish_post(conn, data_dir, &post.id)?;
Ok(serde_json::to_value(post)?)
}
ProposalKind::ProposeScript => {
let kind = required_string(&data, "kind")?
.parse()
.map_err(EngineError::Validation)?;
let script = crate::engine::script::create_script(
conn,
&proposal.project_id,
required_string(&data, "title")?,
kind,
required_string(&data, "content")?,
optional_string(&data, "entrypoint"),
)?;
let script = crate::engine::script::publish_script(conn, data_dir, &script.id)?;
Ok(serde_json::to_value(script)?)
}
ProposalKind::ProposeTemplate => {
let kind = required_string(&data, "kind")?
.parse()
.map_err(EngineError::Validation)?;
let template = crate::engine::template::create_template(
conn,
&proposal.project_id,
required_string(&data, "title")?,
kind,
required_string(&data, "content")?,
)?;
let template = crate::engine::template::publish_template(conn, data_dir, &template.id)?;
Ok(serde_json::to_value(template)?)
}
ProposalKind::ProposeMediaTranslation => {
let translation = crate::engine::media::upsert_media_translation(
conn,
data_dir,
required_string(&data, "mediaId")?,
required_string(&data, "language")?,
optional_string(&data, "title"),
optional_string(&data, "alt"),
optional_string(&data, "caption"),
)?;
Ok(serde_json::to_value(translation)?)
}
ProposalKind::ProposeMediaMetadata => {
let media = crate::engine::media::update_media(
conn,
data_dir,
required_string(&data, "mediaId")?,
optional_optional_string(&data, "title"),
optional_optional_string(&data, "alt"),
optional_optional_string(&data, "caption"),
None,
None,
data.get("tags")
.is_some()
.then(|| string_array(&data, "tags")),
)?;
Ok(serde_json::to_value(media)?)
}
ProposalKind::ProposePostMetadata => {
let post = crate::engine::post::update_post(
conn,
data_dir,
required_string(&data, "postId")?,
optional_string(&data, "title"),
None,
optional_optional_string(&data, "excerpt"),
None,
data.get("tags")
.is_some()
.then(|| string_array(&data, "tags")),
data.get("categories")
.is_some()
.then(|| string_array(&data, "categories")),
None,
None,
None,
None,
)?;
Ok(serde_json::to_value(post)?)
}
}
}
pub(crate) fn active_project(
conn: &DbConnection,
) -> EngineResult<(crate::model::Project, PathBuf)> {
let project = project_q::get_active_project(conn)
.map_err(|_| EngineError::NotFound("active project".into()))?;
let data_dir = project
.data_path
.as_deref()
.map(PathBuf::from)
.ok_or_else(|| EngineError::Validation("active project has no data path".into()))?;
Ok((project, data_dir))
}
pub(crate) fn required_string<'a>(value: &'a Value, key: &str) -> EngineResult<&'a str> {
value
.get(key)
.and_then(Value::as_str)
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| EngineError::Validation(format!("{key} is required")))
}
pub(crate) fn optional_string<'a>(value: &'a Value, key: &str) -> Option<&'a str> {
value.get(key).and_then(Value::as_str)
}
fn optional_optional_string<'a>(value: &'a Value, key: &str) -> Option<Option<&'a str>> {
value.get(key).map(|value| value.as_str())
}
pub(crate) fn string_array(value: &Value, key: &str) -> Vec<String> {
value
.get(key)
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string)
.collect()
}
fn notify_proposals_changed() {
domain_events::settings_changed(None, PROPOSALS_EVENT_KEY);
}

View File

@@ -0,0 +1,130 @@
use serde_json::{Value, json};
use crate::engine::{EngineError, EngineResult};
use super::McpContext;
pub const MCP_PROTOCOL_VERSION: &str = "2025-06-18";
pub fn handle_rpc(context: &McpContext, request: &Value) -> Option<Value> {
let id = request.get("id").cloned();
let Some(method) = request.get("method").and_then(Value::as_str) else {
return Some(error(id.unwrap_or(Value::Null), -32600, "Invalid Request"));
};
if request.get("jsonrpc").and_then(Value::as_str) != Some("2.0") {
return Some(error(id.unwrap_or(Value::Null), -32600, "Invalid Request"));
}
let id = id?;
let params = request.get("params").cloned().unwrap_or_else(|| json!({}));
let result = match method {
"initialize" => Ok(json!({
"protocolVersion": negotiated_version(&params),
"capabilities": {
"tools": {"listChanged": false},
"resources": {"subscribe": false, "listChanged": false}
},
"serverInfo": {
"name": "Blogging Desktop Server",
"version": env!("CARGO_PKG_VERSION")
}
})),
"ping" => Ok(json!({})),
"tools/list" => Ok(json!({"tools": context.list_tools()})),
"tools/call" => call_tool(context, &params),
"resources/list" => Ok(json!({"resources": context.list_resources()})),
"resources/templates/list" => Ok(json!({
"resourceTemplates": context.list_resource_templates()
})),
"resources/read" => read_resource(context, &params),
_ => return Some(error(id, -32601, "Method not found")),
};
Some(match result {
Ok(result) => success(id, result),
Err(EngineError::NotFound(message)) => error(id, -32004, &message),
Err(EngineError::Validation(message) | EngineError::Conflict(message)) => {
error(id, -32602, &message)
}
Err(error_value) => error(id, -32000, &error_value.to_string()),
})
}
fn negotiated_version(params: &Value) -> &str {
match params.get("protocolVersion").and_then(Value::as_str) {
Some("2025-03-26") => "2025-03-26",
Some("2025-06-18") => "2025-06-18",
_ => MCP_PROTOCOL_VERSION,
}
}
fn call_tool(context: &McpContext, params: &Value) -> EngineResult<Value> {
let name = params
.get("name")
.and_then(Value::as_str)
.filter(|name| !name.is_empty())
.ok_or_else(|| EngineError::Validation("tool name is required".into()))?;
let arguments = params
.get("arguments")
.cloned()
.unwrap_or_else(|| json!({}));
let result = context.call_tool(name, arguments)?;
Ok(json!({
"content": [{
"type": "text",
"text": serde_json::to_string(&result)?
}],
"structuredContent": result,
"isError": false
}))
}
fn read_resource(context: &McpContext, params: &Value) -> EngineResult<Value> {
let uri = params
.get("uri")
.and_then(Value::as_str)
.filter(|uri| !uri.is_empty())
.ok_or_else(|| EngineError::Validation("resource URI is required".into()))?;
let content = context.read_resource(uri)?;
let content = if let Some(blob) = content.blob {
json!({
"uri": content.uri,
"mimeType": content.mime_type,
"blob": blob
})
} else {
json!({
"uri": content.uri,
"mimeType": content.mime_type,
"text": content.text.unwrap_or_default()
})
};
Ok(json!({"contents": [content]}))
}
fn success(id: Value, result: Value) -> Value {
json!({"jsonrpc":"2.0","id":id,"result":result})
}
pub(crate) fn error(id: Value, code: i64, message: &str) -> Value {
json!({"jsonrpc":"2.0","id":id,"error":{"code":code,"message":message}})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn notifications_have_no_response_and_invalid_requests_are_rejected() {
let context = McpContext::new("missing.sqlite".into());
assert!(
handle_rpc(
&context,
&json!({"jsonrpc":"2.0","method":"notifications/initialized"})
)
.is_none()
);
assert_eq!(
handle_rpc(&context, &json!({"jsonrpc":"2.0","id":1})).unwrap()["error"]["code"],
-32600
);
}
}

View File

@@ -0,0 +1,382 @@
use std::path::Path;
use base64::Engine as _;
use serde_json::{Value, json};
use crate::db::DbConnection;
use crate::db::queries::{media as media_q, post as post_q, post_link, post_media, tag as tag_q};
use crate::engine::{EngineError, EngineResult};
use crate::model::{Media, Post};
use super::active_project;
const PAGE_SIZE: usize = 50;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResourceContent {
pub uri: String,
pub mime_type: String,
pub text: Option<String>,
pub blob: Option<String>,
}
impl ResourceContent {
fn json(uri: &str, value: &Value) -> EngineResult<Self> {
Ok(Self {
uri: uri.to_string(),
mime_type: "application/json".into(),
text: Some(serde_json::to_string(value)?),
blob: None,
})
}
}
pub fn list() -> Vec<Value> {
[
("project", "Active project", "bds://project"),
("posts", "Blog posts", "bds://posts"),
("media", "Media", "bds://media"),
("tags", "Tags", "bds://tags"),
("categories", "Categories", "bds://categories"),
("stats", "Blog statistics", "bds://stats"),
]
.into_iter()
.map(|(name, title, uri)| {
json!({
"name": name,
"title": title,
"uri": uri,
"mimeType": "application/json"
})
})
.collect()
}
pub fn templates() -> Vec<Value> {
[
("posts", "Paginated blog posts", "bds://posts{?cursor}"),
("media", "Paginated media", "bds://media{?cursor}"),
(
"post media",
"Media linked to a post",
"bds://posts/{id}/media",
),
(
"media image",
"Original media bytes",
"bds://media/{id}/image",
),
]
.into_iter()
.map(|(name, title, uri_template)| {
json!({
"name": name,
"title": title,
"uriTemplate": uri_template
})
})
.collect()
}
pub fn read(conn: &DbConnection, uri: &str) -> EngineResult<ResourceContent> {
let url = url::Url::parse(uri)
.map_err(|_| EngineError::Validation("invalid MCP resource URI".into()))?;
if url.scheme() != "bds" {
return Err(EngineError::NotFound(uri.into()));
}
let host = url
.host_str()
.ok_or_else(|| EngineError::NotFound(uri.into()))?;
let path = url.path().trim_matches('/');
let (project, data_dir) = active_project(conn)?;
let value = match (host, path) {
("project", "") => project_resource(&project, &data_dir),
("posts", "") => posts_page(conn, &project.id, cursor_offset(&url)?),
("media", "") => media_page(conn, &project.id, cursor_offset(&url)?),
("tags", "") => tags(conn, &project.id),
("categories", "") => categories(conn, &project.id, &data_dir),
("stats", "") => stats(conn, &project.id, &data_dir),
("posts", path) => {
let parts = path.split('/').collect::<Vec<_>>();
match parts.as_slice() {
[post_id] => post_detail_by_id(conn, &project.id, &data_dir, post_id),
[post_id, "media"] => post_media_items(conn, &project.id, post_id),
_ => return Err(EngineError::NotFound(uri.into())),
}
}
("media", path) => {
let parts = path.split('/').collect::<Vec<_>>();
match parts.as_slice() {
[media_id] => media_detail_by_id(conn, &project.id, media_id),
[media_id, "image"] => {
return media_image(conn, &project.id, &data_dir, media_id, uri);
}
_ => return Err(EngineError::NotFound(uri.into())),
}
}
_ => return Err(EngineError::NotFound(uri.into())),
}?;
ResourceContent::json(uri, &value)
}
fn project_resource(project: &crate::model::Project, data_dir: &Path) -> EngineResult<Value> {
let metadata = crate::engine::meta::read_project_json(data_dir).ok();
Ok(json!({
"id": project.id,
"name": project.name,
"slug": project.slug,
"description": project.description,
"public_url": metadata.as_ref().and_then(|value| value.public_url.clone()),
"main_language": metadata.as_ref().and_then(|value| value.main_language.clone()),
"blog_languages": metadata.map(|value| value.blog_languages).unwrap_or_default()
}))
}
fn cursor_offset(url: &url::Url) -> EngineResult<usize> {
let Some(cursor) = url
.query_pairs()
.find_map(|(key, value)| (key == "cursor").then(|| value.into_owned()))
else {
return Ok(0);
};
if cursor.is_empty() {
return Err(EngineError::Validation("invalid cursor".into()));
}
let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD
.decode(cursor)
.map_err(|_| EngineError::Validation("invalid cursor".into()))?;
let value: Value = serde_json::from_slice(&decoded)
.map_err(|_| EngineError::Validation("invalid cursor".into()))?;
value["offset"]
.as_u64()
.map(|offset| offset as usize)
.ok_or_else(|| EngineError::Validation("invalid cursor".into()))
}
fn encode_cursor(offset: usize) -> String {
base64::engine::general_purpose::URL_SAFE_NO_PAD
.encode(serde_json::to_vec(&json!({"offset": offset})).unwrap_or_default())
}
fn page(items: Vec<Value>, total: usize, offset: usize) -> Value {
let mut value = json!({
"items": items,
"total": total,
"offset": offset,
"limit": PAGE_SIZE
});
let next = offset.saturating_add(PAGE_SIZE);
if next < total {
value["nextCursor"] = Value::String(encode_cursor(next));
}
value
}
fn posts_page(conn: &DbConnection, project_id: &str, offset: usize) -> EngineResult<Value> {
let posts = post_q::list_posts_by_project(conn, project_id)?;
let total = posts.len();
let items = posts
.into_iter()
.skip(offset)
.take(PAGE_SIZE)
.map(|post| post_summary(conn, &post))
.collect::<EngineResult<Vec<_>>>()?;
Ok(page(items, total, offset))
}
fn media_page(conn: &DbConnection, project_id: &str, offset: usize) -> EngineResult<Value> {
let media = media_q::list_media_by_project(conn, project_id)?;
let total = media.len();
let items = media
.into_iter()
.skip(offset)
.take(PAGE_SIZE)
.map(|item| media_summary(&item))
.collect();
Ok(page(items, total, offset))
}
pub(crate) fn post_summary(conn: &DbConnection, post: &Post) -> EngineResult<Value> {
Ok(json!({
"id": post.id,
"title": post.title,
"slug": post.slug,
"status": post.status,
"tags": post.tags,
"categories": post.categories,
"created_at": post.created_at,
"backlinks": linked_posts(conn, &post.id, false)?,
"linksTo": linked_posts(conn, &post.id, true)?
}))
}
fn linked_posts(conn: &DbConnection, post_id: &str, outgoing: bool) -> EngineResult<Vec<Value>> {
let links = if outgoing {
post_link::list_links_by_source(conn, post_id)?
.into_iter()
.map(|link| link.target_post_id)
.collect::<Vec<_>>()
} else {
post_link::list_links_by_target(conn, post_id)?
.into_iter()
.map(|link| link.source_post_id)
.collect::<Vec<_>>()
};
Ok(links
.into_iter()
.filter_map(|id| post_q::get_post_by_id(conn, &id).ok())
.map(|post| json!({"id": post.id, "title": post.title, "slug": post.slug}))
.collect())
}
pub(crate) fn post_detail(
conn: &DbConnection,
data_dir: &Path,
post: &Post,
) -> EngineResult<Value> {
let mut value = serde_json::to_value(post)?;
value["content"] = Value::String(post_body(data_dir, post));
value["backlinks"] = Value::Array(linked_posts(conn, &post.id, false)?);
value["linksTo"] = Value::Array(linked_posts(conn, &post.id, true)?);
let translations =
crate::db::queries::post_translation::list_post_translations_by_post(conn, &post.id)?;
let mut languages = post.language.clone().into_iter().collect::<Vec<_>>();
languages.extend(
translations
.into_iter()
.map(|translation| translation.language),
);
languages.sort();
languages.dedup();
value["availableLanguages"] = serde_json::to_value(languages)?;
Ok(value)
}
fn post_body(data_dir: &Path, post: &Post) -> String {
if let Some(content) = &post.content {
return content.clone();
}
if post.file_path.is_empty() {
return String::new();
}
std::fs::read_to_string(data_dir.join(&post.file_path))
.ok()
.and_then(|source| {
crate::util::frontmatter::read_post_file(&source)
.ok()
.map(|(_, body)| body)
})
.unwrap_or_default()
}
fn post_detail_by_id(
conn: &DbConnection,
project_id: &str,
data_dir: &Path,
id: &str,
) -> EngineResult<Value> {
let post = post_q::get_post_by_id(conn, id)
.map_err(|_| EngineError::NotFound(format!("post {id}")))?;
ensure_project(project_id, &post.project_id, "post", id)?;
post_detail(conn, data_dir, &post)
}
pub(crate) fn media_summary(media: &Media) -> Value {
json!({
"id": media.id,
"filename": media.filename,
"title": media.title,
"alt": media.alt,
"caption": media.caption,
"tags": media.tags
})
}
fn media_detail_by_id(conn: &DbConnection, project_id: &str, id: &str) -> EngineResult<Value> {
let media = media_q::get_media_by_id(conn, id)
.map_err(|_| EngineError::NotFound(format!("media {id}")))?;
ensure_project(project_id, &media.project_id, "media", id)?;
Ok(serde_json::to_value(media)?)
}
fn post_media_items(conn: &DbConnection, project_id: &str, post_id: &str) -> EngineResult<Value> {
let post = post_q::get_post_by_id(conn, post_id)
.map_err(|_| EngineError::NotFound(format!("post {post_id}")))?;
ensure_project(project_id, &post.project_id, "post", post_id)?;
let items = post_media::list_post_media_by_post(conn, post_id)?
.into_iter()
.filter_map(|link| media_q::get_media_by_id(conn, &link.media_id).ok())
.map(|media| media_summary(&media))
.collect::<Vec<_>>();
Ok(json!({"items": items}))
}
fn media_image(
conn: &DbConnection,
project_id: &str,
data_dir: &Path,
media_id: &str,
uri: &str,
) -> EngineResult<ResourceContent> {
let media = media_q::get_media_by_id(conn, media_id)
.map_err(|_| EngineError::NotFound(format!("media {media_id}")))?;
ensure_project(project_id, &media.project_id, "media", media_id)?;
let bytes = std::fs::read(data_dir.join(&media.file_path))
.map_err(|_| EngineError::NotFound(format!("media file {media_id}")))?;
Ok(ResourceContent {
uri: uri.to_string(),
mime_type: media.mime_type,
text: None,
blob: Some(base64::engine::general_purpose::STANDARD.encode(bytes)),
})
}
fn tags(conn: &DbConnection, project_id: &str) -> EngineResult<Value> {
let posts = post_q::list_posts_by_project(conn, project_id)?;
let items = tag_q::list_tags_by_project(conn, project_id)?
.into_iter()
.map(|tag| {
let count = posts
.iter()
.filter(|post| post.tags.iter().any(|name| name == &tag.name))
.count();
json!({"name": tag.name, "color": tag.color, "post_count": count})
})
.collect::<Vec<_>>();
Ok(json!({"items": items}))
}
fn categories(conn: &DbConnection, project_id: &str, data_dir: &Path) -> EngineResult<Value> {
let posts = post_q::list_posts_by_project(conn, project_id)?;
let names = crate::engine::meta::read_categories_json(data_dir)
.unwrap_or_else(|_| post_q::distinct_post_categories(conn, project_id).unwrap_or_default());
let items = names
.into_iter()
.map(|name| {
let count = posts
.iter()
.filter(|post| post.categories.iter().any(|value| value == &name))
.count();
json!({"name": name, "post_count": count})
})
.collect::<Vec<_>>();
Ok(json!({"items": items}))
}
fn stats(conn: &DbConnection, project_id: &str, data_dir: &Path) -> EngineResult<Value> {
let categories = categories(conn, project_id, data_dir)?;
Ok(json!({
"post_count": post_q::count_posts_by_project(conn, project_id)?,
"media_count": media_q::count_media_by_project(conn, project_id)?,
"tag_count": tag_q::list_tags_by_project(conn, project_id)?.len(),
"category_count": categories["items"].as_array().map_or(0, Vec::len)
}))
}
fn ensure_project(expected: &str, actual: &str, entity: &str, id: &str) -> EngineResult<()> {
if expected == actual {
Ok(())
} else {
Err(EngineError::NotFound(format!("{entity} {id}")))
}
}

View File

@@ -0,0 +1,646 @@
use std::collections::BTreeMap;
use chrono::{Datelike, TimeZone as _, Utc};
use serde_json::{Map, Value, json};
use crate::db::DbConnection;
use crate::db::queries::{media as media_q, media_translation, post as post_q, post_translation};
use crate::engine::{EngineError, EngineResult};
use crate::model::{Post, ProposalKind};
use super::resources::{post_detail, post_summary};
use super::{active_project, create_proposal, optional_string, required_string, string_array};
const MAX_PAGE_SIZE: usize = 50;
pub fn list() -> Vec<Value> {
[
tool(
"check_term",
"Check Term",
"Check whether a term is a category, a tag, or both, with post counts.",
object_schema(json!({"term": string_schema("Term to check")}), &["term"]),
true,
),
tool(
"search_posts",
"Search Posts",
"Full-text and filtered post search with pagination, backlinks, and outgoing links.",
post_query_schema(false),
true,
),
tool(
"count_posts",
"Count Posts",
"Count filtered posts grouped by year, month, tag, category, or status.",
count_schema(),
true,
),
tool(
"read_post_by_slug",
"Read Post By Slug",
"Read full post content and metadata, optionally in a translated language.",
object_schema(
json!({
"slug": string_schema("Post slug"),
"language": string_schema("Optional translation language")
}),
&["slug"],
),
true,
),
tool(
"get_post_translations",
"Get Post Translations",
"List every translation for a post.",
id_schema("postId", "Post ID"),
true,
),
tool(
"get_media_translations",
"Get Media Translations",
"List every translated metadata record for a media item.",
id_schema("mediaId", "Media ID"),
true,
),
tool(
"upsert_media_translation",
"Propose Media Translation",
"Propose translated media metadata for explicit desktop approval.",
object_schema(
json!({
"mediaId": string_schema("Media ID"),
"language": string_schema("Language code"),
"title": string_schema("Translated title"),
"alt": string_schema("Translated alt text"),
"caption": string_schema("Translated caption")
}),
&["mediaId", "language"],
),
false,
),
tool(
"draft_post",
"Draft Post",
"Propose a post. No post is created until explicit desktop approval.",
object_schema(
json!({
"title": string_schema("Post title"),
"content": string_schema("Markdown body"),
"excerpt": string_schema("Excerpt"),
"tags": string_array_schema("Tags"),
"categories": string_array_schema("Categories"),
"author": string_schema("Author"),
"language": string_schema("Language")
}),
&["title", "content"],
),
false,
),
tool(
"propose_script",
"Propose Script",
"Validate and propose a Lua script for explicit desktop approval.",
object_schema(
json!({
"title": string_schema("Script title"),
"kind": {"type":"string","enum":["macro","utility","transform"]},
"content": string_schema("Lua source"),
"entrypoint": string_schema("Entrypoint function")
}),
&["title", "kind", "content"],
),
false,
),
tool(
"propose_template",
"Propose Template",
"Validate and propose a Liquid template for explicit desktop approval.",
object_schema(
json!({
"title": string_schema("Template title"),
"kind": {"type":"string","enum":["post","list","not-found","partial"]},
"content": string_schema("Liquid source")
}),
&["title", "kind", "content"],
),
false,
),
tool(
"propose_media_metadata",
"Propose Media Metadata",
"Propose media metadata changes for explicit desktop approval.",
object_schema(
json!({
"mediaId": string_schema("Media ID"),
"title": nullable_string_schema("Title"),
"alt": nullable_string_schema("Alt text"),
"caption": nullable_string_schema("Caption"),
"tags": string_array_schema("Tags")
}),
&["mediaId"],
),
false,
),
tool(
"propose_post_metadata",
"Propose Post Metadata",
"Propose post metadata changes for explicit desktop approval.",
object_schema(
json!({
"postId": string_schema("Post ID"),
"title": string_schema("Title"),
"excerpt": nullable_string_schema("Excerpt"),
"tags": string_array_schema("Tags"),
"categories": string_array_schema("Categories")
}),
&["postId"],
),
false,
),
]
.into_iter()
.collect()
}
pub fn call(conn: &DbConnection, name: &str, params: Value) -> EngineResult<Value> {
if !params.is_object() {
return Err(EngineError::Validation(
"tool arguments must be an object".into(),
));
}
match name {
"check_term" => check_term(conn, &params),
"search_posts" => search_posts(conn, &params),
"count_posts" => count_posts(conn, &params),
"read_post_by_slug" => read_post_by_slug(conn, &params),
"get_post_translations" => get_post_translations(conn, &params),
"get_media_translations" => get_media_translations(conn, &params),
"upsert_media_translation" => propose_media_translation(conn, &params),
"draft_post" => propose(conn, ProposalKind::DraftPost, None, &params),
"propose_script" => propose_script(conn, &params),
"propose_template" => propose_template(conn, &params),
"propose_media_metadata" => propose_media_metadata(conn, &params),
"propose_post_metadata" => propose_post_metadata(conn, &params),
_ => Err(EngineError::NotFound(format!("MCP tool {name}"))),
}
}
fn tool(name: &str, title: &str, description: &str, schema: Value, read_only: bool) -> Value {
json!({
"name": name,
"title": title,
"description": description,
"inputSchema": schema,
"annotations": {
"readOnlyHint": read_only,
"destructiveHint": false,
"openWorldHint": false
}
})
}
fn string_schema(description: &str) -> Value {
json!({"type":"string","description":description})
}
fn nullable_string_schema(description: &str) -> Value {
json!({"type":["string","null"],"description":description})
}
fn string_array_schema(description: &str) -> Value {
json!({"type":"array","items":{"type":"string"},"description":description})
}
fn object_schema(properties: Value, required: &[&str]) -> Value {
let mut schema = json!({
"type": "object",
"properties": properties,
"additionalProperties": false
});
if !required.is_empty() {
schema["required"] = serde_json::to_value(required).unwrap_or_default();
}
schema
}
fn id_schema(field: &str, description: &str) -> Value {
let mut properties = Map::new();
properties.insert(field.into(), string_schema(description));
object_schema(Value::Object(properties), &[field])
}
fn post_query_schema(query_required: bool) -> Value {
object_schema(
json!({
"query": string_schema("Full-text query"),
"category": string_schema("Category filter"),
"tags": string_array_schema("All required tags"),
"language": string_schema("Available language"),
"missingTranslationLanguage": string_schema("Missing translation language"),
"year": {"type":"integer"},
"month": {"type":"integer","minimum":1,"maximum":12},
"status": {"type":"string","enum":["draft","published","archived"]},
"offset": {"type":"integer","minimum":0},
"limit": {"type":"integer","minimum":1,"maximum":MAX_PAGE_SIZE}
}),
if query_required { &["query"] } else { &[] },
)
}
fn count_schema() -> Value {
let mut schema = post_query_schema(false);
schema["properties"]["groupBy"] = json!({"type":"array","items":{"type":"string","enum":["year","month","tag","category","status"]},"minItems":1});
schema["required"] = json!(["groupBy"]);
schema
}
fn check_term(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let term = required_string(params, "term")?;
let (project, _) = active_project(conn)?;
let posts = post_q::list_posts_by_project(conn, &project.id)?;
let normalized = term.to_lowercase();
let tag_count = posts
.iter()
.filter(|post| post.tags.iter().any(|tag| tag.to_lowercase() == normalized))
.count();
let category_count = posts
.iter()
.filter(|post| {
post.categories
.iter()
.any(|category| category.to_lowercase() == normalized)
})
.count();
Ok(json!({
"is_category": category_count > 0,
"category_post_count": category_count,
"is_tag": tag_count > 0,
"tag_post_count": tag_count
}))
}
fn filtered_posts(conn: &DbConnection, params: &Value) -> EngineResult<Vec<Post>> {
let (project, _) = active_project(conn)?;
let mut posts = post_q::list_posts_by_project(conn, &project.id)?;
let query = optional_string(params, "query")
.unwrap_or_default()
.trim()
.to_lowercase();
let fts_matches = if query.is_empty() {
None
} else {
let language = optional_string(params, "language").unwrap_or("en");
Some(
crate::db::fts::search_posts(conn, &query, language)?
.into_iter()
.collect::<std::collections::HashSet<_>>(),
)
};
let category = optional_string(params, "category").map(str::to_lowercase);
let tags = string_array(params, "tags")
.into_iter()
.map(|tag| tag.to_lowercase())
.collect::<Vec<_>>();
let language = optional_string(params, "language").map(str::to_lowercase);
let missing_language =
optional_string(params, "missingTranslationLanguage").map(str::to_lowercase);
let status = optional_string(params, "status");
let year = integer(params, "year")?.map(|value| value as i32);
let month = integer(params, "month")?.map(|value| value as u32);
if month.is_some() && year.is_none() {
return Err(EngineError::Validation("month requires year".into()));
}
posts.retain(|post| {
if fts_matches
.as_ref()
.is_some_and(|matches| !matches.contains(&post.id))
{
return false;
}
if category.as_ref().is_some_and(|wanted| {
!post
.categories
.iter()
.any(|value| value.to_lowercase() == *wanted)
}) {
return false;
}
if tags.iter().any(|wanted| {
!post
.tags
.iter()
.any(|value| value.to_lowercase() == *wanted)
}) {
return false;
}
if status.is_some_and(|wanted| post.status.as_str() != wanted) {
return false;
}
if let Some(wanted) = &language {
let canonical = post
.language
.as_deref()
.is_some_and(|value| value.eq_ignore_ascii_case(wanted));
let translated =
post_translation::get_post_translation_by_post_and_language(conn, &post.id, wanted)
.is_ok();
if !canonical && !translated {
return false;
}
}
if let Some(wanted) = &missing_language {
let canonical = post
.language
.as_deref()
.is_some_and(|value| value.eq_ignore_ascii_case(wanted));
let translated =
post_translation::get_post_translation_by_post_and_language(conn, &post.id, wanted)
.is_ok();
if canonical || translated {
return false;
}
}
if let Some(wanted_year) = year {
let Some(date) = Utc.timestamp_millis_opt(post.created_at).single() else {
return false;
};
if date.year() != wanted_year || month.is_some_and(|wanted| date.month() != wanted) {
return false;
}
}
true
});
Ok(posts)
}
fn search_posts(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let posts = filtered_posts(conn, params)?;
let total = posts.len();
let offset = unsigned(params, "offset")?.unwrap_or(0);
let limit = unsigned(params, "limit")?.unwrap_or(MAX_PAGE_SIZE);
if limit == 0 || limit > MAX_PAGE_SIZE {
return Err(EngineError::Validation(format!(
"limit must be between 1 and {MAX_PAGE_SIZE}"
)));
}
let posts = posts
.into_iter()
.skip(offset)
.take(limit)
.map(|post| post_summary(conn, &post))
.collect::<EngineResult<Vec<_>>>()?;
Ok(json!({
"total": total,
"offset": offset,
"limit": limit,
"hasMore": offset.saturating_add(limit) < total,
"posts": posts
}))
}
fn count_posts(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let group_by = string_array(params, "groupBy");
if group_by.is_empty()
|| group_by.iter().any(|dimension| {
!["year", "month", "tag", "category", "status"].contains(&dimension.as_str())
})
{
return Err(EngineError::Validation("invalid groupBy".into()));
}
let posts = filtered_posts(conn, params)?;
let total = posts.len();
let mut counts = BTreeMap::<String, (Map<String, Value>, usize)>::new();
for post in posts {
for row in group_rows(&post, &group_by) {
let key = serde_json::to_string(&row)?;
counts
.entry(key)
.and_modify(|(_, count)| *count += 1)
.or_insert((row, 1));
}
}
let groups = counts
.into_values()
.map(|(mut row, count)| {
row.insert("count".into(), json!(count));
Value::Object(row)
})
.collect::<Vec<_>>();
Ok(json!({"groups": groups, "totalPosts": total}))
}
fn group_rows(post: &Post, dimensions: &[String]) -> Vec<Map<String, Value>> {
let Some((dimension, rest)) = dimensions.split_first() else {
return vec![Map::new()];
};
let values = match dimension.as_str() {
"year" => Utc
.timestamp_millis_opt(post.created_at)
.single()
.map(|date| vec![json!(date.year())])
.unwrap_or_else(|| vec![Value::Null]),
"month" => Utc
.timestamp_millis_opt(post.created_at)
.single()
.map(|date| vec![json!(date.month())])
.unwrap_or_else(|| vec![Value::Null]),
"tag" => values_or_null(&post.tags),
"category" => values_or_null(&post.categories),
"status" => vec![json!(post.status.as_str())],
_ => vec![Value::Null],
};
values
.into_iter()
.flat_map(|value| {
group_rows(post, rest).into_iter().map({
let dimension = dimension.clone();
move |mut row| {
row.insert(dimension.clone(), value.clone());
row
}
})
})
.collect()
}
fn values_or_null(values: &[String]) -> Vec<Value> {
if values.is_empty() {
vec![Value::Null]
} else {
values.iter().map(|value| json!(value)).collect()
}
}
fn read_post_by_slug(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let slug = required_string(params, "slug")?;
let (project, data_dir) = active_project(conn)?;
let post = post_q::get_post_by_project_and_slug(conn, &project.id, slug)
.map_err(|_| EngineError::NotFound(format!("post slug {slug}")))?;
let Some(language) = optional_string(params, "language") else {
return Ok(json!({"post": post_detail(conn, &data_dir, &post)?}));
};
if post
.language
.as_deref()
.is_some_and(|canonical| canonical.eq_ignore_ascii_case(language))
{
return Ok(json!({"post": post_detail(conn, &data_dir, &post)?}));
}
let translation =
post_translation::get_post_translation_by_post_and_language(conn, &post.id, language)
.map_err(|_| EngineError::NotFound(format!("post translation {language}")))?;
let mut detail = post_detail(conn, &data_dir, &post)?;
detail["title"] = json!(translation.title);
detail["excerpt"] = json!(translation.excerpt);
detail["content"] = json!(translation_content(&data_dir, &translation));
detail["language"] = json!(translation.language);
detail["canonicalLanguage"] = json!(post.language);
Ok(json!({"post": detail}))
}
fn translation_content(
data_dir: &std::path::Path,
translation: &crate::model::PostTranslation,
) -> String {
if let Some(content) = &translation.content {
return content.clone();
}
std::fs::read_to_string(data_dir.join(&translation.file_path))
.ok()
.and_then(|source| {
crate::util::frontmatter::read_translation_file(&source)
.ok()
.map(|(_, body)| body)
})
.unwrap_or_default()
}
fn get_post_translations(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let id = required_string(params, "postId")?;
let (project, data_dir) = active_project(conn)?;
let post = post_q::get_post_by_id(conn, id)
.map_err(|_| EngineError::NotFound(format!("post {id}")))?;
if post.project_id != project.id {
return Err(EngineError::NotFound(format!("post {id}")));
}
let translations = post_translation::list_post_translations_by_post(conn, id)?
.into_iter()
.map(|translation| {
let mut value = serde_json::to_value(&translation)?;
value["content"] = json!(translation_content(&data_dir, &translation));
Ok(value)
})
.collect::<EngineResult<Vec<_>>>()?;
Ok(json!({"translations": translations}))
}
fn get_media_translations(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let id = required_string(params, "mediaId")?;
let (project, _) = active_project(conn)?;
let media = media_q::get_media_by_id(conn, id)
.map_err(|_| EngineError::NotFound(format!("media {id}")))?;
if media.project_id != project.id {
return Err(EngineError::NotFound(format!("media {id}")));
}
Ok(json!({
"translations": media_translation::list_media_translations_by_media(conn, id)?
}))
}
fn propose_media_translation(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let id = required_string(params, "mediaId")?;
required_string(params, "language")?;
let (project, _) = active_project(conn)?;
let media = media_q::get_media_by_id(conn, id)
.map_err(|_| EngineError::NotFound(format!("media {id}")))?;
if media.project_id != project.id {
return Err(EngineError::NotFound(format!("media {id}")));
}
propose(
conn,
ProposalKind::ProposeMediaTranslation,
Some(id),
params,
)
}
fn propose_script(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
required_string(params, "title")?;
let content = required_string(params, "content")?;
required_string(params, "kind")?
.parse::<crate::model::ScriptKind>()
.map_err(EngineError::Validation)?;
crate::engine::script::validate_script_syntax(content).map_err(EngineError::Validation)?;
propose(conn, ProposalKind::ProposeScript, None, params)
}
fn propose_template(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
required_string(params, "title")?;
let content = required_string(params, "content")?;
required_string(params, "kind")?
.parse::<crate::model::TemplateKind>()
.map_err(EngineError::Validation)?;
crate::engine::template::validate_template(content).map_err(EngineError::Validation)?;
propose(conn, ProposalKind::ProposeTemplate, None, params)
}
fn propose_media_metadata(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let id = required_string(params, "mediaId")?;
let (project, _) = active_project(conn)?;
let media = media_q::get_media_by_id(conn, id)
.map_err(|_| EngineError::NotFound(format!("media {id}")))?;
if media.project_id != project.id {
return Err(EngineError::NotFound(format!("media {id}")));
}
propose(conn, ProposalKind::ProposeMediaMetadata, Some(id), params)
}
fn propose_post_metadata(conn: &DbConnection, params: &Value) -> EngineResult<Value> {
let id = required_string(params, "postId")?;
let (project, _) = active_project(conn)?;
let post = post_q::get_post_by_id(conn, id)
.map_err(|_| EngineError::NotFound(format!("post {id}")))?;
if post.project_id != project.id {
return Err(EngineError::NotFound(format!("post {id}")));
}
propose(conn, ProposalKind::ProposePostMetadata, Some(id), params)
}
fn propose(
conn: &DbConnection,
kind: ProposalKind,
entity_id: Option<&str>,
params: &Value,
) -> EngineResult<Value> {
if kind == ProposalKind::DraftPost {
required_string(params, "title")?;
required_string(params, "content")?;
}
let (project, _) = active_project(conn)?;
let proposal = create_proposal(conn, kind, &project.id, entity_id, params)?;
Ok(json!({
"proposalId": proposal.id,
"status": proposal.status,
"expiresAt": proposal.expires_at,
"message": "Pending explicit approval in RuDS Settings"
}))
}
fn integer(value: &Value, key: &str) -> EngineResult<Option<i64>> {
match value.get(key) {
None => Ok(None),
Some(value) => value
.as_i64()
.map(Some)
.ok_or_else(|| EngineError::Validation(format!("{key} must be an integer"))),
}
}
fn unsigned(value: &Value, key: &str) -> EngineResult<Option<usize>> {
integer(value, key)?.map_or(Ok(None), |value| {
usize::try_from(value)
.map(Some)
.map_err(|_| EngineError::Validation(format!("{key} cannot be negative")))
})
}

View File

@@ -10,8 +10,8 @@ use crate::db::queries::media as qm;
use crate::db::queries::media_translation as qmt;
use crate::db::queries::post as qp;
use crate::db::queries::post_media as qpm;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Media, MediaTranslation, PostMedia};
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::model::{DomainEntity, Media, MediaTranslation, NotificationAction, PostMedia};
use crate::util::sidecar::{
MediaSidecar, MediaTranslationSidecar, read_sidecar, read_translation_sidecar,
};
@@ -20,7 +20,7 @@ use crate::util::thumbnail::{
mime_from_extension,
};
use crate::util::{
atomic_write_str, content_hash, media_dir_path, media_sidecar_path,
atomic_write_str, media_dir_path, media_file_hash, media_sidecar_path,
media_translation_sidecar_path, now_unix_ms,
};
@@ -34,6 +34,151 @@ pub struct MediaRebuildReport {
pub errors: Vec<String>,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct MediaLinkRebuildReport {
pub links: usize,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct ThumbnailRepairReport {
pub media_processed: usize,
pub media_repaired: usize,
pub media_failed: usize,
pub thumbnails_generated: usize,
}
/// Rebuild the exact post/media relationship set stored in canonical media
/// sidecars. Stale database links are removed as well as missing links added.
pub fn rebuild_media_links(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<MediaLinkRebuildReport> {
rebuild_media_links_with_progress(conn, data_dir, project_id, None)
}
pub fn rebuild_media_links_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
on_item: Option<ItemProgressFn>,
) -> EngineResult<MediaLinkRebuildReport> {
conn.begin_savepoint()?;
match rebuild_media_links_inner(conn, data_dir, project_id, on_item.as_ref()) {
Ok(report) => {
conn.release_savepoint()?;
Ok(report)
}
Err(error) => {
let _ = conn.rollback_savepoint();
Err(error)
}
}
}
fn rebuild_media_links_inner(
conn: &Connection,
data_dir: &Path,
project_id: &str,
on_item: Option<&ItemProgressFn>,
) -> EngineResult<MediaLinkRebuildReport> {
let mut report = MediaLinkRebuildReport::default();
let items = qm::list_media_by_project(conn, project_id)?;
for (index, item) in items.iter().enumerate() {
if let Some(callback) = on_item
&& !callback(index + 1, items.len(), &item.original_name)
{
return Err(EngineError::Cancelled);
}
let sidecar = read_sidecar(&fs::read_to_string(data_dir.join(&item.sidecar_path))?)
.map_err(EngineError::Parse)?;
for link in qpm::list_post_media_by_media(conn, &item.id)? {
qpm::unlink_media(conn, &link.post_id, &item.id)?;
}
for (sort_order, post_id) in sidecar.linked_post_ids.into_iter().enumerate() {
let post = qp::get_post_by_id(conn, &post_id)?;
if post.project_id != project_id {
return Err(EngineError::Validation(format!(
"media {} sidecar links to a post outside the active project",
item.id
)));
}
qpm::link_media(
conn,
&PostMedia {
id: Uuid::new_v4().to_string(),
project_id: project_id.to_string(),
post_id,
media_id: item.id.clone(),
sort_order: sort_order as i32,
created_at: now_unix_ms(),
},
)?;
report.links += 1;
}
}
Ok(report)
}
/// Regenerate all standard thumbnail variants for items missing at least one
/// variant. Existing complete sets are left untouched.
pub fn regenerate_missing_thumbnails(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<ThumbnailRepairReport> {
regenerate_missing_thumbnails_with_progress(conn, data_dir, project_id, |_, _, _| true)
}
pub fn regenerate_missing_thumbnails_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
mut on_item: impl FnMut(usize, usize, &str) -> bool,
) -> EngineResult<ThumbnailRepairReport> {
let mut report = ThumbnailRepairReport::default();
let items = qm::list_media_by_project(conn, project_id)?;
for (index, item) in items.iter().enumerate() {
if !on_item(index + 1, items.len(), &item.original_name) {
return Err(EngineError::Cancelled);
}
if !item.mime_type.starts_with("image/") || item.mime_type.contains("svg") {
continue;
}
report.media_processed += 1;
let prefix = &item.id[..2.min(item.id.len())];
let missing = THUMBNAIL_SIZES
.iter()
.filter(|size| {
let extension = match size.format {
ThumbnailFormat::Webp => "webp",
ThumbnailFormat::Jpeg => "jpg",
};
!data_dir
.join("thumbnails")
.join(prefix)
.join(format!("{}-{}.{}", item.id, size.name, extension))
.is_file()
})
.count();
if missing == 0 {
continue;
}
match generate_all_thumbnails(
&data_dir.join(&item.file_path),
&data_dir.join("thumbnails"),
&item.id,
) {
Ok(_) => {
report.media_repaired += 1;
report.thumbnails_generated += missing;
}
Err(_) => report.media_failed += 1,
}
}
Ok(report)
}
/// Supported image MIME types for import (per media_processing.allium).
const SUPPORTED_IMAGE_TYPES: &[&str] = &[
"image/jpeg",
@@ -46,6 +191,13 @@ const SUPPORTED_IMAGE_TYPES: &[&str] = &[
"image/heif",
];
pub fn is_supported_image_path(path: &Path) -> bool {
path.extension()
.and_then(|extension| extension.to_str())
.map(mime_from_extension)
.is_some_and(|mime_type| SUPPORTED_IMAGE_TYPES.contains(&mime_type))
}
/// Import a media file (image, etc.) into the project.
#[expect(
clippy::too_many_arguments,
@@ -63,6 +215,45 @@ pub fn import_media(
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
) -> EngineResult<Media> {
import_media_at(
conn,
data_dir,
project_id,
source_path,
original_name,
title,
alt,
caption,
author,
language,
tags,
None,
now_unix_ms(),
)
}
/// Import media while preserving a trusted source timestamp. Importers use
/// this path so the canonical date-based location and sidecar metadata match
/// the source system; interactive imports continue to use the current time.
#[expect(
clippy::too_many_arguments,
reason = "arguments are the user-supplied media metadata fields plus source timestamp"
)]
pub(crate) fn import_media_at(
conn: &Connection,
data_dir: &Path,
project_id: &str,
source_path: &Path,
original_name: &str,
title: Option<&str>,
alt: Option<&str>,
caption: Option<&str>,
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
checksum: Option<&str>,
created_at: i64,
) -> EngineResult<Media> {
// Validate file type per spec
let ext = Path::new(original_name)
@@ -70,19 +261,17 @@ pub fn import_media(
.and_then(|e| e.to_str())
.unwrap_or("bin");
let mime_type = mime_from_extension(ext).to_string();
if !SUPPORTED_IMAGE_TYPES.contains(&mime_type.as_str()) {
if !is_supported_image_path(Path::new(original_name)) {
return Err(EngineError::Validation(format!(
"unsupported file type: {mime_type} (file: {original_name})"
)));
}
let id = Uuid::new_v4().to_string();
let now = now_unix_ms();
let filename = format!("{id}.{ext}");
// Compute target directory and copy file
let dir_path = media_dir_path(now);
let dir_path = media_dir_path(created_at);
let rel_file_path = format!("{dir_path}{filename}");
let abs_file_path = data_dir.join(&rel_file_path);
@@ -102,10 +291,6 @@ pub fn import_media(
// Compute sidecar path
let sidecar_rel = media_sidecar_path(&rel_file_path);
// Compute checksum of the copied file
let file_bytes = fs::read(&abs_file_path)?;
let checksum = content_hash(&file_bytes);
// Generate thumbnails (silently ignore errors for non-image files)
let thumbnails_dir = data_dir.join("thumbnails");
let _ = generate_all_thumbnails(&abs_file_path, &thumbnails_dir, &id);
@@ -126,10 +311,10 @@ pub fn import_media(
language: language.map(|s| s.to_string()),
file_path: rel_file_path,
sidecar_path: sidecar_rel.clone(),
checksum: Some(checksum),
checksum: checksum.map(str::to_owned),
tags,
created_at: now,
updated_at: now,
created_at,
updated_at: created_at,
};
// Write sidecar
@@ -143,6 +328,8 @@ pub fn import_media(
// Index in FTS
fts_index_media(conn, &media)?;
emit_media(&media, NotificationAction::Created);
Ok(media)
}
@@ -186,19 +373,31 @@ pub fn update_media(
media.updated_at = now_unix_ms();
qm::update_media(conn, &media)?;
// Rewrite sidecar (need linked_post_ids from post_media table)
let linked = qpm::list_post_media_by_media(conn, media_id).unwrap_or_default();
let linked_post_ids: Vec<String> = linked.iter().map(|pm| pm.post_id.clone()).collect();
let sidecar = MediaSidecar::from_media(&media, &linked_post_ids);
let abs_sidecar = data_dir.join(&media.sidecar_path);
atomic_write_str(&abs_sidecar, &sidecar.to_string())?;
sync_media_sidecar(conn, data_dir, media_id)?;
// Re-index FTS
fts_index_media(conn, &media)?;
emit_media(&media, NotificationAction::Updated);
Ok(media)
}
/// Rewrite a media item's canonical sidecar from its current database state.
pub fn sync_media_sidecar(conn: &Connection, data_dir: &Path, media_id: &str) -> EngineResult<()> {
let media = qm::get_media_by_id(conn, media_id).map_err(|error| match error {
diesel::result::Error::NotFound => EngineError::NotFound(format!("media {media_id}")),
error => EngineError::Db(error),
})?;
let linked_post_ids = qpm::list_post_media_by_media(conn, media_id)?
.into_iter()
.map(|link| link.post_id)
.collect::<Vec<_>>();
let sidecar = MediaSidecar::from_media(&media, &linked_post_ids);
atomic_write_str(&data_dir.join(&media.sidecar_path), &sidecar.to_string())?;
Ok(())
}
/// Replace a media item's binary while preserving its identity, public path,
/// metadata, translations, and post links. Returns `None` when the new file is
/// byte-for-byte identical to the stored binary.
@@ -217,7 +416,7 @@ pub fn replace_media_file(
)));
}
let replacement_checksum = crate::util::file_hash(new_source_path)?;
let replacement_checksum = media_file_hash(new_source_path)?;
if media.checksum.as_deref() == Some(replacement_checksum.as_str()) {
return Ok(None);
}
@@ -278,10 +477,7 @@ pub fn replace_media_file(
let apply_result = (|| -> EngineResult<()> {
conn.begin_savepoint()?;
qm::update_media(conn, &media)?;
let linked = qpm::list_post_media_by_media(conn, media_id).unwrap_or_default();
let linked_post_ids: Vec<String> = linked.iter().map(|item| item.post_id.clone()).collect();
let sidecar = MediaSidecar::from_media(&media, &linked_post_ids);
atomic_write_str(&data_dir.join(&media.sidecar_path), &sidecar.to_string())?;
sync_media_sidecar(conn, data_dir, media_id)?;
fts_index_media(conn, &media)?;
for staged in &staged_paths {
@@ -303,6 +499,7 @@ pub fn replace_media_file(
match apply_result {
Ok(()) => {
fs::remove_file(backup)?;
emit_media(&media, NotificationAction::Updated);
Ok(Some(media))
}
Err(error) => {
@@ -381,9 +578,15 @@ pub fn delete_media(conn: &Connection, data_dir: &Path, media_id: &str) -> Engin
// Delete from media table
qm::delete_media(conn, media_id)?;
emit_media(&media, NotificationAction::Deleted);
Ok(())
}
fn emit_media(media: &Media, action: NotificationAction) {
domain_events::entity_changed(&media.project_id, DomainEntity::Media, &media.id, action);
}
/// Create or update a translation for a media item.
pub fn upsert_media_translation(
conn: &Connection,
@@ -439,6 +642,7 @@ pub fn upsert_media_translation(
// Re-index FTS for parent media
fts_index_media(conn, &media)?;
emit_media(&media, NotificationAction::Updated);
Ok(translation)
}
@@ -478,7 +682,7 @@ pub fn rebuild_media_from_filesystem(
}
/// Per-item progress callback: (current_item, total_items, item_description).
pub type ItemProgressFn = Box<dyn Fn(usize, usize, &str) + Send>;
pub type ItemProgressFn = Box<dyn Fn(usize, usize, &str) -> bool + Send>;
/// Like `rebuild_media_from_filesystem` but with optional per-item progress.
pub fn rebuild_media_from_filesystem_with_progress(
@@ -524,7 +728,9 @@ pub fn rebuild_media_from_filesystem_with_progress(
for (i, path) in canonical_sidecars.iter().enumerate() {
if let Some(ref cb) = on_item {
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("?");
cb(i + 1, total, name);
if !cb(i + 1, total, name) {
return Err(EngineError::Cancelled);
}
}
match rebuild_canonical_media(conn, data_dir, project_id, path) {
Ok(created) => {
@@ -545,7 +751,9 @@ pub fn rebuild_media_from_filesystem_with_progress(
for (i, path) in translation_sidecars.iter().enumerate() {
if let Some(ref cb) = on_item {
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("?");
cb(offset + i + 1, total, name);
if !cb(offset + i + 1, total, name) {
return Err(EngineError::Cancelled);
}
}
match rebuild_translation_sidecar(conn, data_dir, project_id, path) {
Ok(created) => {
@@ -799,6 +1007,12 @@ mod tests {
path
}
fn heic_fixture() -> std::path::PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("testdata")
.join("close.heic")
}
#[test]
fn import_media_creates_artifacts() {
let (db, dir) = setup();
@@ -830,7 +1044,7 @@ mod tests {
assert_eq!(from_db.mime_type, "image/png");
assert_eq!(from_db.width, Some(100));
assert_eq!(from_db.height, Some(80));
assert!(from_db.checksum.is_some());
assert_eq!(from_db.checksum, None);
assert!(from_db.size > 0);
// Verify binary file exists
@@ -863,6 +1077,103 @@ mod tests {
}
}
#[test]
fn import_heic_and_heif_create_dimensions_and_thumbnails() {
let (db, dir) = setup();
for (original_name, mime_type) in
[("photo.heic", "image/heic"), ("photo.heif", "image/heif")]
{
let media = import_media(
db.conn(),
dir.path(),
"p1",
&heic_fixture(),
original_name,
None,
None,
None,
None,
None,
vec![],
)
.unwrap();
assert_eq!(media.mime_type, mime_type);
assert_eq!((media.width, media.height), (Some(27), Some(27)));
for size in THUMBNAIL_SIZES {
let extension = match size.format {
ThumbnailFormat::Webp => "webp",
ThumbnailFormat::Jpeg => "jpg",
};
assert!(
dir.path()
.join("thumbnails")
.join(&media.id[..2])
.join(format!("{}-{}.{}", media.id, size.name, extension))
.is_file(),
"missing {} thumbnail",
size.name
);
}
}
}
#[test]
fn regenerate_missing_thumbnails_reports_undecodable_media_and_continues() {
let (db, dir) = setup();
let good = import_media(
db.conn(),
dir.path(),
"p1",
&create_test_png(dir.path()),
"good.png",
None,
None,
None,
None,
None,
vec![],
)
.unwrap();
let bad_source = dir.path().join("bad.png");
fs::write(&bad_source, "not an image").unwrap();
let bad = import_media(
db.conn(),
dir.path(),
"p1",
&bad_source,
"bad.png",
None,
None,
None,
None,
None,
vec![],
)
.unwrap();
let good_small = dir
.path()
.join("thumbnails")
.join(&good.id[..2])
.join(format!("{}-small.webp", good.id));
fs::remove_file(good_small).unwrap();
let report = regenerate_missing_thumbnails(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(report.media_processed, 2);
assert_eq!(report.media_repaired, 1);
assert_eq!(report.media_failed, 1);
assert_eq!(report.thumbnails_generated, 1);
assert!(
!dir.path()
.join("thumbnails")
.join(&bad.id[..2])
.join(format!("{}-small.webp", bad.id))
.exists()
);
}
#[test]
fn update_media_rewrites_sidecar() {
let (db, dir) = setup();
@@ -913,6 +1224,95 @@ mod tests {
assert_eq!(sc.tags, vec!["updated-tag"]);
}
#[test]
fn rebuild_media_links_replaces_stale_links_and_rolls_back_invalid_sidecars() {
let (db, dir) = setup();
let source = create_test_png(dir.path());
let media = import_media(
db.conn(),
dir.path(),
"p1",
&source,
"photo.png",
None,
None,
None,
None,
None,
vec![],
)
.unwrap();
let first = crate::engine::post::create_post(
db.conn(),
dir.path(),
"p1",
"First",
None,
vec![],
vec![],
None,
None,
None,
)
.unwrap();
let stale = crate::engine::post::create_post(
db.conn(),
dir.path(),
"p1",
"Stale",
None,
vec![],
vec![],
None,
None,
None,
)
.unwrap();
crate::engine::post_media::link_media_to_post(
db.conn(),
dir.path(),
"p1",
&first.id,
&media.id,
0,
)
.unwrap();
crate::engine::post_media::link_media_to_post(
db.conn(),
dir.path(),
"p1",
&stale.id,
&media.id,
1,
)
.unwrap();
atomic_write_str(
&dir.path().join(&media.sidecar_path),
&MediaSidecar::from_media(&media, std::slice::from_ref(&first.id)).to_string(),
)
.unwrap();
let report = rebuild_media_links(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(report.links, 1);
let links = qpm::list_post_media_by_media(db.conn(), &media.id).unwrap();
assert_eq!(links.len(), 1);
assert_eq!(links[0].post_id, first.id);
atomic_write_str(
&dir.path().join(&media.sidecar_path),
&MediaSidecar::from_media(&media, &["missing-post".into()]).to_string(),
)
.unwrap();
assert!(rebuild_media_links(db.conn(), dir.path(), "p1").is_err());
let links = qpm::list_post_media_by_media(db.conn(), &media.id).unwrap();
assert_eq!(
links.len(),
1,
"the failed repair must roll back link deletion"
);
assert_eq!(links[0].post_id, first.id);
}
#[test]
fn replace_media_file_preserves_identity_and_regenerates_artifacts() {
let (db, dir) = setup();
@@ -981,7 +1381,12 @@ mod tests {
)
.unwrap();
let stored = dir.path().join(&media.file_path);
let updated_at = media.updated_at;
let duplicate = dir.path().join("duplicate.png");
fs::copy(&stored, &duplicate).unwrap();
let established = replace_media_file(db.conn(), dir.path(), &media.id, &duplicate)
.unwrap()
.unwrap();
let updated_at = established.updated_at;
assert!(
replace_media_file(db.conn(), dir.path(), &media.id, &stored)
@@ -996,6 +1401,32 @@ mod tests {
);
}
#[test]
fn import_media_at_preserves_caller_supplied_bds2_checksum() {
let (db, dir) = setup();
let source = create_test_png(dir.path());
let checksum = crate::util::media_file_hash(&source).unwrap();
let media = import_media_at(
db.conn(),
dir.path(),
"p1",
&source,
"photo.png",
None,
None,
None,
None,
None,
vec![],
Some(&checksum),
now_unix_ms(),
)
.unwrap();
assert_eq!(media.checksum.as_deref(), Some(checksum.as_str()));
}
#[test]
fn delete_media_removes_everything() {
let (db, dir) = setup();

View File

@@ -6,7 +6,9 @@ use quick_xml::{Reader, Writer, XmlVersion, escape::escape};
use crate::engine::EngineError;
use crate::engine::EngineResult;
use crate::model::Project;
use crate::util::atomic_write_str;
use crate::util::timestamp::unix_ms_to_iso;
/// A navigation menu item per menu.allium.
#[derive(Debug, Clone, PartialEq)]
@@ -31,7 +33,7 @@ impl MenuItemKind {
Self::Home => "home",
Self::Page => "page",
Self::Submenu => "submenu",
Self::CategoryArchive => "category_archive",
Self::CategoryArchive => "category-archive",
}
}
@@ -39,7 +41,7 @@ impl MenuItemKind {
match s {
"home" => Self::Home,
"submenu" => Self::Submenu,
"category_archive" => Self::CategoryArchive,
"category-archive" | "category_archive" => Self::CategoryArchive,
_ => Self::Page,
}
}
@@ -62,32 +64,33 @@ pub fn read_menu(data_dir: &Path) -> EngineResult<Vec<MenuItem>> {
/// Write the navigation menu to meta/menu.opml.
/// Per menu.allium UpdateMenu rule: Home entry is always extracted and prepended.
pub fn write_menu(data_dir: &Path, items: &[MenuItem]) -> EngineResult<()> {
pub fn write_menu(data_dir: &Path, project: &Project, items: &[MenuItem]) -> EngineResult<()> {
let normalized = normalize_menu(items);
let opml = serialize_opml(&normalized)?;
let timestamp = if project.updated_at > 0 {
project.updated_at
} else {
project.created_at
};
let opml = serialize_opml(&project.name, timestamp, &normalized)?;
let path = data_dir.join("meta").join("menu.opml");
atomic_write_str(&path, &opml)?;
Ok(())
}
/// Return the default menu OPML for new projects.
pub fn default_menu_opml() -> String {
pub fn default_menu_opml(project_name: &str, timestamp: i64) -> String {
let items = vec![MenuItem {
kind: MenuItemKind::Home,
label: "Home".to_string(),
slug: None,
children: Vec::new(),
}];
serialize_opml(&items).expect("writing menu XML to memory cannot fail")
serialize_opml(project_name, timestamp, &items).expect("writing menu XML to memory cannot fail")
}
/// Per menu.allium HomeAlwaysPresent: ensure Home is always first.
fn normalize_menu(items: &[MenuItem]) -> Vec<MenuItem> {
let without_home: Vec<MenuItem> = items
.iter()
.filter(|i| i.kind != MenuItemKind::Home)
.cloned()
.collect();
let without_home: Vec<_> = items.iter().filter_map(normalize_non_home).collect();
let mut result = vec![MenuItem {
kind: MenuItemKind::Home,
label: "Home".to_string(),
@@ -98,29 +101,70 @@ fn normalize_menu(items: &[MenuItem]) -> Vec<MenuItem> {
result
}
fn normalize_non_home(item: &MenuItem) -> Option<MenuItem> {
if item.kind == MenuItemKind::Home {
return None;
}
Some(MenuItem {
kind: item.kind.clone(),
label: item.label.clone(),
slug: match item.kind {
MenuItemKind::Page | MenuItemKind::CategoryArchive => item.slug.clone(),
MenuItemKind::Home | MenuItemKind::Submenu => None,
},
children: if item.kind == MenuItemKind::Submenu {
item.children
.iter()
.filter_map(normalize_non_home)
.collect()
} else {
Vec::new()
},
})
}
/// Parse OPML 2.0 XML into menu items.
fn parse_opml(content: &str) -> EngineResult<Vec<MenuItem>> {
let mut reader = Reader::from_str(content);
reader.config_mut().trim_text(true);
let mut items = Vec::new();
let mut parents = Vec::new();
let mut in_body = false;
let mut outlines: Vec<Option<MenuItem>> = Vec::new();
let mut elements: Vec<Vec<u8>> = Vec::new();
loop {
match reader.read_event() {
Ok(Event::Start(event)) if event.name().as_ref() == b"body" => in_body = true,
Ok(Event::End(event)) if event.name().as_ref() == b"body" => in_body = false,
Ok(Event::Start(event)) if in_body && event.name().as_ref() == b"outline" => {
parents.push(parse_outline(&event)?);
Ok(Event::Start(event)) => {
let name = event.name().as_ref().to_vec();
if name == b"outline" {
let collect = collect_outline(&elements, &outlines);
outlines.push(collect.then(|| parse_outline(&event)).transpose()?);
}
elements.push(name);
}
Ok(Event::Empty(event)) if in_body && event.name().as_ref() == b"outline" => {
attach_outline(parse_outline(&event)?, &mut parents, &mut items);
Ok(Event::Empty(event)) if event.name().as_ref() == b"outline" => {
if collect_outline(&elements, &outlines) {
attach_outline(parse_outline(&event)?, &mut outlines, &mut items);
}
}
Ok(Event::End(event)) if event.name().as_ref() == b"outline" => {
let item = parents
Ok(Event::End(event)) => {
let name = event.name().as_ref().to_vec();
let open = elements
.pop()
.ok_or_else(|| EngineError::Parse("unexpected </outline>".to_string()))?;
attach_outline(item, &mut parents, &mut items);
.ok_or_else(|| EngineError::Parse("unexpected closing element".to_string()))?;
if open != name {
return Err(EngineError::Parse("mismatched closing element".to_string()));
}
if name == b"outline" {
let item = outlines
.pop()
.ok_or_else(|| EngineError::Parse("unexpected </outline>".to_string()))?;
if let Some(mut item) = item {
if item.kind != MenuItemKind::Submenu {
item.children.clear();
}
attach_outline(item, &mut outlines, &mut items);
}
}
}
Ok(Event::Eof) => break,
Ok(_) => {}
@@ -128,16 +172,26 @@ fn parse_opml(content: &str) -> EngineResult<Vec<MenuItem>> {
}
}
if !parents.is_empty() {
if !outlines.is_empty() || !elements.is_empty() {
return Err(EngineError::Parse("unclosed <outline>".to_string()));
}
Ok(normalize_menu(&items))
}
fn collect_outline(elements: &[Vec<u8>], outlines: &[Option<MenuItem>]) -> bool {
elements == [b"opml".as_slice(), b"body".as_slice()]
|| (elements.last().is_some_and(|name| name == b"outline")
&& outlines.last().is_some_and(Option::is_some))
}
fn parse_outline(event: &BytesStart<'_>) -> EngineResult<MenuItem> {
let mut label = "Untitled".to_string();
let mut kind = MenuItemKind::Page;
let mut slug = None;
let mut label = String::new();
let mut kind_value = None;
let mut legacy_kind = None;
let mut page_slug = None;
let mut category_name = None;
let mut legacy_slug = None;
let mut html_url = None;
for attribute in event.attributes() {
let attribute = attribute.map_err(|error| EngineError::Parse(error.to_string()))?;
@@ -147,12 +201,28 @@ fn parse_outline(event: &BytesStart<'_>) -> EngineResult<MenuItem> {
.into_owned();
match attribute.key.as_ref() {
b"text" => label = value,
b"type" => kind = MenuItemKind::from_str(&value),
b"htmlUrl" => slug = Some(value),
b"type" => kind_value = Some(value),
b"kind" => legacy_kind = Some(value),
b"pageSlug" => page_slug = Some(value),
b"categoryName" => category_name = Some(value),
b"slug" => legacy_slug = Some(value),
b"htmlUrl" => html_url = Some(value),
_ => {}
}
}
let kind = kind_value
.or(legacy_kind)
.as_deref()
.map(MenuItemKind::from_str)
.unwrap_or(MenuItemKind::Page);
let slug = match kind {
MenuItemKind::Home | MenuItemKind::Submenu => None,
MenuItemKind::Page => page_slug.or(legacy_slug).or(html_url),
MenuItemKind::CategoryArchive => category_name.or(legacy_slug).or(html_url),
}
.filter(|value| !value.is_empty());
Ok(MenuItem {
kind,
label,
@@ -161,8 +231,8 @@ fn parse_outline(event: &BytesStart<'_>) -> EngineResult<MenuItem> {
})
}
fn attach_outline(item: MenuItem, parents: &mut [MenuItem], items: &mut Vec<MenuItem>) {
if let Some(parent) = parents.last_mut() {
fn attach_outline(item: MenuItem, parents: &mut [Option<MenuItem>], items: &mut Vec<MenuItem>) {
if let Some(Some(parent)) = parents.last_mut() {
parent.children.push(item);
} else {
items.push(item);
@@ -170,8 +240,9 @@ fn attach_outline(item: MenuItem, parents: &mut [MenuItem], items: &mut Vec<Menu
}
/// Serialize menu items to OPML 2.0 format.
fn serialize_opml(items: &[MenuItem]) -> EngineResult<String> {
fn serialize_opml(project_name: &str, timestamp: i64, items: &[MenuItem]) -> EngineResult<String> {
let mut writer = Writer::new_with_indent(Vec::new(), b' ', 2);
writer.config_mut().add_space_before_slash_in_empty_elements = true;
writer
.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))
.map_err(|error| EngineError::Parse(error.to_string()))?;
@@ -183,14 +254,12 @@ fn serialize_opml(items: &[MenuItem]) -> EngineResult<String> {
writer
.write_event(Event::Start(BytesStart::new("head")))
.map_err(|error| EngineError::Parse(error.to_string()))?;
writer
.write_event(Event::Start(BytesStart::new("title")))
write_text_element(&mut writer, "title", project_name)
.map_err(|error| EngineError::Parse(error.to_string()))?;
writer
.write_event(Event::Text(BytesText::new("Blog Menu")))
let timestamp = unix_ms_to_iso(timestamp);
write_text_element(&mut writer, "dateCreated", &timestamp)
.map_err(|error| EngineError::Parse(error.to_string()))?;
writer
.write_event(Event::End(BytesEnd::new("title")))
write_text_element(&mut writer, "dateModified", &timestamp)
.map_err(|error| EngineError::Parse(error.to_string()))?;
writer
.write_event(Event::End(BytesEnd::new("head")))
@@ -207,17 +276,41 @@ fn serialize_opml(items: &[MenuItem]) -> EngineResult<String> {
writer
.write_event(Event::End(BytesEnd::new("opml")))
.map_err(|error| EngineError::Parse(error.to_string()))?;
String::from_utf8(writer.into_inner()).map_err(|error| EngineError::Parse(error.to_string()))
let mut rendered = String::from_utf8(writer.into_inner())
.map_err(|error| EngineError::Parse(error.to_string()))?;
rendered.push('\n');
Ok(rendered)
}
fn write_text_element(
writer: &mut Writer<Vec<u8>>,
name: &str,
value: &str,
) -> quick_xml::Result<()> {
writer.write_event(Event::Start(BytesStart::new(name)))?;
writer.write_event(Event::Text(BytesText::new(value)))?;
writer.write_event(Event::End(BytesEnd::new(name)))?;
Ok(())
}
fn write_outline(writer: &mut Writer<Vec<u8>>, item: &MenuItem) -> quick_xml::Result<()> {
let label = escape(&item.label);
let slug = item.slug.as_deref().map(escape);
let mut outline = BytesStart::new("outline");
outline.push_attribute(("text", label.as_ref()));
outline.push_attribute(("type", item.kind.as_str()));
if let Some(slug) = &slug {
outline.push_attribute(("htmlUrl", slug.as_ref()));
match item.kind {
MenuItemKind::Home => outline.push_attribute(("pageSlug", "home")),
MenuItemKind::Page => {
if let Some(slug) = item.slug.as_deref().map(escape) {
outline.push_attribute(("pageSlug", slug.as_ref()));
}
}
MenuItemKind::CategoryArchive => {
if let Some(slug) = item.slug.as_deref().map(escape) {
outline.push_attribute(("categoryName", slug.as_ref()));
}
}
MenuItemKind::Submenu => {}
}
if item.children.is_empty() {
writer.write_event(Event::Empty(outline))?;
@@ -237,7 +330,7 @@ mod tests {
#[test]
fn default_opml_has_home() {
let opml = default_menu_opml();
let opml = default_menu_opml("Test Blog", 1_711_833_600_000);
assert!(opml.contains("type=\"home\""));
assert!(opml.contains("text=\"Home\""));
}
@@ -269,7 +362,7 @@ mod tests {
}],
},
];
let opml = serialize_opml(&items).unwrap();
let opml = serialize_opml("Test Blog", 1_711_833_600_000, &items).unwrap();
let parsed = parse_opml(&opml).unwrap();
assert_eq!(parsed.len(), 3);
assert_eq!(parsed[0].kind, MenuItemKind::Home);
@@ -312,7 +405,21 @@ mod tests {
slug: Some("/blog".into()),
children: Vec::new(),
}];
write_menu(dir.path(), &items).unwrap();
write_menu(
dir.path(),
&Project {
id: "project-1".into(),
name: "Test Blog".into(),
slug: "test-blog".into(),
description: None,
data_path: None,
is_active: true,
created_at: 1_711_833_600_000,
updated_at: 1_711_833_600_000,
},
&items,
)
.unwrap();
let read = read_menu(dir.path()).unwrap();
// Home is always prepended
assert_eq!(read.len(), 2);
@@ -330,6 +437,94 @@ mod tests {
assert_eq!(parsed[1].slug.as_deref(), Some("/about"));
}
#[test]
fn canonical_bds2_attributes_round_trip_without_legacy_output() {
let parsed = parse_opml(
"<opml version='2.0'><body><outline text='Home' type='home' pageSlug='home'/><outline text='Sections' type='submenu'><outline text='About' type='page' pageSlug='about'/><outline text='Notes' type='category-archive' categoryName='notes'/></outline></body></opml>",
)
.unwrap();
assert_eq!(parsed.len(), 2);
assert_eq!(parsed[1].children[0].slug.as_deref(), Some("about"));
assert_eq!(parsed[1].children[1].kind, MenuItemKind::CategoryArchive);
assert_eq!(parsed[1].children[1].slug.as_deref(), Some("notes"));
let serialized = serialize_opml("Test Blog", 1_711_833_600_000, &parsed).unwrap();
assert!(serialized.contains("type=\"home\" pageSlug=\"home\""));
assert!(serialized.contains("type=\"page\" pageSlug=\"about\""));
assert!(serialized.contains("type=\"category-archive\" categoryName=\"notes\""));
assert!(!serialized.contains("htmlUrl"));
assert!(!serialized.contains("category_archive"));
}
#[test]
fn serializer_matches_bds2_head_spacing_and_trailing_newline() {
let items = vec![MenuItem {
kind: MenuItemKind::Home,
label: "Home".into(),
slug: None,
children: Vec::new(),
}];
let serialized = serialize_opml("Test Blog", 1_711_833_600_000, &items).unwrap();
assert_eq!(
serialized,
concat!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
"<opml version=\"2.0\">\n",
" <head>\n",
" <title>Test Blog</title>\n",
" <dateCreated>2024-03-30T21:20:00.000Z</dateCreated>\n",
" <dateModified>2024-03-30T21:20:00.000Z</dateModified>\n",
" </head>\n",
" <body>\n",
" <outline text=\"Home\" type=\"home\" pageSlug=\"home\" />\n",
" </body>\n",
"</opml>\n",
)
);
}
#[test]
fn parser_ignores_foreign_outlines_and_drops_children_of_non_submenus() {
let parsed = parse_opml(
"<opml><head><outline text='Head'/></head><body><section><outline text='Foreign'/></section><outline text='Page' type='page' pageSlug='page'><outline text='Dropped' type='page' pageSlug='dropped'/></outline><outline text='Kept' type='submenu'><outline text='Child' type='page' pageSlug='child'/></outline></body></opml>",
)
.unwrap();
assert_eq!(parsed.len(), 3);
assert_eq!(parsed[1].label, "Page");
assert!(parsed[1].children.is_empty());
assert_eq!(parsed[2].label, "Kept");
assert_eq!(parsed[2].children[0].label, "Child");
}
#[test]
fn normalization_removes_home_entries_at_every_depth() {
let normalized = normalize_menu(&[
MenuItem {
kind: MenuItemKind::Home,
label: "Duplicate".into(),
slug: None,
children: Vec::new(),
},
MenuItem {
kind: MenuItemKind::Submenu,
label: "Sections".into(),
slug: None,
children: vec![MenuItem {
kind: MenuItemKind::Home,
label: "Nested Home".into(),
slug: None,
children: Vec::new(),
}],
},
]);
assert_eq!(normalized[0].kind, MenuItemKind::Home);
assert_eq!(normalized[0].label, "Home");
assert!(normalized[1].children.is_empty());
}
#[test]
fn read_menu_rejects_malformed_xml() {
let dir = tempfile::TempDir::new().unwrap();

View File

@@ -1,13 +1,99 @@
use std::collections::HashMap;
use std::collections::{BTreeMap, HashMap};
use std::fs;
use std::path::Path;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use crate::db::DbConnection as Connection;
use crate::engine::EngineResult;
use crate::model::PublishingPreferences;
use crate::engine::{EngineError, EngineResult};
use crate::model::metadata::{CategorySettings, ProjectMetadata, TagEntry};
use crate::model::{DomainEntity, NotificationAction, Project, PublishingPreferences};
use crate::util::atomic_write_str;
const PROJECT_SETTING_SUFFIX: &str = "project";
const CATEGORIES_SETTING_SUFFIX: &str = "categories";
const CATEGORY_META_SETTING_SUFFIX: &str = "category_meta";
const PUBLISHING_SETTING_SUFFIX: &str = "publishing";
#[derive(Debug, Serialize, Deserialize)]
struct CategoriesSnapshot {
categories: Vec<String>,
}
#[derive(Debug, Serialize, Deserialize)]
struct CategoryMetaSnapshot {
categories: HashMap<String, CategorySettings>,
}
fn metadata_setting_key(project_id: &str, suffix: &str) -> String {
format!("project:{project_id}:{suffix}")
}
fn persist_snapshot<T: Serialize>(
conn: &Connection,
project_id: &str,
suffix: &str,
value: &T,
) -> EngineResult<()> {
crate::db::queries::setting::set_setting_value(
conn,
&metadata_setting_key(project_id, suffix),
&serde_json::to_string(value)?,
crate::util::now_unix_ms(),
)?;
Ok(())
}
fn load_snapshot<T: DeserializeOwned>(
conn: &Connection,
project_id: &str,
suffix: &str,
) -> EngineResult<Option<T>> {
match crate::db::queries::setting::get_setting_by_key(
conn,
&metadata_setting_key(project_id, suffix),
) {
Ok(setting) => Ok(Some(serde_json::from_str(&setting.value)?)),
Err(diesel::result::Error::NotFound) => Ok(None),
Err(error) => Err(EngineError::Db(error)),
}
}
pub fn read_project_metadata_snapshot(
conn: &Connection,
project_id: &str,
) -> EngineResult<Option<ProjectMetadata>> {
load_snapshot(conn, project_id, PROJECT_SETTING_SUFFIX)
}
pub fn read_categories_snapshot(
conn: &Connection,
project_id: &str,
) -> EngineResult<Option<Vec<String>>> {
Ok(
load_snapshot::<CategoriesSnapshot>(conn, project_id, CATEGORIES_SETTING_SUFFIX)?
.map(|snapshot| snapshot.categories),
)
}
pub fn read_category_meta_snapshot(
conn: &Connection,
project_id: &str,
) -> EngineResult<Option<HashMap<String, CategorySettings>>> {
Ok(
load_snapshot::<CategoryMetaSnapshot>(conn, project_id, CATEGORY_META_SETTING_SUFFIX)?
.map(|snapshot| snapshot.categories),
)
}
pub fn read_publishing_snapshot(
conn: &Connection,
project_id: &str,
) -> EngineResult<Option<PublishingPreferences>> {
load_snapshot(conn, project_id, PUBLISHING_SETTING_SUFFIX)
}
// ── project.json ────────────────────────────────────────────────────
/// Read and parse meta/project.json.
@@ -26,25 +112,155 @@ pub fn write_project_json(data_dir: &Path, meta: &ProjectMetadata) -> EngineResu
Ok(())
}
/// Copy the project fields persisted in both representations from project.json
/// into the machine-local project row.
pub fn sync_project_from_file(
/// Persist the shared project row and its filesystem metadata, then publish
/// the normal project-updated domain event.
pub fn update_project_metadata(
conn: &Connection,
data_dir: &Path,
project: &Project,
metadata: &ProjectMetadata,
) -> EngineResult<ProjectMetadata> {
if project.id.is_empty() {
return Err(crate::engine::EngineError::Validation(
"project id is required".to_string(),
));
}
metadata
.validate()
.map_err(crate::engine::EngineError::Validation)?;
let mut persisted_project = project.clone();
persisted_project.name = metadata.name.clone();
persisted_project.description = metadata.description.clone();
persisted_project.updated_at = crate::util::now_unix_ms();
let category_metadata = read_category_meta_json(data_dir)?;
persist_snapshot(conn, &project.id, PROJECT_SETTING_SUFFIX, metadata)?;
write_project_json(data_dir, metadata)?;
write_category_meta_json(data_dir, &category_metadata)?;
crate::db::queries::project::update_project(conn, &persisted_project)?;
crate::engine::domain_events::entity_changed(
&project.id,
DomainEntity::Project,
&project.id,
NotificationAction::Updated,
);
Ok(metadata.clone())
}
/// Replace all database metadata snapshots with the portable filesystem state.
pub fn sync_metadata_from_filesystem(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
let metadata = read_project_json(data_dir)?;
metadata
.validate()
.map_err(crate::engine::EngineError::Validation)?;
let mut project = crate::db::queries::project::get_project_by_id(conn, project_id)?;
project.name = metadata.name;
project.description = metadata.description;
project.updated_at = crate::util::now_unix_ms();
crate::db::queries::project::update_project(conn, &project)?;
let project = read_project_json(data_dir)?;
project.validate().map_err(EngineError::Validation)?;
let categories = read_categories_json(data_dir)?;
let category_meta = read_category_meta_json(data_dir)?;
let publishing = read_publishing_json(data_dir)?;
conn.begin_savepoint()?;
let result = (|| {
let mut project_row = crate::db::queries::project::get_project_by_id(conn, project_id)?;
project_row.name = project.name.clone();
project_row.description = project.description.clone();
project_row.updated_at = crate::util::now_unix_ms();
crate::db::queries::project::update_project(conn, &project_row)?;
persist_snapshot(conn, project_id, PROJECT_SETTING_SUFFIX, &project)?;
persist_snapshot(
conn,
project_id,
CATEGORIES_SETTING_SUFFIX,
&CategoriesSnapshot { categories },
)?;
persist_snapshot(
conn,
project_id,
CATEGORY_META_SETTING_SUFFIX,
&CategoryMetaSnapshot {
categories: category_meta,
},
)?;
persist_snapshot(conn, project_id, PUBLISHING_SETTING_SUFFIX, &publishing)
})();
match result {
Ok(()) => conn.release_savepoint().map_err(Into::into),
Err(error) => {
let _ = conn.rollback_savepoint();
Err(error)
}
}
}
/// Seed database snapshots for pre-snapshot projects without overwriting
/// deliberate database/file divergence on later activations.
pub fn initialize_metadata_snapshots(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
if read_project_metadata_snapshot(conn, project_id)?.is_none() {
let value = read_project_json(data_dir)?;
persist_snapshot(conn, project_id, PROJECT_SETTING_SUFFIX, &value)?;
}
if read_categories_snapshot(conn, project_id)?.is_none() {
let categories = read_categories_json(data_dir)?;
persist_snapshot(
conn,
project_id,
CATEGORIES_SETTING_SUFFIX,
&CategoriesSnapshot { categories },
)?;
}
if read_category_meta_snapshot(conn, project_id)?.is_none() {
let categories = read_category_meta_json(data_dir)?;
persist_snapshot(
conn,
project_id,
CATEGORY_META_SETTING_SUFFIX,
&CategoryMetaSnapshot { categories },
)?;
}
if read_publishing_snapshot(conn, project_id)?.is_none() {
let value = read_publishing_json(data_dir)?;
persist_snapshot(conn, project_id, PUBLISHING_SETTING_SUFFIX, &value)?;
}
Ok(())
}
pub fn flush_metadata_to_filesystem(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
let project = read_project_metadata_snapshot(conn, project_id)?.ok_or_else(|| {
EngineError::NotFound(format!(
"project metadata snapshot for project {project_id}"
))
})?;
let categories = read_categories_snapshot(conn, project_id)?.ok_or_else(|| {
EngineError::NotFound(format!(
"categories metadata snapshot for project {project_id}"
))
})?;
let category_meta = read_category_meta_snapshot(conn, project_id)?.ok_or_else(|| {
EngineError::NotFound(format!(
"category metadata snapshot for project {project_id}"
))
})?;
let publishing = read_publishing_snapshot(conn, project_id)?.ok_or_else(|| {
EngineError::NotFound(format!(
"publishing metadata snapshot for project {project_id}"
))
})?;
write_project_json(data_dir, &project)?;
write_categories_json(data_dir, &categories)?;
write_category_meta_json(data_dir, &category_meta)?;
write_publishing_json(data_dir, &publishing)
}
// ── categories.json ─────────────────────────────────────────────────
/// Read meta/categories.json as a sorted array of strings.
@@ -58,13 +274,32 @@ pub fn read_categories_json(data_dir: &Path) -> EngineResult<Vec<String>> {
/// Sort categories, then atomic write to meta/categories.json.
pub fn write_categories_json(data_dir: &Path, categories: &[String]) -> EngineResult<()> {
let mut sorted = categories.to_vec();
sorted.sort_by_key(|a| a.to_lowercase());
sorted.sort();
let path = data_dir.join("meta").join("categories.json");
let json = serde_json::to_string_pretty(&sorted)?;
atomic_write_str(&path, &json)?;
Ok(())
}
pub fn set_categories(
conn: &Connection,
data_dir: &Path,
project_id: &str,
categories: &[String],
) -> EngineResult<()> {
let mut sorted = categories.to_vec();
sorted.sort();
persist_snapshot(
conn,
project_id,
CATEGORIES_SETTING_SUFFIX,
&CategoriesSnapshot {
categories: sorted.clone(),
},
)?;
write_categories_json(data_dir, &sorted)
}
// ── category-meta.json ──────────────────────────────────────────────
/// Read meta/category-meta.json.
@@ -81,11 +316,68 @@ pub fn write_category_meta_json(
meta: &HashMap<String, CategorySettings>,
) -> EngineResult<()> {
let path = data_dir.join("meta").join("category-meta.json");
let json = serde_json::to_string_pretty(meta)?;
let sorted = meta.iter().collect::<BTreeMap<_, _>>();
let json = serde_json::to_string_pretty(&sorted)?;
atomic_write_str(&path, &json)?;
Ok(())
}
pub fn set_category_meta(
conn: &Connection,
data_dir: &Path,
project_id: &str,
meta: &HashMap<String, CategorySettings>,
) -> EngineResult<()> {
persist_snapshot(
conn,
project_id,
CATEGORY_META_SETTING_SUFFIX,
&CategoryMetaSnapshot {
categories: meta.clone(),
},
)?;
write_category_meta_json(data_dir, meta)
}
pub fn set_categories_and_meta(
conn: &Connection,
data_dir: &Path,
project_id: &str,
categories: &[String],
meta: &HashMap<String, CategorySettings>,
) -> EngineResult<()> {
let mut sorted = categories.to_vec();
sorted.sort();
conn.begin_savepoint()?;
let result = (|| {
persist_snapshot(
conn,
project_id,
CATEGORIES_SETTING_SUFFIX,
&CategoriesSnapshot {
categories: sorted.clone(),
},
)?;
persist_snapshot(
conn,
project_id,
CATEGORY_META_SETTING_SUFFIX,
&CategoryMetaSnapshot {
categories: meta.clone(),
},
)
})();
match result {
Ok(()) => conn.release_savepoint()?,
Err(error) => {
let _ = conn.rollback_savepoint();
return Err(error);
}
}
write_categories_json(data_dir, &sorted)?;
write_category_meta_json(data_dir, meta)
}
// ── publishing.json ─────────────────────────────────────────────────
/// Read meta/publishing.json.
@@ -104,6 +396,16 @@ pub fn write_publishing_json(data_dir: &Path, prefs: &PublishingPreferences) ->
Ok(())
}
pub fn set_publishing_preferences(
conn: &Connection,
data_dir: &Path,
project_id: &str,
prefs: &PublishingPreferences,
) -> EngineResult<()> {
persist_snapshot(conn, project_id, PUBLISHING_SETTING_SUFFIX, prefs)?;
write_publishing_json(data_dir, prefs)
}
// ── tags.json ───────────────────────────────────────────────────────
/// Read meta/tags.json.
@@ -127,11 +429,15 @@ pub fn write_tags_json(data_dir: &Path, tags: &[TagEntry]) -> EngineResult<()> {
// ── category helpers ────────────────────────────────────────────────
/// Add a category to categories.json and initialize it in category-meta.json.
pub fn add_category(data_dir: &Path, category: &str) -> EngineResult<()> {
pub fn add_category(
conn: &Connection,
data_dir: &Path,
project_id: &str,
category: &str,
) -> EngineResult<()> {
let mut cats = read_categories_json(data_dir)?;
if !cats.iter().any(|c| c.eq_ignore_ascii_case(category)) {
cats.push(category.to_string());
write_categories_json(data_dir, &cats)?;
}
let mut meta = read_category_meta_json(data_dir)?;
@@ -139,29 +445,30 @@ pub fn add_category(data_dir: &Path, category: &str) -> EngineResult<()> {
meta.insert(
category.to_string(),
CategorySettings {
title: None,
titles: BTreeMap::new(),
render_in_lists: true,
show_title: true,
post_template_slug: None,
list_template_slug: None,
},
);
write_category_meta_json(data_dir, &meta)?;
}
Ok(())
set_categories_and_meta(conn, data_dir, project_id, &cats, &meta)
}
/// Remove a category from both categories.json and category-meta.json.
pub fn remove_category(data_dir: &Path, category: &str) -> EngineResult<()> {
pub fn remove_category(
conn: &Connection,
data_dir: &Path,
project_id: &str,
category: &str,
) -> EngineResult<()> {
let mut cats = read_categories_json(data_dir)?;
cats.retain(|c| !c.eq_ignore_ascii_case(category));
write_categories_json(data_dir, &cats)?;
let mut meta = read_category_meta_json(data_dir)?;
meta.remove(category);
write_category_meta_json(data_dir, &meta)?;
Ok(())
set_categories_and_meta(conn, data_dir, project_id, &cats, &meta)
}
// ── startup sync ────────────────────────────────────────────────────
@@ -209,7 +516,7 @@ pub fn startup_sync(data_dir: &Path) -> EngineResult<()> {
// Ensure publishing.json exists
if !meta_dir.join("publishing.json").exists() {
atomic_write_str(&meta_dir.join("publishing.json"), "{}")?;
write_publishing_json(data_dir, &PublishingPreferences::default())?;
}
// Ensure tags.json exists
@@ -219,7 +526,9 @@ pub fn startup_sync(data_dir: &Path) -> EngineResult<()> {
// Ensure menu.opml exists
if !meta_dir.join("menu.opml").exists() {
let opml = crate::engine::menu::default_menu_opml();
let project = read_project_json(data_dir)?;
let opml =
crate::engine::menu::default_menu_opml(&project.name, crate::util::now_unix_ms());
atomic_write_str(&meta_dir.join("menu.opml"), &opml)?;
}
@@ -263,6 +572,45 @@ mod tests {
assert_eq!(read.description.as_deref(), Some("A blog"));
}
#[test]
fn project_json_matches_bds2_canonical_key_order() {
let dir = setup();
let meta = ProjectMetadata {
name: "Test".into(),
description: Some("A blog".into()),
public_url: Some("https://example.com".into()),
main_language: Some("en".into()),
default_author: Some("Writer".into()),
max_posts_per_page: 25,
image_import_concurrency: 3,
blogmark_category: Some("links".into()),
pico_theme: Some("amber".into()),
semantic_similarity_enabled: true,
blog_languages: vec!["en".into(), "de".into()],
};
write_project_json(dir.path(), &meta).unwrap();
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/project.json")).unwrap(),
r#"{
"blogLanguages": [
"en",
"de"
],
"blogmarkCategory": "links",
"defaultAuthor": "Writer",
"description": "A blog",
"imageImportConcurrency": 3,
"mainLanguage": "en",
"maxPostsPerPage": 25,
"name": "Test",
"picoTheme": "amber",
"publicUrl": "https://example.com",
"semanticSimilarityEnabled": true
}"#
);
}
// ── categories.json ─────────────────────────────────────────────
#[test]
@@ -274,6 +622,17 @@ mod tests {
assert_eq!(read, vec!["article", "aside", "picture"]);
}
#[test]
fn categories_json_uses_bds2_case_sensitive_sort() {
let dir = setup();
write_categories_json(dir.path(), &["alpha".into(), "Zebra".into(), "Beta".into()])
.unwrap();
assert_eq!(
read_categories_json(dir.path()).unwrap(),
vec!["Beta", "Zebra", "alpha"]
);
}
// ── category-meta.json ──────────────────────────────────────────
#[test]
@@ -283,6 +642,8 @@ mod tests {
meta.insert(
"article".to_string(),
CategorySettings {
title: None,
titles: BTreeMap::new(),
render_in_lists: true,
show_title: true,
post_template_slug: None,
@@ -295,6 +656,82 @@ mod tests {
assert!(read["article"].render_in_lists);
}
#[test]
fn category_meta_json_preserves_bds2_title() {
let dir = setup();
let mut meta = HashMap::new();
meta.insert(
"news".to_string(),
CategorySettings {
title: Some("News Archive".to_string()),
titles: BTreeMap::from([("de".into(), "Nachrichten".into())]),
render_in_lists: false,
show_title: true,
post_template_slug: Some("article".to_string()),
list_template_slug: Some("listing".to_string()),
},
);
write_category_meta_json(dir.path(), &meta).unwrap();
let content = std::fs::read_to_string(dir.path().join("meta/category-meta.json")).unwrap();
let json: serde_json::Value = serde_json::from_str(&content).unwrap();
assert_eq!(json["news"]["title"], "News Archive");
assert_eq!(json["news"]["titles"]["de"], "Nachrichten");
assert_eq!(json["news"]["renderInLists"], false);
assert_eq!(json["news"]["showTitle"], true);
let read = read_category_meta_json(dir.path()).unwrap();
assert_eq!(read["news"].title.as_deref(), Some("News Archive"));
assert_eq!(read["news"].title_for("de", "en"), Some("Nachrichten"));
}
#[test]
fn category_meta_json_matches_bds2_deterministic_key_order() {
let dir = setup();
let mut meta = HashMap::new();
meta.insert(
"zebra".to_string(),
CategorySettings {
title: Some("Zebra".into()),
titles: BTreeMap::new(),
render_in_lists: true,
show_title: false,
post_template_slug: Some("post".into()),
list_template_slug: Some("list".into()),
},
);
meta.insert(
"alpha".to_string(),
CategorySettings {
title: None,
titles: BTreeMap::new(),
render_in_lists: false,
show_title: true,
post_template_slug: None,
list_template_slug: None,
},
);
write_category_meta_json(dir.path(), &meta).unwrap();
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/category-meta.json")).unwrap(),
r#"{
"alpha": {
"renderInLists": false,
"showTitle": true
},
"zebra": {
"listTemplateSlug": "list",
"postTemplateSlug": "post",
"renderInLists": true,
"showTitle": false,
"title": "Zebra"
}
}"#
);
}
// ── publishing.json ─────────────────────────────────────────────
#[test]
@@ -312,6 +749,77 @@ mod tests {
assert_eq!(read.ssh_mode, SshMode::Rsync);
}
#[test]
fn publishing_json_matches_bds2_canonical_key_order() {
let dir = setup();
let prefs = PublishingPreferences {
ssh_host: Some("example.com".into()),
ssh_user: Some("deploy".into()),
ssh_remote_path: Some("/var/www".into()),
ssh_mode: SshMode::Rsync,
};
write_publishing_json(dir.path(), &prefs).unwrap();
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/publishing.json")).unwrap(),
r#"{
"sshHost": "example.com",
"sshMode": "rsync",
"sshRemotePath": "/var/www",
"sshUser": "deploy"
}"#
);
}
#[test]
fn canonical_empty_metadata_collections_and_default_publishing_are_compact() {
let dir = setup();
write_project_json(
dir.path(),
&ProjectMetadata {
name: "Minimal".into(),
description: None,
public_url: None,
main_language: None,
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
blog_languages: Vec::new(),
},
)
.unwrap();
write_category_meta_json(dir.path(), &HashMap::new()).unwrap();
write_tags_json(dir.path(), &[]).unwrap();
write_publishing_json(dir.path(), &PublishingPreferences::default()).unwrap();
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/project.json")).unwrap(),
concat!(
"{\n",
" \"blogLanguages\": [],\n",
" \"imageImportConcurrency\": 4,\n",
" \"maxPostsPerPage\": 50,\n",
" \"name\": \"Minimal\",\n",
" \"semanticSimilarityEnabled\": false\n",
"}"
)
);
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/category-meta.json")).unwrap(),
"{}"
);
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/tags.json")).unwrap(),
"[]"
);
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/publishing.json")).unwrap(),
"{\n \"sshMode\": \"scp\"\n}"
);
}
// ── tags.json ───────────────────────────────────────────────────
#[test]
@@ -335,16 +843,41 @@ mod tests {
assert_eq!(read[1].name, "Zebra");
}
#[test]
fn tags_json_matches_bds2_entry_order_and_omits_blank_optional_values() {
let dir = setup();
write_tags_json(
dir.path(),
&[TagEntry {
name: "rust".into(),
color: Some("".into()),
post_template_slug: Some("article".into()),
}],
)
.unwrap();
assert_eq!(
std::fs::read_to_string(dir.path().join("meta/tags.json")).unwrap(),
r#"[
{
"name": "rust",
"postTemplateSlug": "article"
}
]"#
);
}
// ── add / remove category ───────────────────────────────────────
#[test]
fn add_category_creates_entries() {
let dir = setup();
let db = crate::db::Database::open_in_memory().unwrap();
db.migrate().unwrap();
// Seed files
write_categories_json(dir.path(), &["article".into()]).unwrap();
write_category_meta_json(dir.path(), &HashMap::new()).unwrap();
add_category(dir.path(), "page").unwrap();
add_category(db.conn(), dir.path(), "p1", "page").unwrap();
let cats = read_categories_json(dir.path()).unwrap();
assert!(cats.contains(&"page".to_string()));
@@ -356,10 +889,12 @@ mod tests {
#[test]
fn add_category_idempotent() {
let dir = setup();
let db = crate::db::Database::open_in_memory().unwrap();
db.migrate().unwrap();
write_categories_json(dir.path(), &["article".into()]).unwrap();
write_category_meta_json(dir.path(), &HashMap::new()).unwrap();
add_category(dir.path(), "article").unwrap();
add_category(db.conn(), dir.path(), "p1", "article").unwrap();
let cats = read_categories_json(dir.path()).unwrap();
assert_eq!(cats.iter().filter(|c| *c == "article").count(), 1);
}
@@ -367,11 +902,15 @@ mod tests {
#[test]
fn remove_category_deletes_entries() {
let dir = setup();
let db = crate::db::Database::open_in_memory().unwrap();
db.migrate().unwrap();
write_categories_json(dir.path(), &["article".into(), "page".into()]).unwrap();
let mut meta = HashMap::new();
meta.insert(
"article".to_string(),
CategorySettings {
title: None,
titles: BTreeMap::new(),
render_in_lists: true,
show_title: true,
post_template_slug: None,
@@ -380,7 +919,7 @@ mod tests {
);
write_category_meta_json(dir.path(), &meta).unwrap();
remove_category(dir.path(), "article").unwrap();
remove_category(db.conn(), dir.path(), "p1", "article").unwrap();
let cats = read_categories_json(dir.path()).unwrap();
assert!(!cats.contains(&"article".to_string()));
@@ -389,4 +928,26 @@ mod tests {
let meta = read_category_meta_json(dir.path()).unwrap();
assert!(!meta.contains_key("article"));
}
#[test]
fn snapshot_initialization_does_not_hide_later_filesystem_drift() {
let dir = setup();
let db = crate::db::Database::open_in_memory().unwrap();
db.migrate().unwrap();
crate::db::queries::project::insert_project(
db.conn(),
&crate::db::queries::project::make_test_project("p1", "blog"),
)
.unwrap();
startup_sync(dir.path()).unwrap();
initialize_metadata_snapshots(db.conn(), dir.path(), "p1").unwrap();
write_categories_json(dir.path(), &["filesystem-only".into()]).unwrap();
initialize_metadata_snapshots(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(
read_categories_snapshot(db.conn(), "p1").unwrap().unwrap(),
vec!["article", "aside", "page", "picture"]
);
}
}

View File

@@ -1,10 +1,11 @@
use std::collections::HashSet;
use std::collections::{BTreeMap, HashSet};
use std::fs;
use std::path::Path;
use crate::db::DbConnection as Connection;
use walkdir::WalkDir;
use crate::db::queries::embedding as qe;
use crate::db::queries::media as qm;
use crate::db::queries::media_translation as qmt;
use crate::db::queries::post as qp;
@@ -16,8 +17,7 @@ use crate::engine::{EngineError, EngineResult};
use crate::model::{Media, MediaTranslation, Post, PostTranslation, Script, Template};
use crate::util::frontmatter::{
ScriptFrontmatter, TemplateFrontmatter, read_post_file, read_script_file, read_template_file,
read_translation_file, write_post_file, write_script_file, write_template_file,
write_translation_file,
read_translation_file, write_script_file, write_template_file, write_translation_file,
};
use crate::util::sidecar::{
MediaSidecar, MediaTranslationSidecar, read_sidecar, read_translation_sidecar,
@@ -70,17 +70,50 @@ pub fn compute_metadata_diff(
data_dir: &Path,
project_id: &str,
) -> EngineResult<DiffReport> {
compute_metadata_diff_with_progress(conn, data_dir, project_id, |_, _| true)
}
pub fn compute_metadata_diff_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
mut on_progress: impl FnMut(usize, usize) -> bool,
) -> EngineResult<DiffReport> {
const PHASES: usize = 8;
let mut phase = 0;
let mut next_phase = || -> EngineResult<()> {
if !on_progress(phase, PHASES) {
return Err(crate::engine::EngineError::Cancelled);
}
phase += 1;
Ok(())
};
let mut report = DiffReport::default();
next_phase()?;
if let Ok(project) = qproject::get_project_by_id(conn, project_id) {
match diff_project(data_dir, &project) {
match diff_project(conn, data_dir, &project) {
Ok(Some(diff)) => report.diffs.push(diff),
Ok(None) => {}
Err(error) => report.errors.push(format!("project {project_id}: {error}")),
}
for result in [
diff_categories(conn, data_dir, project_id),
diff_category_meta(conn, data_dir, project_id),
diff_publishing(conn, data_dir, project_id),
] {
match result {
Ok(Some(diff)) => report.diffs.push(diff),
Ok(None) => {}
Err(error) => report
.errors
.push(format!("project metadata {project_id}: {error}")),
}
}
}
// 1. Diff posts
next_phase()?;
let posts = qp::list_posts_by_project(conn, project_id)?;
for post in &posts {
if post.file_path.is_empty() {
@@ -94,6 +127,7 @@ pub fn compute_metadata_diff(
}
// 2. Diff translations
next_phase()?;
for post in &posts {
let translations = qt::list_post_translations_by_post(conn, &post.id)?;
for t in &translations {
@@ -109,6 +143,7 @@ pub fn compute_metadata_diff(
}
// 3. Diff media
next_phase()?;
let media_items = qm::list_media_by_project(conn, project_id)?;
for m in &media_items {
if m.sidecar_path.is_empty() {
@@ -133,6 +168,7 @@ pub fn compute_metadata_diff(
}
// 4. Diff templates
next_phase()?;
let templates = qtpl::list_templates_by_project(conn, project_id)?;
for t in &templates {
if t.file_path.is_empty() {
@@ -146,6 +182,7 @@ pub fn compute_metadata_diff(
}
// 5. Diff scripts
next_phase()?;
let scripts = qs::list_scripts_by_project(conn, project_id)?;
for s in &scripts {
if s.file_path.is_empty() {
@@ -159,9 +196,58 @@ pub fn compute_metadata_diff(
}
// 6. Detect orphans
next_phase()?;
if crate::engine::meta::read_project_json(data_dir)
.is_ok_and(|metadata| metadata.semantic_similarity_enabled)
{
let service = crate::engine::embedding::EmbeddingService::production(conn, data_dir);
let keys = qe::list_keys(conn, project_id)?
.into_iter()
.map(|key| (key.post_id.clone(), key))
.collect::<std::collections::HashMap<_, _>>();
for post in &posts {
let expected = service.content_hash_for_post(post)?;
let key = keys.get(&post.id);
let current_hash = key.map(|key| key.content_hash.as_str()).unwrap_or("");
let vector_status = key
.filter(|key| crate::engine::embedding::decode_vector(&key.vector).is_ok())
.map(|_| "ready")
.unwrap_or("missing");
if current_hash != expected || vector_status != "ready" {
let mut fields = Vec::new();
if current_hash != expected {
fields.push(DiffField {
field_name: "content_hash".into(),
db_value: current_hash.into(),
file_value: expected,
});
}
if vector_status != "ready" {
fields.push(DiffField {
field_name: "embedding".into(),
db_value: vector_status.into(),
file_value: "ready".into(),
});
}
report.diffs.push(EntityDiff {
entity_type: "embedding".into(),
entity_id: post.id.clone(),
file_path: format!("projects/{project_id}/embeddings.usearch"),
fields,
});
}
}
}
// 7. Detect orphans
next_phase()?;
let orphans = detect_orphan_files(conn, data_dir, project_id)?;
report.orphans = orphans;
if !on_progress(PHASES, PHASES) {
return Err(crate::engine::EngineError::Cancelled);
}
Ok(report)
}
@@ -177,7 +263,9 @@ pub fn repair_metadata_diff_item(
RepairDirection::FileToDatabase => {
let path = data_dir.join(&item.file_path);
match item.entity_type.as_str() {
"project" => sync_project_from_file(conn, data_dir, project_id)?,
"project" | "categories" | "category_meta" | "publishing" => {
crate::engine::meta::sync_metadata_from_filesystem(conn, data_dir, project_id)?
}
"post" => {
crate::engine::post::rebuild_canonical_post(conn, data_dir, project_id, &path)?;
}
@@ -204,11 +292,18 @@ pub fn repair_metadata_diff_item(
conn, data_dir, project_id, &path,
)?;
}
"embedding" => {
let post = qp::get_post_by_id(conn, &item.entity_id)?;
crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.sync_post(&post)?;
}
other => return unsupported_repair(other),
}
}
RepairDirection::DatabaseToFile => match item.entity_type.as_str() {
"project" => rewrite_project_from_database(conn, data_dir, project_id)?,
"project" | "categories" | "category_meta" | "publishing" => {
crate::engine::meta::flush_metadata_to_filesystem(conn, data_dir, project_id)?
}
"post" => rewrite_post_from_database(conn, data_dir, &item.entity_id)?,
"post_translation" => {
rewrite_post_translation_from_database(conn, data_dir, &item.entity_id)?
@@ -222,25 +317,158 @@ pub fn repair_metadata_diff_item(
)?,
"script" => rewrite_script_from_database(conn, data_dir, &item.entity_id)?,
"template" => rewrite_template_from_database(conn, data_dir, &item.entity_id)?,
"embedding" => crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.flush_project(project_id)?,
other => return unsupported_repair(other),
},
}
Ok(())
}
/// Import one content file reported as a filesystem orphan by
/// [`compute_metadata_diff`]. The normal per-entity rebuild paths remain the
/// sole parsers and writers for these formats.
pub fn import_orphan_file(
conn: &Connection,
data_dir: &Path,
project_id: &str,
orphan: &OrphanFile,
) -> EngineResult<()> {
if orphan.reason != "file_without_db_entry" {
return Err(EngineError::Validation(format!(
"cannot import an orphan that is absent from the filesystem: {}",
orphan.file_path
)));
}
let path = data_dir.join(&orphan.file_path);
let canonical_data_dir = data_dir.canonicalize()?;
let canonical_path = path.canonicalize()?;
if !canonical_path.starts_with(&canonical_data_dir) {
return Err(EngineError::Validation(
"orphan path is outside the active project".into(),
));
}
if orphan.file_path.starts_with("posts/") && orphan.file_path.ends_with(".md") {
let stem = path
.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("");
if crate::engine::post::is_translation_filename(stem) {
crate::engine::post::rebuild_translation(conn, data_dir, project_id, &path)?;
} else {
crate::engine::post::rebuild_canonical_post(conn, data_dir, project_id, &path)?;
}
} else if orphan.file_path.starts_with("media/") && orphan.file_path.ends_with(".meta") {
let raw = fs::read_to_string(&path)?;
if read_translation_sidecar(&raw).is_ok() {
crate::engine::media::rebuild_translation_sidecar(conn, data_dir, project_id, &path)?;
} else {
crate::engine::media::rebuild_canonical_media(conn, data_dir, project_id, &path)?;
}
} else if orphan.file_path.starts_with("scripts/") && orphan.file_path.ends_with(".lua") {
crate::engine::script_rebuild::rebuild_single_script(conn, data_dir, project_id, &path)?;
} else if orphan.file_path.starts_with("templates/") && orphan.file_path.ends_with(".liquid") {
crate::engine::template_rebuild::rebuild_single_template(
conn, data_dir, project_id, &path,
)?;
} else {
return Err(EngineError::Validation(format!(
"unsupported orphan file: {}",
orphan.file_path
)));
}
Ok(())
}
fn diff_project(
conn: &Connection,
data_dir: &Path,
project: &crate::model::Project,
) -> EngineResult<Option<EntityDiff>> {
let metadata = crate::engine::meta::read_project_json(data_dir)?;
let mut fields = Vec::new();
compare_field(&mut fields, "name", &project.name, &metadata.name);
let snapshot = crate::engine::meta::read_project_metadata_snapshot(conn, &project.id)?;
let database = snapshot.as_ref();
compare_field(
&mut fields,
"name",
database.map_or(project.name.as_str(), |value| value.name.as_str()),
&metadata.name,
);
compare_field(
&mut fields,
"description",
project.description.as_deref().unwrap_or(""),
database.map_or_else(
|| project.description.as_deref().unwrap_or(""),
|value| value.description.as_deref().unwrap_or(""),
),
metadata.description.as_deref().unwrap_or(""),
);
if let Some(database) = database {
compare_optional_field(
&mut fields,
"public_url",
&database.public_url,
&metadata.public_url,
);
compare_optional_field(
&mut fields,
"main_language",
&database.main_language,
&metadata.main_language,
);
compare_optional_field(
&mut fields,
"default_author",
&database.default_author,
&metadata.default_author,
);
compare_field(
&mut fields,
"max_posts_per_page",
&database.max_posts_per_page.to_string(),
&metadata.max_posts_per_page.to_string(),
);
compare_field(
&mut fields,
"image_import_concurrency",
&database.image_import_concurrency.to_string(),
&metadata.image_import_concurrency.to_string(),
);
compare_optional_field(
&mut fields,
"blogmark_category",
&database.blogmark_category,
&metadata.blogmark_category,
);
compare_optional_field(
&mut fields,
"pico_theme",
&database.pico_theme,
&metadata.pico_theme,
);
compare_field(
&mut fields,
"semantic_similarity_enabled",
if database.semantic_similarity_enabled {
"true"
} else {
"false"
},
if metadata.semantic_similarity_enabled {
"true"
} else {
"false"
},
);
compare_field(
&mut fields,
"blog_languages",
&json_value(&database.blog_languages)?,
&json_value(&metadata.blog_languages)?,
);
}
Ok((!fields.is_empty()).then_some(EntityDiff {
entity_type: "project".into(),
entity_id: project.id.clone(),
@@ -249,24 +477,96 @@ fn diff_project(
}))
}
fn sync_project_from_file(
fn diff_categories(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
crate::engine::meta::sync_project_from_file(conn, data_dir, project_id)
) -> EngineResult<Option<EntityDiff>> {
let Some(database) = crate::engine::meta::read_categories_snapshot(conn, project_id)? else {
return Ok(None);
};
let filesystem = crate::engine::meta::read_categories_json(data_dir)?;
let mut fields = Vec::new();
compare_field(
&mut fields,
"categories",
&json_value(&database)?,
&json_value(&filesystem)?,
);
Ok((!fields.is_empty()).then_some(EntityDiff {
entity_type: "categories".into(),
entity_id: project_id.into(),
file_path: "meta/categories.json".into(),
fields,
}))
}
fn rewrite_project_from_database(
fn diff_category_meta(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
let project = qproject::get_project_by_id(conn, project_id)?;
let mut metadata = crate::engine::meta::read_project_json(data_dir)?;
metadata.name = project.name;
metadata.description = project.description;
crate::engine::meta::write_project_json(data_dir, &metadata)
) -> EngineResult<Option<EntityDiff>> {
let Some(database) = crate::engine::meta::read_category_meta_snapshot(conn, project_id)? else {
return Ok(None);
};
let filesystem = crate::engine::meta::read_category_meta_json(data_dir)?;
let database = database.into_iter().collect::<BTreeMap<_, _>>();
let filesystem = filesystem.into_iter().collect::<BTreeMap<_, _>>();
let mut fields = Vec::new();
compare_field(
&mut fields,
"category_settings",
&json_value(&database)?,
&json_value(&filesystem)?,
);
Ok((!fields.is_empty()).then_some(EntityDiff {
entity_type: "category_meta".into(),
entity_id: project_id.into(),
file_path: "meta/category-meta.json".into(),
fields,
}))
}
fn diff_publishing(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<Option<EntityDiff>> {
let Some(database) = crate::engine::meta::read_publishing_snapshot(conn, project_id)? else {
return Ok(None);
};
let filesystem = crate::engine::meta::read_publishing_json(data_dir)?;
let mut fields = Vec::new();
compare_optional_field(
&mut fields,
"ssh_host",
&database.ssh_host,
&filesystem.ssh_host,
);
compare_optional_field(
&mut fields,
"ssh_user",
&database.ssh_user,
&filesystem.ssh_user,
);
compare_optional_field(
&mut fields,
"ssh_remote_path",
&database.ssh_remote_path,
&filesystem.ssh_remote_path,
);
compare_field(
&mut fields,
"ssh_mode",
ssh_mode_name(&database.ssh_mode),
ssh_mode_name(&filesystem.ssh_mode),
);
Ok((!fields.is_empty()).then_some(EntityDiff {
entity_type: "publishing".into(),
entity_id: project_id.into(),
file_path: "meta/publishing.json".into(),
fields,
}))
}
fn unsupported_repair<T>(entity_type: &str) -> EngineResult<T> {
@@ -281,13 +581,7 @@ fn rewrite_post_from_database(
entity_id: &str,
) -> EngineResult<()> {
let post = qp::get_post_by_id(conn, entity_id)?;
let path = data_dir.join(&post.file_path);
let body = fs::read_to_string(&path)
.ok()
.and_then(|content| read_post_file(&content).ok().map(|(_, body)| body))
.unwrap_or_default();
atomic_write_str(&path, &write_post_file(&post, &body))?;
Ok(())
crate::engine::post::rewrite_post_file_from_database(data_dir, &post)
}
fn rewrite_post_translation_from_database(
@@ -436,6 +730,31 @@ fn compare_field(fields: &mut Vec<DiffField>, name: &str, db_val: &str, file_val
}
}
fn compare_optional_field(
fields: &mut Vec<DiffField>,
name: &str,
db_val: &Option<String>,
file_val: &Option<String>,
) {
compare_field(
fields,
name,
db_val.as_deref().unwrap_or(""),
file_val.as_deref().unwrap_or(""),
);
}
fn json_value(value: &impl serde::Serialize) -> EngineResult<String> {
Ok(serde_json::to_string(value)?)
}
fn ssh_mode_name(mode: &crate::model::SshMode) -> &'static str {
match mode {
crate::model::SshMode::Scp => "scp",
crate::model::SshMode::Rsync => "rsync",
}
}
fn diff_post(data_dir: &Path, post: &Post) -> EngineResult<Option<EntityDiff>> {
let abs_path = data_dir.join(&post.file_path);
if !abs_path.exists() {
@@ -935,10 +1254,10 @@ mod tests {
use crate::db::queries::project::{insert_project, make_test_project};
use crate::db::queries::script::insert_script;
use crate::db::queries::template::insert_template;
use crate::engine::post::{create_post, publish_post};
use crate::engine::post::{archive_post, create_post, publish_post};
use crate::model::{
Media, Post, PostStatus, ProjectMetadata, Script, ScriptKind, ScriptStatus, Template,
TemplateKind, TemplateStatus,
Media, Post, PostStatus, ProjectMetadata, PublishingPreferences, Script, ScriptKind,
ScriptStatus, SshMode, Template, TemplateKind, TemplateStatus,
};
use crate::util::frontmatter::{
ScriptFrontmatter, TemplateFrontmatter, write_script_file, write_template_file,
@@ -995,6 +1314,58 @@ mod tests {
assert!(report.orphans.is_empty(), "orphans: {:?}", report.orphans);
}
#[test]
fn archived_published_post_reports_only_bds2_frontmatter_drift() {
let (db, dir) = setup();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Archived Post",
Some("body text"),
vec!["rust".into()],
vec!["tech".into()],
Some("Alice"),
Some("en"),
None,
)
.unwrap();
let published = publish_post(db.conn(), dir.path(), &post.id).unwrap();
let path = dir.path().join(&published.file_path);
let original_file = fs::read(&path).unwrap();
archive_post(db.conn(), dir.path(), &post.id).unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(fs::read(path).unwrap(), original_file);
assert!(
report
.errors
.iter()
.all(|error| !error.starts_with("post ")),
"post diff errors: {:?}",
report.errors
);
assert!(report.orphans.is_empty());
let diff = report
.diffs
.iter()
.find(|diff| diff.entity_type == "post" && diff.entity_id == post.id)
.expect("archived database status must differ from untouched published file");
assert!(diff.fields.iter().any(|field| {
field.field_name == "status"
&& field.db_value == "archived"
&& field.file_value == "published"
}));
assert!(
diff.fields
.iter()
.all(|field| matches!(field.field_name.as_str(), "status" | "updatedAt")),
"unexpected archived-post drift: {:?}",
diff.fields
);
}
#[test]
fn detects_title_drift_in_post() {
let (db, dir) = setup();
@@ -1071,6 +1442,137 @@ mod tests {
}));
}
fn seed_portable_metadata(db: &Database, dir: &TempDir) {
crate::engine::meta::startup_sync(dir.path()).unwrap();
crate::engine::meta::sync_metadata_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
crate::engine::meta::set_categories(
db.conn(),
dir.path(),
"p1",
&["article".into(), "news".into()],
)
.unwrap();
let mut category_meta = std::collections::HashMap::new();
category_meta.insert(
"news".to_string(),
crate::model::metadata::CategorySettings {
title: Some("News".into()),
titles: std::collections::BTreeMap::new(),
render_in_lists: false,
show_title: true,
post_template_slug: Some("article".into()),
list_template_slug: None,
},
);
crate::engine::meta::set_category_meta(db.conn(), dir.path(), "p1", &category_meta)
.unwrap();
crate::engine::meta::set_publishing_preferences(
db.conn(),
dir.path(),
"p1",
&PublishingPreferences {
ssh_host: Some("example.net".into()),
ssh_user: Some("deploy".into()),
ssh_remote_path: Some("/srv/blog".into()),
ssh_mode: SshMode::Rsync,
},
)
.unwrap();
}
#[test]
fn detects_categories_category_meta_and_publishing_drift() {
let (db, dir) = setup();
seed_portable_metadata(&db, &dir);
crate::engine::meta::write_categories_json(dir.path(), &["article".into()]).unwrap();
crate::engine::meta::write_category_meta_json(dir.path(), &Default::default()).unwrap();
crate::engine::meta::write_publishing_json(dir.path(), &PublishingPreferences::default())
.unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
for entity_type in ["categories", "category_meta", "publishing"] {
assert!(
report
.diffs
.iter()
.any(|item| item.entity_type == entity_type),
"missing {entity_type} diff: {:?}",
report.diffs
);
}
}
#[test]
fn repairs_portable_metadata_in_both_directions() {
let (db, dir) = setup();
seed_portable_metadata(&db, &dir);
crate::engine::meta::write_categories_json(dir.path(), &["filesystem".into()]).unwrap();
crate::engine::meta::write_category_meta_json(dir.path(), &Default::default()).unwrap();
crate::engine::meta::write_publishing_json(dir.path(), &PublishingPreferences::default())
.unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
let categories = report
.diffs
.iter()
.find(|item| item.entity_type == "categories")
.unwrap();
repair_metadata_diff_item(
db.conn(),
dir.path(),
"p1",
RepairDirection::DatabaseToFile,
categories,
)
.unwrap();
assert_eq!(
crate::engine::meta::read_categories_json(dir.path()).unwrap(),
vec!["article", "news"]
);
assert!(
crate::engine::meta::read_category_meta_json(dir.path())
.unwrap()
.contains_key("news")
);
assert_eq!(
crate::engine::meta::read_publishing_json(dir.path())
.unwrap()
.ssh_host
.as_deref(),
Some("example.net")
);
crate::engine::meta::write_categories_json(dir.path(), &["filesystem".into()]).unwrap();
crate::engine::meta::write_category_meta_json(dir.path(), &Default::default()).unwrap();
crate::engine::meta::write_publishing_json(dir.path(), &PublishingPreferences::default())
.unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
let categories = report
.diffs
.iter()
.find(|item| item.entity_type == "categories")
.unwrap();
repair_metadata_diff_item(
db.conn(),
dir.path(),
"p1",
RepairDirection::FileToDatabase,
categories,
)
.unwrap();
assert!(
compute_metadata_diff(db.conn(), dir.path(), "p1")
.unwrap()
.diffs
.iter()
.all(|item| !matches!(
item.entity_type.as_str(),
"categories" | "category_meta" | "publishing"
))
);
}
#[test]
fn ignores_python_scripts_during_orphan_scan() {
let (db, dir) = setup();
@@ -1091,12 +1593,38 @@ mod tests {
#[test]
fn repairs_one_post_diff_from_file_to_database() {
let (db, dir) = setup();
let database_target = create_post(
db.conn(),
dir.path(),
"p1",
"Database Target",
None,
vec![],
vec![],
None,
None,
None,
)
.unwrap();
let filesystem_target = create_post(
db.conn(),
dir.path(),
"p1",
"Filesystem Target",
None,
vec![],
vec![],
None,
None,
None,
)
.unwrap();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Database Title",
Some("body"),
Some("[database](/2024/01/01/database-target)"),
vec![],
vec![],
None,
@@ -1107,7 +1635,16 @@ mod tests {
let published = publish_post(db.conn(), dir.path(), &post.id).unwrap();
let path = dir.path().join(&published.file_path);
let content = fs::read_to_string(&path).unwrap();
fs::write(&path, content.replace("Database Title", "Filesystem Title")).unwrap();
fs::write(
&path,
content
.replace("Database Title", "Filesystem Title")
.replace(
"[database](/2024/01/01/database-target)",
"[filesystem](/2024/01/01/filesystem-target)",
),
)
.unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
let item = report
.diffs
@@ -1128,6 +1665,11 @@ mod tests {
qp::get_post_by_id(db.conn(), &post.id).unwrap().title,
"Filesystem Title"
);
let links =
crate::db::queries::post_link::list_links_by_source(db.conn(), &post.id).unwrap();
assert_eq!(links.len(), 1);
assert_eq!(links[0].target_post_id, filesystem_target.id);
assert_ne!(links[0].target_post_id, database_target.id);
}
#[test]

View File

@@ -2,8 +2,18 @@ pub mod ai;
pub mod auto_translation;
pub mod blogmark;
pub mod calendar;
pub mod chat;
pub mod chat_surfaces;
mod chat_tools;
pub mod cli_launcher;
pub mod cli_sync;
pub mod domain_events;
pub mod embedding;
pub mod error;
pub mod gallery_import;
pub mod generation;
pub mod git;
pub mod mcp;
pub mod media;
pub mod menu;
pub mod meta;
@@ -17,13 +27,16 @@ pub mod rebuild;
pub mod script;
pub mod script_rebuild;
pub mod search;
pub mod settings;
pub mod site_assets;
pub mod tag;
pub mod task;
pub mod template;
pub mod template_rebuild;
pub mod ui_state;
pub mod validate_media;
pub mod validate_site;
pub mod validate_translations;
pub mod wordpress_import;
pub use error::{EngineError, EngineResult};

File diff suppressed because it is too large Load Diff

View File

@@ -8,8 +8,7 @@ use crate::db::queries::post as qp;
use crate::db::queries::post_media as qpm;
use crate::engine::EngineResult;
use crate::model::{Media, Post, PostMedia};
use crate::util::sidecar::MediaSidecar;
use crate::util::{atomic_write_str, now_unix_ms};
use crate::util::now_unix_ms;
/// Link a media item to a post and sync the media sidecar.
pub fn link_media_to_post(
@@ -31,7 +30,7 @@ pub fn link_media_to_post(
created_at: now,
};
qpm::link_media(conn, &pm)?;
sync_sidecar_linked_post_ids(conn, data_dir, media_id)?;
crate::engine::media::sync_media_sidecar(conn, data_dir, media_id)?;
Ok(pm)
}
@@ -43,7 +42,7 @@ pub fn unlink_media_from_post(
media_id: &str,
) -> EngineResult<()> {
qpm::unlink_media(conn, post_id, media_id)?;
sync_sidecar_linked_post_ids(conn, data_dir, media_id)?;
crate::engine::media::sync_media_sidecar(conn, data_dir, media_id)?;
Ok(())
}
@@ -84,22 +83,6 @@ pub fn list_posts_for_media(conn: &Connection, media_id: &str) -> EngineResult<V
Ok(posts)
}
/// Rebuild the media sidecar file so that `linkedPostIds` reflects the current
/// set of posts linked to this media item.
fn sync_sidecar_linked_post_ids(
conn: &Connection,
data_dir: &Path,
media_id: &str,
) -> EngineResult<()> {
let links = qpm::list_post_media_by_media(conn, media_id)?;
let post_ids: Vec<String> = links.iter().map(|pm| pm.post_id.clone()).collect();
let media = qm::get_media_by_id(conn, media_id)?;
let sidecar = MediaSidecar::from_media(&media, &post_ids);
let abs_path = data_dir.join(&media.sidecar_path);
atomic_write_str(&abs_path, &sidecar.to_string())?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -112,6 +95,7 @@ mod tests {
use crate::db::queries::post::{insert_post, make_test_post};
use crate::db::queries::post_media::list_post_media_by_post;
use crate::db::queries::project::{insert_project, make_test_project};
use crate::util::sidecar::MediaSidecar;
use crate::util::sidecar::read_sidecar;
fn setup() -> (Database, TempDir) {

File diff suppressed because it is too large Load Diff

View File

@@ -7,9 +7,9 @@ use uuid::Uuid;
use crate::db::queries::project as q;
use crate::engine::site_assets::copy_bundled_site_assets;
use crate::engine::{EngineError, EngineResult};
use crate::model::Project;
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::model::metadata::ProjectMetadata;
use crate::model::{DomainEntity, NotificationAction, Project};
use crate::util::{atomic_write_str, ensure_unique, now_unix_ms, slugify};
/// The well-known ID of the default project (spec: DefaultProjectExists).
@@ -56,7 +56,10 @@ pub fn create_project(
}
// Write default meta files
write_default_meta_files(&data_dir, name)?;
write_default_meta_files(&data_dir, &project)?;
crate::engine::meta::sync_metadata_from_filesystem(conn, &data_dir, &project.id)?;
emit_project(&project, NotificationAction::Created);
Ok(project)
}
@@ -120,6 +123,9 @@ pub fn open_project(conn: &Connection, folder_path: &Path) -> EngineResult<Proje
project.data_path = Some(resolved_path);
project.updated_at = now_unix_ms();
q::update_project(conn, &project)?;
crate::engine::meta::startup_sync(&folder_path)?;
crate::engine::meta::initialize_metadata_snapshots(conn, &folder_path, &project.id)?;
emit_project(&project, NotificationAction::Updated);
return Ok(project);
}
@@ -138,6 +144,9 @@ pub fn open_project(conn: &Connection, folder_path: &Path) -> EngineResult<Proje
updated_at: now,
};
q::insert_project(conn, &project)?;
crate::engine::meta::startup_sync(&folder_path)?;
crate::engine::meta::sync_metadata_from_filesystem(conn, &folder_path, &project.id)?;
emit_project(&project, NotificationAction::Created);
Ok(project)
}
@@ -149,7 +158,13 @@ pub fn ensure_default_project(
default_data_dir: Option<&Path>,
) -> EngineResult<Project> {
match q::get_project_by_id(conn, DEFAULT_PROJECT_ID) {
Ok(p) => Ok(p),
Ok(p) => {
if let Some(data_dir) = default_data_dir {
crate::engine::meta::startup_sync(data_dir)?;
crate::engine::meta::initialize_metadata_snapshots(conn, data_dir, &p.id)?;
}
Ok(p)
}
Err(diesel::result::Error::NotFound) => {
let now = now_unix_ms();
let project = Project {
@@ -169,7 +184,9 @@ pub fn ensure_default_project(
None => std::path::PathBuf::from("projects").join(DEFAULT_PROJECT_ID),
};
create_directory_structure(&data_dir)?;
write_default_meta_files(&data_dir, "My Blog")?;
write_default_meta_files(&data_dir, &project)?;
crate::engine::meta::sync_metadata_from_filesystem(conn, &data_dir, &project.id)?;
emit_project(&project, NotificationAction::Created);
Ok(project)
}
Err(e) => Err(EngineError::Db(e)),
@@ -188,6 +205,12 @@ pub fn get_active_project(conn: &Connection) -> EngineResult<Option<Project>> {
/// Deactivate all projects, then activate the given one.
pub fn set_active_project(conn: &Connection, project_id: &str) -> EngineResult<()> {
q::set_active_project(conn, project_id)?;
domain_events::entity_changed(
project_id,
DomainEntity::Project,
project_id,
NotificationAction::Updated,
);
Ok(())
}
@@ -226,7 +249,23 @@ pub fn delete_project(
.map_err(|_| EngineError::NotFound(format!("project {project_id}")))?;
let is_custom_path = project.data_path.is_some();
q::delete_project(conn, project_id)?;
conn.begin_savepoint()?;
let deleted = (|| {
q::delete_project(conn, project_id)?;
crate::db::queries::setting::delete_settings_by_prefix(
conn,
&format!("project:{project_id}:"),
)?;
Ok::<_, EngineError>(())
})();
match deleted {
Ok(()) => conn.release_savepoint()?,
Err(error) => {
let _ = conn.rollback_savepoint();
return Err(error);
}
}
crate::engine::embedding::EmbeddingService::forget_project(project_id);
// Clean up internal filesystem only (not custom external paths per spec)
if !is_custom_path
@@ -236,9 +275,15 @@ pub fn delete_project(
let _ = fs::remove_dir_all(dir);
}
emit_project(&project, NotificationAction::Deleted);
Ok(())
}
fn emit_project(project: &Project, action: NotificationAction) {
domain_events::entity_changed(&project.id, DomainEntity::Project, &project.id, action);
}
// ── helpers ──────────────────────────────────────────────────────────
fn create_directory_structure(data_dir: &Path) -> EngineResult<()> {
@@ -257,12 +302,12 @@ fn create_directory_structure(data_dir: &Path) -> EngineResult<()> {
Ok(())
}
fn write_default_meta_files(data_dir: &Path, project_name: &str) -> EngineResult<()> {
fn write_default_meta_files(data_dir: &Path, project: &Project) -> EngineResult<()> {
let meta_dir = data_dir.join("meta");
// project.json
let project_meta = ProjectMetadata {
name: project_name.to_string(),
name: project.name.clone(),
description: None,
public_url: None,
main_language: None,
@@ -287,14 +332,19 @@ fn write_default_meta_files(data_dir: &Path, project_name: &str) -> EngineResult
let json = serde_json::to_string_pretty(&empty_map)?;
atomic_write_str(&meta_dir.join("category-meta.json"), &json)?;
// publishing.json — empty object
atomic_write_str(&meta_dir.join("publishing.json"), "{}")?;
// publishing.json
crate::engine::meta::write_publishing_json(data_dir, &Default::default())?;
// tags.json — empty array
atomic_write_str(&meta_dir.join("tags.json"), "[]")?;
// menu.opml — default empty menu per menu.allium HomeAlwaysPresent
let default_opml = crate::engine::menu::default_menu_opml();
let timestamp = if project.updated_at > 0 {
project.updated_at
} else {
project.created_at
};
let default_opml = crate::engine::menu::default_menu_opml(&project.name, timestamp);
atomic_write_str(&meta_dir.join("menu.opml"), &default_opml)?;
copy_bundled_site_assets(data_dir)?;
@@ -474,12 +524,26 @@ mod tests {
updated_at: now,
};
crate::db::queries::project::insert_project(db.conn(), &project).unwrap();
crate::db::queries::setting::set_setting_value(
db.conn(),
&format!("project:{}:categories", project.id),
r#"{"categories":[]}"#,
now,
)
.unwrap();
// Create the internal directory structure under the temp dir
let internal_dir = dir.path().join("projects").join(&project.id);
create_directory_structure(&internal_dir).unwrap();
assert!(internal_dir.join("posts").is_dir());
delete_project(db.conn(), &project.id, Some(&internal_dir)).unwrap();
assert!(list_projects(db.conn()).unwrap().is_empty());
assert!(
crate::db::queries::setting::get_setting_by_key(
db.conn(),
&format!("project:{}:categories", project.id)
)
.is_err()
);
assert!(
!internal_dir.exists(),
"internal directory should be cleaned up"

View File

@@ -1,7 +1,10 @@
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use serde::{Deserialize, Serialize};
use walkdir::WalkDir;
@@ -124,6 +127,26 @@ pub fn upload_site(
private_cache_dir: &Path,
preferences: &PublishingPreferences,
mut on_progress: impl FnMut(usize, usize, UploadTargetKind),
) -> EngineResult<PublishJob> {
let cancelled = AtomicBool::new(false);
upload_site_cancellable(
data_dir,
private_cache_dir,
preferences,
&cancelled,
move |current, total, kind| {
on_progress(current, total, kind);
true
},
)
}
pub fn upload_site_cancellable(
data_dir: &Path,
private_cache_dir: &Path,
preferences: &PublishingPreferences,
cancel_flag: &AtomicBool,
mut on_progress: impl FnMut(usize, usize, UploadTargetKind) -> bool,
) -> EngineResult<PublishJob> {
if std::env::var_os("SSH_AUTH_SOCK").is_none() {
return Err(EngineError::Validation(
@@ -134,7 +157,7 @@ pub fn upload_site(
data_dir,
private_cache_dir,
preferences,
&mut |program, args| run_command(program, args),
&mut |program, args| run_command_cancellable(program, args, cancel_flag),
&mut on_progress,
)
}
@@ -144,7 +167,7 @@ fn upload_site_with_runner(
private_cache_dir: &Path,
preferences: &PublishingPreferences,
runner: &mut CommandRunner<'_>,
on_progress: &mut dyn FnMut(usize, usize, UploadTargetKind),
on_progress: &mut dyn FnMut(usize, usize, UploadTargetKind) -> bool,
) -> EngineResult<PublishJob> {
let credentials = Credentials::from_preferences(preferences)?;
let targets = build_upload_targets(data_dir, &credentials);
@@ -162,7 +185,9 @@ fn upload_site_with_runner(
let cache_path = private_cache_dir.join("publishing-scp-mtimes.json");
let mut cache = read_cache(&cache_path);
for (index, target) in targets.iter().enumerate() {
on_progress(index + 1, targets.len(), target.kind);
if !on_progress(index + 1, targets.len(), target.kind) {
return Err(EngineError::Cancelled);
}
let result = match credentials.mode {
SshMode::Rsync => upload_rsync(target, &credentials, runner),
SshMode::Scp => upload_scp(target, &credentials, &mut cache, runner),
@@ -328,21 +353,49 @@ fn write_cache(path: &Path, cache: &ScpMtimeCache) -> EngineResult<()> {
Ok(())
}
fn run_command(program: &str, args: &[String]) -> Result<(), String> {
let output = Command::new(program)
fn run_command_cancellable(
program: &str,
args: &[String],
cancel_flag: &AtomicBool,
) -> Result<(), String> {
let mut child = Command::new(program)
.args(args)
.output()
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| format!("failed to start {program}: {error}"))?;
if output.status.success() {
Ok(())
} else {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
Err(if stderr.is_empty() {
format!("{program} exited with {}", output.status)
} else {
stderr
})
let stderr = child.stderr.take();
let stderr_reader = std::thread::spawn(move || {
let mut output = String::new();
if let Some(mut stderr) = stderr {
let _ = stderr.read_to_string(&mut output);
}
output
});
let status = loop {
if cancel_flag.load(Ordering::Acquire) {
let _ = child.kill();
let _ = child.wait();
let _ = stderr_reader.join();
return Err("operation cancelled".into());
}
if let Some(status) = child
.try_wait()
.map_err(|error| format!("failed to wait for {program}: {error}"))?
{
break status;
}
std::thread::sleep(Duration::from_millis(50));
};
let stderr = stderr_reader.join().unwrap_or_default().trim().to_string();
if status.success() {
return Ok(());
}
Err(if stderr.is_empty() {
format!("{program} exited with {status}")
} else {
stderr
})
}
#[cfg(test)]
@@ -376,7 +429,7 @@ mod tests {
commands.push((program.to_owned(), args.to_vec()));
Ok(())
},
&mut |_, _, _| {},
&mut |_, _, _| true,
)
.unwrap();
@@ -397,6 +450,28 @@ mod tests {
);
}
#[test]
fn cancellation_stops_before_starting_the_next_target() {
let dir = TempDir::new().unwrap();
let cache = TempDir::new().unwrap();
fs::create_dir_all(dir.path().join("html")).unwrap();
let mut commands = 0;
let result = upload_site_with_runner(
dir.path(),
cache.path(),
&preferences(SshMode::Rsync),
&mut |_, _| {
commands += 1;
Ok(())
},
&mut |_, _, _| false,
);
assert!(matches!(result, Err(EngineError::Cancelled)));
assert_eq!(commands, 0);
}
#[test]
fn scp_excludes_sidecars_and_skips_unchanged_files() {
let dir = TempDir::new().unwrap();
@@ -415,7 +490,7 @@ mod tests {
first.push((program.to_owned(), args.to_vec()));
Ok(())
},
&mut |_, _, _| {},
&mut |_, _, _| true,
)
.unwrap();
assert_eq!(
@@ -437,7 +512,7 @@ mod tests {
second.push(program.to_owned());
Ok(())
},
&mut |_, _, _| {},
&mut |_, _, _| true,
)
.unwrap();
assert!(second.is_empty());
@@ -454,7 +529,7 @@ mod tests {
cache.path(),
&preferences(SshMode::Rsync),
&mut |_, _| Err("network down".into()),
&mut |_, _, _| {},
&mut |_, _, _| true,
)
.unwrap_err();
assert!(error.to_string().contains("network down"));

View File

@@ -10,6 +10,7 @@ use crate::engine::media;
use crate::engine::post;
use crate::engine::script_rebuild;
use crate::engine::template_rebuild;
use crate::i18n::{UiLocale, translate, translate_with};
/// Report from a full rebuild operation.
#[derive(Debug, Default)]
@@ -26,11 +27,141 @@ pub struct FullRebuildReport {
pub templates_updated: usize,
pub scripts_created: usize,
pub scripts_updated: usize,
pub thumbnails_generated: usize,
pub thumbnail_media_failed: usize,
pub errors: Vec<String>,
}
/// Progress callback: (percent 0.0..1.0, phase description).
pub type ProgressFn = Arc<dyn Fn(f32, &str) + Send + Sync>;
#[derive(Debug, Default, PartialEq, Eq)]
pub struct IncrementalRebuildReport {
pub differences_applied: usize,
pub orphans_imported: usize,
pub orphans_failed: usize,
}
/// Apply portable filesystem changes to one project's cache database.
pub fn rebuild_incremental(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<IncrementalRebuildReport> {
let report = super::metadata_diff::compute_metadata_diff(conn, data_dir, project_id)?;
if !report.errors.is_empty() {
return Err(crate::engine::EngineError::Validation(
report.errors.join("; "),
));
}
for item in &report.diffs {
super::metadata_diff::repair_metadata_diff_item(
conn,
data_dir,
project_id,
super::metadata_diff::RepairDirection::FileToDatabase,
item,
)?;
}
let mut result = IncrementalRebuildReport {
differences_applied: report.diffs.len(),
..Default::default()
};
for orphan in report
.orphans
.iter()
.filter(|orphan| orphan.reason == "file_without_db_entry")
{
match super::metadata_diff::import_orphan_file(conn, data_dir, project_id, orphan) {
Ok(()) => result.orphans_imported += 1,
Err(_) => result.orphans_failed += 1,
}
}
Ok(result)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RebuildProgress {
LoadingProjectMetadata,
ScanningPosts,
PostItem {
current: usize,
total: usize,
name: String,
},
ScanningMedia,
MediaItem {
current: usize,
total: usize,
name: String,
},
RebuildingTemplates,
TemplateItem {
current: usize,
total: usize,
name: String,
},
RebuildingScripts,
ScriptItem {
current: usize,
total: usize,
name: String,
},
ImportingTags,
RebuildingThumbnails,
RefreshingSemanticIndex,
Complete,
}
impl RebuildProgress {
pub fn localized(&self, locale: UiLocale) -> String {
match self {
Self::LoadingProjectMetadata => {
translate(locale, "engine.progress.loadingProjectMetadata")
}
Self::ScanningPosts => translate(locale, "engine.progress.scanningPosts"),
Self::PostItem {
current,
total,
name,
} => localize_item(locale, "engine.progress.postItem", *current, *total, name),
Self::ScanningMedia => translate(locale, "engine.progress.scanningMedia"),
Self::MediaItem {
current,
total,
name,
} => localize_item(locale, "engine.progress.mediaItem", *current, *total, name),
Self::RebuildingTemplates => translate(locale, "engine.progress.rebuildingTemplates"),
Self::TemplateItem {
current,
total,
name,
} => localize_item(locale, "engine.checkingItem", *current, *total, name),
Self::RebuildingScripts => translate(locale, "engine.progress.rebuildingScripts"),
Self::ScriptItem {
current,
total,
name,
} => localize_item(locale, "engine.checkingItem", *current, *total, name),
Self::ImportingTags => translate(locale, "engine.progress.importingTags"),
Self::RebuildingThumbnails => translate(locale, "engine.progress.rebuildingThumbnails"),
Self::RefreshingSemanticIndex => {
translate(locale, "engine.progress.refreshingSemanticIndex")
}
Self::Complete => translate(locale, "engine.progress.rebuildComplete"),
}
}
}
fn localize_item(locale: UiLocale, key: &str, current: usize, total: usize, name: &str) -> String {
let current = current.to_string();
let total = total.to_string();
translate_with(
locale,
key,
&[("current", &current), ("total", &total), ("name", name)],
)
}
/// Progress callback: (percent 0.0..1.0, semantic progress event).
pub type ProgressFn = Arc<dyn Fn(f32, &RebuildProgress) -> bool + Send + Sync>;
/// Orchestrate a full rebuild from filesystem into the database.
///
@@ -71,22 +202,26 @@ fn rebuild_from_filesystem_inner(
on_progress: Option<ProgressFn>,
) -> EngineResult<FullRebuildReport> {
let mut report = FullRebuildReport::default();
let progress = |pct: f32, msg: &str| {
if let Some(ref f) = on_progress {
f(pct, msg);
let progress = |pct: f32, event: RebuildProgress| -> EngineResult<()> {
if let Some(ref f) = on_progress
&& !f(pct, &event)
{
return Err(crate::engine::EngineError::Cancelled);
}
Ok(())
};
// Phase weights: posts 0.0..0.35, media 0.35..0.70, templates 0.70..0.85, scripts 0.85..1.0
// 1. Load portable project metadata and clear all reconstructible rows.
progress(0.0, "Loading project metadata...");
progress(0.0, RebuildProgress::LoadingProjectMetadata)?;
fts::ensure_fts_tables(conn)?;
crate::engine::meta::sync_project_from_file(conn, data_dir, project_id)?;
crate::engine::meta::startup_sync(data_dir)?;
crate::engine::meta::sync_metadata_from_filesystem(conn, data_dir, project_id)?;
clear_project_rows(conn, project_id)?;
// 2. Rebuild posts (0.00 .. 0.35)
progress(0.01, "Scanning posts...");
progress(0.01, RebuildProgress::ScanningPosts)?;
let post_item_cb: Option<post::ItemProgressFn> = on_progress.as_ref().map(|cb| {
let cb = Arc::clone(cb);
let f: post::ItemProgressFn = Box::new(move |current, total, name| {
@@ -96,8 +231,14 @@ fn rebuild_from_filesystem_inner(
1.0
};
let global_pct = 0.01 + phase_pct * 0.34;
let msg = format!("Posts: {current}/{total} \u{2014} {name}");
cb(global_pct, &msg);
cb(
global_pct,
&RebuildProgress::PostItem {
current,
total,
name: name.to_string(),
},
)
});
f
});
@@ -114,7 +255,7 @@ fn rebuild_from_filesystem_inner(
report.errors.extend(post_report.errors);
// 3. Rebuild media (0.35 .. 0.70)
progress(0.35, "Scanning media...");
progress(0.35, RebuildProgress::ScanningMedia)?;
let media_item_cb: Option<media::ItemProgressFn> = on_progress.as_ref().map(|cb| {
let cb = Arc::clone(cb);
let f: media::ItemProgressFn = Box::new(move |current, total, name| {
@@ -124,8 +265,14 @@ fn rebuild_from_filesystem_inner(
1.0
};
let global_pct = 0.35 + phase_pct * 0.35;
let msg = format!("Media: {current}/{total} \u{2014} {name}");
cb(global_pct, &msg);
cb(
global_pct,
&RebuildProgress::MediaItem {
current,
total,
name: name.to_string(),
},
)
});
f
});
@@ -142,26 +289,87 @@ fn rebuild_from_filesystem_inner(
report.errors.extend(media_report.errors);
// 4. Rebuild templates (0.70 .. 0.85)
progress(0.70, "Rebuilding templates...");
let tpl_report =
template_rebuild::rebuild_templates_from_filesystem(conn, data_dir, project_id)?;
progress(0.70, RebuildProgress::RebuildingTemplates)?;
let template_progress = on_progress.as_ref().map(|callback| {
let callback = Arc::clone(callback);
Box::new(move |current: usize, total: usize, name: &str| {
callback(
0.70 + current as f32 / total.max(1) as f32 * 0.15,
&RebuildProgress::TemplateItem {
current,
total,
name: name.to_string(),
},
)
}) as template_rebuild::ItemProgressFn
});
let tpl_report = template_rebuild::rebuild_templates_from_filesystem_with_progress(
conn,
data_dir,
project_id,
template_progress,
)?;
report.templates_created = tpl_report.created;
report.templates_updated = tpl_report.updated;
report.errors.extend(tpl_report.errors);
// 5. Rebuild scripts (0.85 .. 0.95)
progress(0.85, "Rebuilding scripts...");
let script_report =
script_rebuild::rebuild_scripts_from_filesystem(conn, data_dir, project_id)?;
progress(0.85, RebuildProgress::RebuildingScripts)?;
let script_progress = on_progress.as_ref().map(|callback| {
let callback = Arc::clone(callback);
Box::new(move |current: usize, total: usize, name: &str| {
callback(
0.85 + current as f32 / total.max(1) as f32 * 0.10,
&RebuildProgress::ScriptItem {
current,
total,
name: name.to_string(),
},
)
}) as script_rebuild::ItemProgressFn
});
let script_report = script_rebuild::rebuild_scripts_from_filesystem_with_progress(
conn,
data_dir,
project_id,
script_progress,
)?;
report.scripts_created = script_report.created;
report.scripts_updated = script_report.updated;
report.errors.extend(script_report.errors);
// 6. Restore relationships and tags (0.95 .. 1.0)
progress(0.95, "Importing tags...");
progress(0.95, RebuildProgress::ImportingTags)?;
super::tag::import_tags_from_file(conn, data_dir, project_id)?;
super::tag::sync_tags_from_posts(conn, project_id)?;
post::rebuild_all_links(conn, data_dir, project_id)?;
let link_progress = on_progress.as_ref().map(|callback| {
let callback = Arc::clone(callback);
Box::new(move |current: usize, total: usize, name: &str| {
callback(
0.95 + current as f32 / total.max(1) as f32 * 0.01,
&RebuildProgress::PostItem {
current,
total,
name: name.to_string(),
},
)
}) as post::ItemProgressFn
});
post::rebuild_all_links_with_progress(conn, data_dir, project_id, link_progress)?;
let media_link_progress = on_progress.as_ref().map(|callback| {
let callback = Arc::clone(callback);
Box::new(move |current: usize, total: usize, name: &str| {
callback(
0.96 + current as f32 / total.max(1) as f32 * 0.01,
&RebuildProgress::MediaItem {
current,
total,
name: name.to_string(),
},
)
}) as media::ItemProgressFn
});
media::rebuild_media_links_with_progress(conn, data_dir, project_id, media_link_progress)?;
if !report.errors.is_empty() {
return Err(crate::engine::EngineError::Validation(format!(
@@ -170,15 +378,46 @@ fn rebuild_from_filesystem_inner(
)));
}
progress(1.0, "Rebuild complete");
progress(0.97, RebuildProgress::RebuildingThumbnails)?;
let thumbnail_report = media::regenerate_missing_thumbnails_with_progress(
conn,
data_dir,
project_id,
|current, total, name| {
on_progress.as_ref().is_none_or(|callback| {
callback(
0.97 + current as f32 / total.max(1) as f32 * 0.02,
&RebuildProgress::MediaItem {
current,
total,
name: name.to_string(),
},
)
})
},
)?;
report.thumbnails_generated = thumbnail_report.thumbnails_generated;
report.thumbnail_media_failed = thumbnail_report.media_failed;
progress(0.99, RebuildProgress::RefreshingSemanticIndex)?;
crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.index_unindexed_with_progress(project_id, |current, total| {
on_progress.as_ref().is_none_or(|callback| {
callback(
0.99 + current as f32 / total.max(1) as f32 * 0.01,
&RebuildProgress::RefreshingSemanticIndex,
)
})
})?;
progress(1.0, RebuildProgress::Complete)?;
Ok(report)
}
fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
use crate::db::schema::{
dismissed_duplicate_pairs, embedding_keys, generated_file_hashes, import_definitions,
media, media_translations, post_links, post_media, post_translations, posts, scripts, tags,
templates,
generated_file_hashes, import_definitions, media, media_translations, post_links,
post_media, post_translations, posts, scripts, tags, templates,
};
let post_ids = crate::db::queries::post::list_posts_by_project(conn, project_id)?
@@ -219,13 +458,6 @@ fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
generated_file_hashes::table.filter(generated_file_hashes::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(embedding_keys::table.filter(embedding_keys::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
import_definitions::table.filter(import_definitions::project_id.eq(project_id)),
)
@@ -261,6 +493,27 @@ mod tests {
use std::fs;
use tempfile::TempDir;
#[test]
fn progress_events_use_the_selected_ui_locale() {
assert_eq!(
RebuildProgress::LoadingProjectMetadata.localized(UiLocale::It),
"Caricamento dei metadati del progetto…"
);
assert_eq!(
RebuildProgress::PostItem {
current: 2,
total: 4,
name: "Bonjour".into(),
}
.localized(UiLocale::Fr),
"Articles : 2/4 — Bonjour"
);
assert_eq!(
RebuildProgress::Complete.localized(UiLocale::De),
"Neuaufbau abgeschlossen"
);
}
fn setup() -> (Database, TempDir) {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
@@ -308,6 +561,36 @@ mod tests {
assert!(report.errors.is_empty());
}
#[test]
fn cancellation_rolls_back_the_full_rebuild_savepoint() {
let (db, dir) = setup();
let post = crate::engine::post::create_post(
db.conn(),
dir.path(),
"p1",
"Preserved",
Some("body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
let result = rebuild_from_filesystem_with_progress(
db.conn(),
dir.path(),
"p1",
Some(Arc::new(|_, event| {
!matches!(event, RebuildProgress::ScanningPosts)
})),
);
assert!(matches!(result, Err(crate::engine::EngineError::Cancelled)));
assert!(qp::get_post_by_id(db.conn(), &post.id).is_ok());
}
#[test]
fn rebuild_creates_posts_and_media() {
let (db, dir) = setup();
@@ -587,6 +870,18 @@ function render() end
let project = crate::db::queries::project::get_project_by_id(db.conn(), "p1").unwrap();
assert_eq!(project.name, "Rebuilt Project");
assert!(project.description.is_none());
assert_eq!(
crate::engine::meta::read_categories_snapshot(db.conn(), "p1")
.unwrap()
.unwrap(),
vec!["article", "aside", "page", "picture"]
);
assert_eq!(
crate::engine::meta::read_publishing_snapshot(db.conn(), "p1")
.unwrap()
.unwrap(),
crate::model::PublishingPreferences::default()
);
let links =
crate::db::queries::post_media::list_post_media_by_media(db.conn(), "test-media-1")

View File

@@ -5,9 +5,10 @@ use crate::db::DbConnection as Connection;
use uuid::Uuid;
use crate::db::queries::script as qs;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Script, ScriptKind, ScriptStatus};
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::model::{DomainEntity, NotificationAction, Script, ScriptKind, ScriptStatus};
use crate::util::frontmatter::{ScriptFrontmatter, write_script_file};
use crate::util::paths::script_file_path;
use crate::util::{atomic_write_str, ensure_unique, now_unix_ms, slugify};
/// Create a new draft script. Content stored in DB, no file written.
@@ -53,6 +54,7 @@ pub fn create_script(
};
qs::insert_script(conn, &script)?;
emit_script(&script, NotificationAction::Created);
Ok(script)
}
@@ -63,6 +65,7 @@ pub fn create_script(
)]
pub fn update_script(
conn: &Connection,
data_dir: &Path,
script_id: &str,
project_id: &str,
title: Option<&str>,
@@ -73,23 +76,37 @@ pub fn update_script(
content: Option<&str>,
) -> EngineResult<Script> {
let mut script = qs::get_script_by_id(conn, script_id)?;
if script.project_id != project_id {
return Err(EngineError::NotFound(format!("script {script_id}")));
}
let original_slug = script.slug.clone();
let original_file_path = script.file_path.clone();
let was_published = script.status == ScriptStatus::Published;
let effective_content = script.content.clone().unwrap_or_else(|| {
if original_file_path.is_empty() {
String::new()
} else {
read_published_script_body(data_dir, &original_file_path)
}
});
let content_changed = content.is_some_and(|new_content| new_content != effective_content);
// Slug uniqueness
if let Some(new_slug) = slug
&& new_slug != script.slug
&& qs::get_script_by_slug(conn, project_id, new_slug).is_ok()
{
return Err(EngineError::Conflict(format!(
"script slug '{new_slug}' already exists"
)));
if let Some(requested_slug) = slug {
let slug = slugify(requested_slug);
let slug = if slug.is_empty() {
"script".to_string()
} else {
slug
};
script.slug = ensure_unique(&slug, |candidate| {
qs::get_script_by_slug(conn, project_id, candidate)
.is_ok_and(|existing| existing.id != script_id)
});
}
if let Some(t) = title {
script.title = t.to_string();
}
if let Some(s) = slug {
script.slug = s.to_string();
}
if let Some(k) = kind {
script.kind = k;
}
@@ -103,29 +120,62 @@ pub fn update_script(
script.content = Some(c.to_string());
}
// If published, transition back to draft on edit
if script.status == ScriptStatus::Published {
let slug_changed = script.slug != original_slug;
let published_body = if !original_file_path.is_empty()
&& (slug_changed || (was_published && !content_changed))
{
Some(effective_content)
} else {
None
};
if slug_changed && published_body.is_some() {
script.file_path = script_file_path(&script.slug);
}
if was_published && content_changed {
script.status = ScriptStatus::Draft;
}
script.version += 1;
script.updated_at = now_unix_ms();
qs::update_script(conn, &script)?;
if let Some(body) = published_body {
rewrite_renamed_script_file(data_dir, &original_file_path, &script, &body)?;
}
emit_script(&script, NotificationAction::Updated);
Ok(script)
}
/// Save script content (editor save). Updates DB, bumps version.
pub fn save_script(conn: &Connection, script_id: &str, content: &str) -> EngineResult<Script> {
pub fn save_script(
conn: &Connection,
data_dir: &Path,
script_id: &str,
content: &str,
) -> EngineResult<Script> {
let mut script = qs::get_script_by_id(conn, script_id)?;
let was_published = script.status == ScriptStatus::Published;
let effective_content = script.content.clone().unwrap_or_else(|| {
if script.file_path.is_empty() {
String::new()
} else {
read_published_script_body(data_dir, &script.file_path)
}
});
let content_changed = content != effective_content;
script.content = Some(content.to_string());
script.version += 1;
script.updated_at = now_unix_ms();
if script.status == ScriptStatus::Published {
if was_published && content_changed {
script.status = ScriptStatus::Draft;
}
qs::update_script(conn, &script)?;
if was_published && !content_changed && !script.file_path.is_empty() {
rewrite_renamed_script_file(data_dir, &script.file_path, &script, &effective_content)?;
}
emit_script(&script, NotificationAction::Updated);
Ok(script)
}
@@ -206,6 +256,8 @@ pub fn publish_script(conn: &Connection, data_dir: &Path, script_id: &str) -> En
script.updated_at = now;
qs::update_script(conn, &script)?;
emit_script(&script, NotificationAction::Updated);
Ok(script)
}
@@ -235,6 +287,8 @@ pub fn unpublish_script(
script.updated_at = now_unix_ms();
qs::update_script(conn, &script)?;
emit_script(&script, NotificationAction::Updated);
Ok(script)
}
@@ -251,9 +305,14 @@ pub fn delete_script(conn: &Connection, data_dir: &Path, script_id: &str) -> Eng
}
qs::delete_script(conn, script_id)?;
emit_script(&script, NotificationAction::Deleted);
Ok(())
}
fn emit_script(script: &Script, action: NotificationAction) {
domain_events::entity_changed(&script.project_id, DomainEntity::Script, &script.id, action);
}
// --- Helper functions ---
fn script_kind_to_frontmatter(kind: &ScriptKind) -> String {
@@ -264,6 +323,46 @@ fn script_kind_to_frontmatter(kind: &ScriptKind) -> String {
}
}
fn read_published_script_body(data_dir: &Path, file_path: &str) -> String {
fs::read_to_string(data_dir.join(file_path))
.ok()
.and_then(|file| crate::util::frontmatter::read_script_file(&file).ok())
.map(|(_, body)| body)
.unwrap_or_default()
}
fn rewrite_renamed_script_file(
data_dir: &Path,
original_file_path: &str,
script: &Script,
body: &str,
) -> EngineResult<()> {
let frontmatter = ScriptFrontmatter {
id: script.id.clone(),
project_id: Some(script.project_id.clone()),
slug: script.slug.clone(),
title: script.title.clone(),
kind: script_kind_to_frontmatter(&script.kind),
entrypoint: script.entrypoint.clone(),
enabled: script.enabled,
version: script.version,
created_at: script.created_at,
updated_at: script.updated_at,
};
atomic_write_str(
&data_dir.join(&script.file_path),
&write_script_file(&frontmatter, body),
)?;
if original_file_path != script.file_path {
match fs::remove_file(data_dir.join(original_file_path)) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error.into()),
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -327,10 +426,11 @@ mod tests {
#[test]
fn update_script_bumps_version() {
let (db, _dir) = setup();
let (db, dir) = setup();
let s = create_script(db.conn(), "p1", "S", ScriptKind::Macro, "old", None).unwrap();
let updated = update_script(
db.conn(),
dir.path(),
&s.id,
"p1",
Some("New"),
@@ -346,11 +446,189 @@ mod tests {
assert_eq!(updated.version, 2);
}
#[test]
fn update_script_slug_normalizes_and_uniquifies() {
let (db, dir) = setup();
create_script(db.conn(), "p1", "Alpha", ScriptKind::Utility, "", None).unwrap();
let script = create_script(db.conn(), "p1", "Beta", ScriptKind::Utility, "", None).unwrap();
let updated = update_script(
db.conn(),
dir.path(),
&script.id,
"p1",
None,
Some(" Alpha! "),
None,
None,
None,
None,
)
.unwrap();
assert_eq!(updated.slug, "alpha-2");
assert!(updated.file_path.is_empty());
let unchanged = update_script(
db.conn(),
dir.path(),
&script.id,
"p1",
None,
Some(" Alpha 2 "),
None,
None,
None,
None,
)
.unwrap();
assert_eq!(unchanged.slug, "alpha-2");
}
#[test]
fn update_published_script_slug_rewrites_and_renames_file() {
let (db, dir) = setup();
let script = create_script(
db.conn(),
"p1",
"Published Script",
ScriptKind::Utility,
"function main()\nend",
None,
)
.unwrap();
let published = publish_script(db.conn(), dir.path(), &script.id).unwrap();
let old_path = dir.path().join(&published.file_path);
let updated = update_script(
db.conn(),
dir.path(),
&script.id,
"p1",
None,
Some(" Renamed Script! "),
None,
None,
None,
None,
)
.unwrap();
assert_eq!(updated.slug, "renamed-script");
assert_eq!(updated.file_path, "scripts/renamed-script.lua");
assert!(!old_path.exists());
let new_file = fs::read_to_string(dir.path().join(&updated.file_path)).unwrap();
let (frontmatter, body) = crate::util::frontmatter::read_script_file(&new_file).unwrap();
assert_eq!(frontmatter.slug, "renamed-script");
assert_eq!(body, "function main()\nend");
assert_eq!(updated.status, ScriptStatus::Published);
}
#[test]
fn published_script_metadata_update_stays_published_and_rewrites_frontmatter() {
let (db, dir) = setup();
let script = create_script(
db.conn(),
"p1",
"Published Script",
ScriptKind::Utility,
"function main()\nend",
None,
)
.unwrap();
let published = publish_script(db.conn(), dir.path(), &script.id).unwrap();
let updated = update_script(
db.conn(),
dir.path(),
&script.id,
"p1",
Some("Renamed title"),
None,
Some(ScriptKind::Transform),
Some("transform"),
Some(false),
None,
)
.unwrap();
assert_eq!(updated.status, ScriptStatus::Published);
let file = fs::read_to_string(dir.path().join(&published.file_path)).unwrap();
let (frontmatter, body) = crate::util::frontmatter::read_script_file(&file).unwrap();
assert_eq!(frontmatter.title, "Renamed title");
assert_eq!(frontmatter.kind, "transform");
assert_eq!(frontmatter.entrypoint, "transform");
assert!(!frontmatter.enabled);
assert_eq!(frontmatter.version, updated.version);
assert_eq!(body, "function main()\nend");
}
#[test]
fn published_script_content_change_reopens_draft_and_preserves_published_file() {
let (db, dir) = setup();
let script = create_script(
db.conn(),
"p1",
"Published Script",
ScriptKind::Utility,
"function main()\n return 'old'\nend",
None,
)
.unwrap();
let published = publish_script(db.conn(), dir.path(), &script.id).unwrap();
let updated = update_script(
db.conn(),
dir.path(),
&script.id,
"p1",
None,
None,
None,
None,
None,
Some("function main()\n return 'new'\nend"),
)
.unwrap();
assert_eq!(updated.status, ScriptStatus::Draft);
assert_eq!(
updated.content.as_deref(),
Some("function main()\n return 'new'\nend")
);
let file = fs::read_to_string(dir.path().join(&published.file_path)).unwrap();
let (_, body) = crate::util::frontmatter::read_script_file(&file).unwrap();
assert_eq!(body, "function main()\n return 'old'\nend");
}
#[test]
fn identical_published_script_save_stays_published() {
let (db, dir) = setup();
let script = create_script(
db.conn(),
"p1",
"Published Script",
ScriptKind::Utility,
"function main()\nend",
None,
)
.unwrap();
let published = publish_script(db.conn(), dir.path(), &script.id).unwrap();
let saved = save_script(db.conn(), dir.path(), &script.id, "function main()\nend").unwrap();
assert_eq!(saved.status, ScriptStatus::Published);
let file = fs::read_to_string(dir.path().join(&published.file_path)).unwrap();
let (frontmatter, body) = crate::util::frontmatter::read_script_file(&file).unwrap();
assert_eq!(frontmatter.version, saved.version);
assert_eq!(body, "function main()\nend");
}
#[test]
fn save_script_updates_content() {
let (db, _dir) = setup();
let (db, dir) = setup();
let s = create_script(db.conn(), "p1", "S", ScriptKind::Macro, "old", None).unwrap();
let saved = save_script(db.conn(), &s.id, "new body").unwrap();
let saved = save_script(db.conn(), dir.path(), &s.id, "new body").unwrap();
assert_eq!(saved.content, Some("new body".to_string()));
assert_eq!(saved.version, 2);
}

View File

@@ -8,7 +8,6 @@ use crate::db::queries::script as qs;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Script, ScriptStatus};
use crate::util::frontmatter::read_script_file;
use crate::util::now_unix_ms;
/// Report returned by `rebuild_scripts_from_filesystem`.
#[derive(Debug, Default)]
@@ -18,6 +17,8 @@ pub struct ScriptRebuildReport {
pub errors: Vec<String>,
}
pub type ItemProgressFn = Box<dyn Fn(usize, usize, &str) -> bool + Send>;
/// Rebuild scripts from the filesystem into the database.
///
/// Walks the `scripts/` directory for `*.lua` files, parses each
@@ -27,6 +28,15 @@ pub fn rebuild_scripts_from_filesystem(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<ScriptRebuildReport> {
rebuild_scripts_from_filesystem_with_progress(conn, data_dir, project_id, None)
}
pub fn rebuild_scripts_from_filesystem_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
on_item: Option<ItemProgressFn>,
) -> EngineResult<ScriptRebuildReport> {
let mut report = ScriptRebuildReport::default();
let scripts_dir = data_dir.join("scripts");
@@ -35,17 +45,23 @@ pub fn rebuild_scripts_from_filesystem(
return Ok(report);
}
for entry in WalkDir::new(&scripts_dir)
let files = WalkDir::new(&scripts_dir)
.into_iter()
.filter_map(|e| e.ok())
{
.filter(|entry| entry.path().is_file())
.filter(|entry| entry.path().extension().and_then(|ext| ext.to_str()) == Some("lua"))
.collect::<Vec<_>>();
for (index, entry) in files.iter().enumerate() {
let path = entry.path();
if !path.is_file() {
continue;
}
let ext = path.extension().and_then(|e| e.to_str());
if ext != Some("lua") {
continue;
let name = path
.file_stem()
.and_then(|stem| stem.to_str())
.unwrap_or("?");
if let Some(ref callback) = on_item
&& !callback(index + 1, files.len(), name)
{
return Err(EngineError::Cancelled);
}
match rebuild_single_script(conn, data_dir, project_id, path) {
@@ -83,8 +99,6 @@ pub(crate) fn rebuild_single_script(
.to_string();
let kind = fm.kind.parse().map_err(EngineError::Parse)?;
let now = now_unix_ms();
// File exists on disk -> Published; content is None in DB
let status = ScriptStatus::Published;
@@ -101,7 +115,7 @@ pub(crate) fn rebuild_single_script(
script.status = status;
script.content = None;
script.created_at = fm.created_at;
script.updated_at = now;
script.updated_at = fm.updated_at;
qs::update_script(conn, &script)?;
Ok(false)
}
@@ -119,7 +133,7 @@ pub(crate) fn rebuild_single_script(
status,
content: None,
created_at: fm.created_at,
updated_at: now,
updated_at: fm.updated_at,
};
qs::insert_script(conn, &script)?;
Ok(true)
@@ -188,6 +202,47 @@ end
assert_eq!(script.file_path, "scripts/my-macro.lua");
}
#[test]
fn rebuild_defaults_missing_entrypoint_from_script_kind() {
let (db, dir) = setup();
let scripts_dir = dir.path().join("scripts");
fs::create_dir_all(&scripts_dir).unwrap();
for (id, kind, expected_entrypoint) in [
("legacy-macro", "macro", "render"),
("legacy-utility", "utility", "main"),
("legacy-transform", "transform", "main"),
] {
let content = format!(
"---\nid: \"{id}\"\nslug: \"{id}\"\ntitle: \"{id}\"\nkind: \"{kind}\"\nenabled: true\nversion: 1\ncreatedAt: \"2024-01-01T00:00:00.000Z\"\nupdatedAt: \"2024-01-01T00:00:00.000Z\"\n---\nfunction {expected_entrypoint}() end\n"
);
fs::write(scripts_dir.join(format!("{id}.lua")), content).unwrap();
}
let report = rebuild_scripts_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(report.created, 3);
assert!(report.errors.is_empty(), "errors: {:?}", report.errors);
assert_eq!(
qs::get_script_by_id(db.conn(), "legacy-macro")
.unwrap()
.entrypoint,
"render"
);
assert_eq!(
qs::get_script_by_id(db.conn(), "legacy-utility")
.unwrap()
.entrypoint,
"main"
);
assert_eq!(
qs::get_script_by_id(db.conn(), "legacy-transform")
.unwrap()
.entrypoint,
"main"
);
}
#[test]
fn rebuild_updates_existing() {
let (db, dir) = setup();
@@ -246,6 +301,7 @@ end
assert!(script.enabled);
assert_eq!(script.version, 5);
assert_eq!(script.status, ScriptStatus::Published);
assert_eq!(script.updated_at, 1_704_067_200_000);
assert!(script.content.is_none());
}

View File

@@ -1,5 +1,6 @@
//! Full-text search reindexing engine functions.
use std::fs;
use std::path::Path;
use crate::db::DbConnection as Connection;
@@ -9,11 +10,56 @@ use crate::db::queries::{
media as media_q, media_translation, post as post_q, post_translation, project as project_q,
setting,
};
use crate::engine::EngineResult;
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::util::frontmatter::{read_post_file, read_translation_file};
use crate::util::now_unix_ms;
const REBUILD_REQUIRED_SETTING: &str = "app.search-index-rebuild-required";
/// Deterministic offline language fallback used when no permitted AI endpoint
/// is configured. This intentionally mirrors the legacy application's small
/// heuristic; it is a notice-worthy fallback, not a language model.
pub fn detect_language(text: &str) -> &'static str {
let normalized = text.to_lowercase();
if normalized.trim().is_empty() {
"en"
} else if normalized.contains(['ä', 'ö', 'ü', 'ß']) {
"de"
} else if normalized.contains([
'à', 'â', 'ç', 'é', 'è', 'ê', 'ë', 'î', 'ï', 'ô', 'ù', 'û', 'ÿ', 'œ',
]) {
"fr"
} else if normalized.contains(['ñ', '¡', '¿']) {
"es"
} else {
detect_language_from_hints(&normalized)
}
}
fn detect_language_from_hints(text: &str) -> &'static str {
let padded = format!(" {text} ");
let scores = [
(
"de",
[" der ", " die ", " das ", " und ", " ist ", " nicht "],
),
("fr", [" le ", " la ", " les ", " et ", " est ", " pas "]),
("es", [" el ", " la ", " los ", " y ", " es ", " no "]),
];
scores
.into_iter()
.map(|(language, hints)| {
let score = hints
.into_iter()
.filter(|hint| padded.contains(hint))
.count();
(language, score)
})
.max_by_key(|(_, score)| *score)
.filter(|(_, score)| *score >= 2)
.map_or("en", |(language, _)| language)
}
/// Result of a full reindex operation.
pub struct ReindexReport {
pub posts_indexed: usize,
@@ -21,7 +67,7 @@ pub struct ReindexReport {
}
/// Per-item progress callback: (current_item, total_items, item_description).
pub type ItemProgressFn = Box<dyn Fn(usize, usize, &str) + Send>;
pub type ItemProgressFn = Box<dyn Fn(usize, usize, &str) -> bool + Send>;
/// Repair a missing or previously deployed FTS schema and report whether its
/// derived content still needs to be rebuilt.
@@ -57,6 +103,7 @@ pub fn prepare_search_index(conn: &Connection) -> EngineResult<bool> {
return Err(error.into());
}
}
domain_events::settings_changed(None, REBUILD_REQUIRED_SETTING);
Ok(has_content)
}
@@ -84,6 +131,7 @@ pub fn rebuild_search_index(
match result {
Ok(report) => {
conn.release_savepoint()?;
domain_events::settings_changed(None, REBUILD_REQUIRED_SETTING);
Ok(report)
}
Err(error) => {
@@ -93,6 +141,26 @@ pub fn rebuild_search_index(
}
}
/// Reindex one project without disturbing rows belonging to other projects.
pub fn reindex_project(
conn: &Connection,
project_id: &str,
on_item: Option<ItemProgressFn>,
) -> EngineResult<ReindexReport> {
let project = project_q::get_project_by_id(conn, project_id)?;
let total = post_q::count_posts_by_project(conn, project_id)? as usize
+ media_q::count_media_by_project(conn, project_id)? as usize;
let mut current = 0;
index_project(
conn,
project_id,
project.data_path.as_deref(),
total,
&mut current,
on_item.as_ref(),
)
}
fn index_all_projects(
conn: &Connection,
on_item: Option<&ItemProgressFn>,
@@ -112,81 +180,150 @@ fn index_all_projects(
};
for project in projects {
let main_language = project
.data_path
.as_deref()
.and_then(|path| crate::engine::meta::read_project_json(Path::new(path)).ok())
.and_then(|metadata| metadata.main_language)
.unwrap_or_else(|| "en".to_string());
for post in post_q::list_posts_by_project(conn, &project.id)? {
current += 1;
if let Some(callback) = on_item {
callback(current, total, &post.title);
}
let translations = post_translation::list_post_translations_by_post(conn, &post.id)?;
let translation_data = translations
.iter()
.map(|translation| fts::PostTranslationFts {
title: translation.title.clone(),
excerpt: translation.excerpt.clone(),
content: translation.content.clone(),
language: translation.language.clone(),
})
.collect::<Vec<_>>();
fts::index_post(
conn,
&post.id,
&post.title,
post.excerpt.as_deref(),
post.content.as_deref(),
&post.tags,
&post.categories,
&translation_data,
post.language.as_deref().unwrap_or(&main_language),
)?;
report.posts_indexed += 1;
}
for media in media_q::list_media_by_project(conn, &project.id)? {
current += 1;
if let Some(callback) = on_item {
callback(current, total, &media.original_name);
}
let translations =
media_translation::list_media_translations_by_media(conn, &media.id)?;
let translation_data = translations
.iter()
.map(|translation| fts::MediaTranslationFts {
title: translation.title.clone(),
alt: translation.alt.clone(),
caption: translation.caption.clone(),
language: translation.language.clone(),
})
.collect::<Vec<_>>();
fts::index_media(
conn,
&media.id,
media.title.as_deref(),
media.alt.as_deref(),
media.caption.as_deref(),
&media.original_name,
&media.tags,
&translation_data,
media.language.as_deref().unwrap_or(&main_language),
)?;
report.media_indexed += 1;
}
let indexed = index_project(
conn,
&project.id,
project.data_path.as_deref(),
total,
&mut current,
on_item,
)?;
report.posts_indexed += indexed.posts_indexed;
report.media_indexed += indexed.media_indexed;
}
Ok(report)
}
fn index_project(
conn: &Connection,
project_id: &str,
data_path: Option<&str>,
total: usize,
current: &mut usize,
on_item: Option<&ItemProgressFn>,
) -> EngineResult<ReindexReport> {
let data_dir = data_path.map(Path::new);
let main_language = data_dir
.and_then(|path| crate::engine::meta::read_project_json(path).ok())
.and_then(|metadata| metadata.main_language)
.unwrap_or_else(|| "en".to_string());
let mut report = ReindexReport {
posts_indexed: 0,
media_indexed: 0,
};
for post in post_q::list_posts_by_project(conn, project_id)? {
*current += 1;
if let Some(callback) = on_item
&& !callback(*current, total, &post.title)
{
return Err(EngineError::Cancelled);
}
let translations = post_translation::list_post_translations_by_post(conn, &post.id)?;
let translation_data = translations
.iter()
.map(|translation| {
Ok(fts::PostTranslationFts {
title: translation.title.clone(),
excerpt: translation.excerpt.clone(),
content: data_dir
.map(|dir| resolve_translation_fts_content(dir, translation))
.transpose()?
.flatten()
.or_else(|| translation.content.clone()),
language: translation.language.clone(),
})
})
.collect::<EngineResult<Vec<_>>>()?;
let content = data_dir
.map(|dir| resolve_post_fts_content(dir, &post))
.transpose()?
.flatten()
.or_else(|| post.content.clone());
fts::index_post(
conn,
&post.id,
&post.title,
post.excerpt.as_deref(),
content.as_deref(),
&post.tags,
&post.categories,
&translation_data,
post.language.as_deref().unwrap_or(&main_language),
)?;
report.posts_indexed += 1;
}
for media in media_q::list_media_by_project(conn, project_id)? {
*current += 1;
if let Some(callback) = on_item
&& !callback(*current, total, &media.original_name)
{
return Err(EngineError::Cancelled);
}
let translations = media_translation::list_media_translations_by_media(conn, &media.id)?;
let translation_data = translations
.iter()
.map(|translation| fts::MediaTranslationFts {
title: translation.title.clone(),
alt: translation.alt.clone(),
caption: translation.caption.clone(),
language: translation.language.clone(),
})
.collect::<Vec<_>>();
fts::index_media(
conn,
&media.id,
media.title.as_deref(),
media.alt.as_deref(),
media.caption.as_deref(),
&media.original_name,
&media.tags,
&translation_data,
media.language.as_deref().unwrap_or(&main_language),
)?;
report.media_indexed += 1;
}
Ok(report)
}
fn resolve_post_fts_content(
data_dir: &Path,
post: &crate::model::Post,
) -> EngineResult<Option<String>> {
if post.content.is_some() {
return Ok(post.content.clone());
}
if post.file_path.is_empty() {
return Ok(None);
}
let raw = fs::read_to_string(data_dir.join(&post.file_path))?;
let (_fm, body) = read_post_file(&raw).map_err(EngineError::Parse)?;
Ok(Some(body))
}
fn resolve_translation_fts_content(
data_dir: &Path,
translation: &crate::model::PostTranslation,
) -> EngineResult<Option<String>> {
if translation.content.is_some() {
return Ok(translation.content.clone());
}
if translation.file_path.is_empty() {
return Ok(None);
}
let raw = fs::read_to_string(data_dir.join(&translation.file_path))?;
let (_fm, body) = read_translation_file(&raw).map_err(EngineError::Parse)?;
Ok(Some(body))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::db::fts::ensure_fts_tables;
use crate::db::fts::{self, ensure_fts_tables};
use crate::engine;
fn setup() -> (Database, String) {
@@ -241,6 +378,80 @@ mod tests {
assert_eq!(results.len(), 1);
}
#[test]
fn cancellation_rolls_back_shared_index_rebuild() {
let (db, project_id) = setup();
let dir = tempfile::tempdir().unwrap();
engine::post::create_post(
db.conn(),
dir.path(),
&project_id,
"Cancel Me",
Some("body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
let result = rebuild_search_index(db.conn(), Some(Box::new(|_, _, _| false)));
assert!(matches!(result, Err(EngineError::Cancelled)));
}
#[test]
fn rebuild_indexes_published_post_and_translation_bodies_from_files() {
let db = Database::open_in_memory().unwrap();
let _ = db.migrate();
ensure_fts_tables(db.conn()).unwrap();
let dir = tempfile::tempdir().unwrap();
let project = engine::project::create_project(
db.conn(),
"Published Project",
Some(dir.path().to_str().unwrap()),
)
.unwrap();
let post = engine::post::create_post(
db.conn(),
dir.path(),
&project.id,
"Published Rebuild",
Some("distinctive platypusnova body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
engine::post::upsert_translation(
db.conn(),
dir.path(),
&post.id,
"de",
"Veröffentlichter Neuaufbau",
None,
Some("markantes schmetterlingskomet wort"),
)
.unwrap();
engine::post::publish_post(db.conn(), dir.path(), &post.id).unwrap();
fts::remove_post_from_index(db.conn(), &post.id).unwrap();
let report = reindex_project(db.conn(), &project.id, None).unwrap();
assert_eq!(report.posts_indexed, 1);
assert_eq!(
fts::search_posts(db.conn(), "platypusnova", "en").unwrap(),
vec![post.id.clone()]
);
assert_eq!(
fts::search_posts(db.conn(), "schmetterlingskomet", "de").unwrap(),
vec![post.id]
);
}
#[test]
fn shared_rebuild_indexes_every_project() {
let (db, first_project_id) = setup();
@@ -286,4 +497,40 @@ mod tests {
assert_eq!(report.posts_indexed, 2);
assert_eq!(results.len(), 2);
}
#[test]
fn project_reindex_leaves_other_project_rows_intact() {
let (db, first_project_id) = setup();
let first_dir = tempfile::tempdir().unwrap();
let second_dir = tempfile::tempdir().unwrap();
let second = engine::project::create_project(
db.conn(),
"Second Project",
Some(second_dir.path().to_str().unwrap()),
)
.unwrap();
for (project_id, data_dir, title) in [
(&first_project_id, first_dir.path(), "First Searchable"),
(&second.id, second_dir.path(), "Second Searchable"),
] {
engine::post::create_post(
db.conn(),
data_dir,
project_id,
title,
Some("body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
}
reindex_project(db.conn(), &first_project_id, None).unwrap();
let results = crate::db::fts::search_posts(db.conn(), "searchable", "en").unwrap();
assert_eq!(results.len(), 2);
}
}

View File

@@ -0,0 +1,185 @@
use std::collections::BTreeMap;
use crate::db::DbConnection as Connection;
use crate::db::queries::setting;
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::util::now_unix_ms;
pub const UI_LANGUAGE_KEY: &str = "ui.language";
pub const AIRPLANE_MODE_KEY: &str = "ai.airplane_mode_enabled";
pub const ONLINE_API_KEY: &str = "ai.endpoint.online.api_key";
pub const AIRPLANE_API_KEY: &str = "ai.endpoint.airplane.api_key";
const ONLINE_API_KEY_CONFIGURED: &str = "ai.endpoint.online.api_key_configured";
const AIRPLANE_API_KEY_CONFIGURED: &str = "ai.endpoint.airplane.api_key_configured";
const DEFAULTS: &[(&str, &str)] = &[
("editor.default_mode", "markdown"),
("editor.diff_view_style", "inline"),
("editor.wrap_long_lines", "true"),
("editor.hide_unchanged_regions", "false"),
("ai.endpoint.online.url", ""),
("ai.endpoint.online.model", ""),
("ai.endpoint.online.title_model", ""),
("ai.endpoint.online.image_model", ""),
("ai.endpoint.online.chat_supports_tools", ""),
("ai.endpoint.online.image_supports_vision", ""),
(ONLINE_API_KEY, ""),
("ai.endpoint.airplane.url", ""),
("ai.endpoint.airplane.model", ""),
("ai.endpoint.airplane.title_model", ""),
("ai.endpoint.airplane.image_model", ""),
("ai.endpoint.airplane.chat_supports_tools", ""),
("ai.endpoint.airplane.image_supports_vision", ""),
(AIRPLANE_API_KEY, ""),
("ai.default_model", ""),
("ai.title_model", ""),
("ai.image_model", ""),
("ai.system_prompt", ""),
("mcp.http.enabled", "false"),
("data.automatic_rebuild", "true"),
];
pub fn get(conn: &Connection, key: &str) -> EngineResult<Option<String>> {
match setting::get_setting_by_key(conn, key) {
Ok(value) => Ok(Some(value.value)),
Err(diesel::result::Error::NotFound) => Ok(None),
Err(error) => Err(EngineError::Db(error)),
}
}
pub fn get_effective(conn: &Connection, key: &str) -> EngineResult<Option<String>> {
if key == ONLINE_API_KEY || key == AIRPLANE_API_KEY {
let configured_key = if key == ONLINE_API_KEY {
ONLINE_API_KEY_CONFIGURED
} else {
AIRPLANE_API_KEY_CONFIGURED
};
return Ok(Some(
get(conn, configured_key)?
.filter(|value| value == "true")
.map_or_else(String::new, |_| "configured".to_string()),
));
}
if let Some(value) = get(conn, key)? {
return Ok(Some(value));
}
if key == UI_LANGUAGE_KEY {
return Ok(Some(crate::i18n::detect_os_locale().code().to_string()));
}
Ok(DEFAULTS
.iter()
.find(|(candidate, _)| *candidate == key)
.map(|(_, value)| (*value).to_string()))
}
pub fn list_effective(conn: &Connection) -> EngineResult<BTreeMap<String, String>> {
let mut values = DEFAULTS
.iter()
.map(|(key, value)| ((*key).to_string(), (*value).to_string()))
.collect::<BTreeMap<_, _>>();
values.insert(
UI_LANGUAGE_KEY.to_string(),
crate::i18n::detect_os_locale().code().to_string(),
);
for value in setting::list_all_settings(conn)? {
if !value.key.starts_with("app.")
&& !value.key.starts_with("project:")
&& value.key != ONLINE_API_KEY
&& value.key != ONLINE_API_KEY_CONFIGURED
&& value.key != AIRPLANE_API_KEY
&& value.key != AIRPLANE_API_KEY_CONFIGURED
{
values.insert(value.key, value.value);
}
}
if get(conn, ONLINE_API_KEY_CONFIGURED)?.as_deref() == Some("true") {
values.insert(ONLINE_API_KEY.to_string(), "configured".to_string());
}
if get(conn, AIRPLANE_API_KEY_CONFIGURED)?.as_deref() == Some("true") {
values.insert(AIRPLANE_API_KEY.to_string(), "configured".to_string());
}
Ok(values)
}
pub fn set(conn: &Connection, key: &str, value: &str) -> EngineResult<()> {
set_at(conn, key, value, now_unix_ms())
}
pub fn set_at(conn: &Connection, key: &str, value: &str, updated_at: i64) -> EngineResult<()> {
setting::set_setting_value(conn, key, value, updated_at)?;
domain_events::settings_changed(None, key);
Ok(())
}
pub fn ui_language(conn: &Connection) -> EngineResult<Option<String>> {
get(conn, UI_LANGUAGE_KEY)
}
pub fn airplane_mode(conn: &Connection) -> EngineResult<bool> {
Ok(get(conn, AIRPLANE_MODE_KEY)?.as_deref() != Some("false"))
}
pub fn set_airplane_mode(conn: &Connection, enabled: bool) -> EngineResult<()> {
set(
conn,
AIRPLANE_MODE_KEY,
if enabled { "true" } else { "false" },
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
#[test]
fn effective_defaults_exclude_removed_style_settings() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
let values = list_effective(db.conn()).unwrap();
let removed_prefix = ["style", "."].concat();
assert!(!values.keys().any(|key| key.starts_with(&removed_prefix)));
}
#[test]
fn airplane_mode_defaults_to_safe_mode_and_persists_changes() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("bds.db");
let db = Database::open(&path).unwrap();
db.migrate().unwrap();
assert!(airplane_mode(db.conn()).unwrap());
set_airplane_mode(db.conn(), false).unwrap();
drop(db);
let db = Database::open(&path).unwrap();
assert!(!airplane_mode(db.conn()).unwrap());
set_airplane_mode(db.conn(), true).unwrap();
drop(db);
let db = Database::open(&path).unwrap();
assert!(airplane_mode(db.conn()).unwrap());
}
#[test]
fn effective_settings_hide_internal_project_metadata_snapshots() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
setting::set_setting_value(
db.conn(),
"project:p1:categories",
r#"{"categories":["article"]}"#,
1,
)
.unwrap();
assert!(
!list_effective(db.conn())
.unwrap()
.contains_key("project:p1:categories")
);
}
}

View File

@@ -1,10 +1,7 @@
use std::fs;
use std::path::Path;
use crate::db::DbConnection as Connection;
use crate::engine::{EngineError, EngineResult};
use crate::render::{GeneratedWriteOutcome, write_generated_bytes};
use crate::engine::EngineResult;
#[derive(Debug, Clone, Copy)]
pub(crate) struct BundledSiteAsset {
@@ -231,6 +228,17 @@ const BUNDLED_SITE_ASSETS: &[BundledSiteAsset] = &[
},
];
pub(crate) fn bundled_site_asset(relative_path: &str) -> Option<&'static [u8]> {
BUNDLED_SITE_ASSETS
.iter()
.find(|asset| asset.relative_path == relative_path)
.map(|asset| asset.bytes)
}
pub(crate) fn bundled_site_assets() -> &'static [BundledSiteAsset] {
BUNDLED_SITE_ASSETS
}
pub(crate) fn copy_bundled_site_assets(project_dir: &Path) -> EngineResult<()> {
for asset in BUNDLED_SITE_ASSETS {
let target = project_dir.join(asset.relative_path);
@@ -245,29 +253,29 @@ pub(crate) fn copy_bundled_site_assets(project_dir: &Path) -> EngineResult<()> {
Ok(())
}
pub(crate) fn write_bundled_site_assets(
conn: &Connection,
output_dir: &Path,
project_id: &str,
report: &mut crate::engine::generation::GenerationReport,
) -> EngineResult<()> {
for asset in BUNDLED_SITE_ASSETS {
match write_generated_bytes(
conn,
output_dir,
project_id,
asset.relative_path,
asset.bytes,
)
.map_err(|error| EngineError::Parse(error.to_string()))?
{
GeneratedWriteOutcome::Written => {
report.written_paths.push(asset.relative_path.to_string())
}
GeneratedWriteOutcome::SkippedUnchanged => {
report.skipped_paths.push(asset.relative_path.to_string())
}
}
#[cfg(test)]
mod tests {
use super::bundled_site_asset;
fn bundled_text(path: &str) -> &'static str {
std::str::from_utf8(bundled_site_asset(path).unwrap()).unwrap()
}
#[test]
fn calendar_heatmap_marks_only_populated_periods_with_one_accent() {
let runtime = bundled_text("assets/calendar-runtime.js");
let styles = bundled_text("assets/bds.css");
assert!(runtime.contains("--blog-calendar-heat-strength"));
assert!(!runtime.contains("--blog-calendar-heat-hue"));
assert!(styles.contains(
"[data-vc-date]:not([data-blog-calendar-has-posts='true']) [data-vc-date-btn]"
));
assert!(styles.contains("background-color: transparent !important;"));
assert!(styles.contains("border-color: transparent !important;"));
assert!(styles.contains("color-mix(in srgb, var(--pico-primary"));
assert!(styles.contains("var(--blog-calendar-heat-strength"));
assert!(styles.contains("transparent) !important;"));
assert!(!styles.contains("hsl(var(--blog-calendar-heat-hue"));
}
Ok(())
}

View File

@@ -6,9 +6,9 @@ use uuid::Uuid;
use crate::db::queries::post as post_q;
use crate::db::queries::tag as tag_q;
use crate::engine::meta;
use crate::engine::{EngineError, EngineResult};
use crate::model::Tag;
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::model::metadata::TagEntry;
use crate::model::{DomainEntity, NotificationAction, Tag};
use crate::util::now_unix_ms;
/// Create a new tag. Case-insensitive duplicate check.
@@ -38,6 +38,7 @@ pub fn create_tag(
};
tag_q::insert_tag(conn, &tag)?;
rewrite_tags_json(conn, data_dir, project_id)?;
emit_tag(&tag, NotificationAction::Created);
Ok(tag)
}
@@ -73,10 +74,13 @@ pub fn update_tag(
tag.updated_at = now_unix_ms();
tag_q::update_tag(conn, &tag)?;
rewrite_tags_json(conn, data_dir, &tag.project_id)?;
emit_tag(&tag, NotificationAction::Updated);
Ok(())
}
/// Delete a tag: remove from all posts' tag arrays, delete from DB, rewrite tags.json.
/// Delete a tag: remove its exact name from posts, delete from DB, rewrite tags.json.
/// Tag entity lookup remains case-insensitive, but portable post tag arrays
/// follow bDS2 and Allium's exact string membership semantics.
pub fn delete_tag(
conn: &Connection,
data_dir: &Path,
@@ -90,10 +94,13 @@ pub fn delete_tag(
tag_q::delete_tag(conn, tag_id)?;
rewrite_tags_json(conn, data_dir, project_id)?;
flush_post_frontmatter(conn, data_dir, &modified)?;
emit_tag(&tag, NotificationAction::Deleted);
Ok(())
}
/// Rename a tag: update all posts' tag arrays, update tag in DB, rewrite tags.json.
/// Rename a tag: replace its exact name in posts, update the DB and tags.json.
/// Tag entity lookup remains case-insensitive, but portable post tag arrays
/// follow bDS2 and Allium's exact string membership semantics.
pub fn rename_tag(
conn: &Connection,
data_dir: &Path,
@@ -110,12 +117,12 @@ pub fn rename_tag(
let now = now_unix_ms();
let mut modified = Vec::new();
for mut post in posts {
if post.tags.iter().any(|t| t.eq_ignore_ascii_case(&old_name)) {
if post.tags.iter().any(|t| t == &old_name) {
post.tags = post
.tags
.into_iter()
.map(|t| {
if t.eq_ignore_ascii_case(&old_name) {
if t == old_name {
new_name.to_string()
} else {
t
@@ -133,11 +140,14 @@ pub fn rename_tag(
tag_q::update_tag(conn, &tag)?;
rewrite_tags_json(conn, data_dir, project_id)?;
flush_post_frontmatter(conn, data_dir, &modified)?;
emit_tag(&tag, NotificationAction::Updated);
Ok(())
}
/// Merge multiple source tags into one target tag.
/// For each source: update posts (remove source name, add target name if not present), delete source.
/// Source and target membership in portable post tag arrays is exact, matching
/// bDS2 and Allium; tag entity lookup and uniqueness remain case-insensitive.
pub fn merge_tags(
conn: &Connection,
data_dir: &Path,
@@ -149,6 +159,7 @@ pub fn merge_tags(
.map_err(|_| EngineError::NotFound(format!("target tag {target_id}")))?;
let mut all_modified = Vec::new();
let mut deleted_tags = Vec::new();
for &source_id in source_ids {
let source_tag = tag_q::get_tag_by_id(conn, source_id)
.map_err(|_| EngineError::NotFound(format!("source tag {source_id}")))?;
@@ -156,20 +167,12 @@ pub fn merge_tags(
let posts = post_q::list_posts_by_project(conn, project_id)?;
let now = now_unix_ms();
for mut post in posts {
let has_source = post
.tags
.iter()
.any(|t| t.eq_ignore_ascii_case(&source_tag.name));
let has_source = post.tags.iter().any(|t| t == &source_tag.name);
if has_source {
// Remove source tag name
post.tags
.retain(|t| !t.eq_ignore_ascii_case(&source_tag.name));
post.tags.retain(|t| t != &source_tag.name);
// Add target tag name if not already present
if !post
.tags
.iter()
.any(|t| t.eq_ignore_ascii_case(&target_tag.name))
{
if !post.tags.iter().any(|t| t == &target_tag.name) {
post.tags.push(target_tag.name.clone());
}
post.updated_at = now;
@@ -181,13 +184,22 @@ pub fn merge_tags(
}
tag_q::delete_tag(conn, source_id)?;
deleted_tags.push(source_tag);
}
rewrite_tags_json(conn, data_dir, project_id)?;
flush_post_frontmatter(conn, data_dir, &all_modified)?;
for tag in &deleted_tags {
emit_tag(tag, NotificationAction::Deleted);
}
emit_tag(&target_tag, NotificationAction::Updated);
Ok(())
}
fn emit_tag(tag: &Tag, action: NotificationAction) {
domain_events::entity_changed(&tag.project_id, DomainEntity::Tag, &tag.id, action);
}
/// Import tags from meta/tags.json into DB, preserving colors and properties.
/// Creates new tags or updates existing ones with file-based properties.
pub fn import_tags_from_file(
@@ -340,8 +352,8 @@ fn remove_tag_name_from_posts(
let now = now_unix_ms();
let mut modified = Vec::new();
for mut post in posts {
if post.tags.iter().any(|t| t.eq_ignore_ascii_case(tag_name)) {
post.tags.retain(|t| !t.eq_ignore_ascii_case(tag_name));
if post.tags.iter().any(|t| t == tag_name) {
post.tags.retain(|t| t != tag_name);
post.updated_at = now;
post_q::update_post(conn, &post)?;
modified.push(post.id.clone());
@@ -505,6 +517,57 @@ mod tests {
assert_eq!(all[0].name, "target");
}
#[test]
fn tag_mutations_match_post_tag_names_exactly() {
let (db, dir) = setup();
let renamed = create_tag(db.conn(), dir.path(), "p1", "rust", None).unwrap();
insert_post(
db.conn(),
&make_post("rename", "rename", vec!["Rust".into(), "rust".into()]),
)
.unwrap();
rename_tag(db.conn(), dir.path(), "p1", &renamed.id, "golang").unwrap();
assert_eq!(
post_q::get_post_by_id(db.conn(), "rename").unwrap().tags,
vec!["Rust", "golang"]
);
let source = create_tag(db.conn(), dir.path(), "p1", "source", None).unwrap();
let target = create_tag(db.conn(), dir.path(), "p1", "target", None).unwrap();
insert_post(
db.conn(),
&make_post(
"merge",
"merge",
vec!["SOURCE".into(), "source".into(), "Target".into()],
),
)
.unwrap();
merge_tags(db.conn(), dir.path(), "p1", &[&source.id], &target.id).unwrap();
assert_eq!(
post_q::get_post_by_id(db.conn(), "merge").unwrap().tags,
vec!["SOURCE", "Target", "target"]
);
let deleted = create_tag(db.conn(), dir.path(), "p1", "delete", None).unwrap();
insert_post(
db.conn(),
&make_post("delete", "delete", vec!["DELETE".into(), "delete".into()]),
)
.unwrap();
delete_tag(db.conn(), dir.path(), "p1", &deleted.id).unwrap();
assert_eq!(
post_q::get_post_by_id(db.conn(), "delete").unwrap().tags,
vec!["DELETE"]
);
}
#[test]
fn sync_tags_from_posts_creates_missing() {
let (db, _dir) = setup();

View File

@@ -1,7 +1,10 @@
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::Notify;
/// Unique task identifier.
pub type TaskId = u64;
@@ -25,6 +28,7 @@ pub struct TaskSnapshot {
pub status: TaskStatus,
pub progress: Option<f32>,
pub message: Option<String>,
pub cancellation_requested: bool,
pub created_at: Instant,
}
@@ -42,184 +46,340 @@ struct TaskEntry {
created_at: Instant,
finished_at: Option<Instant>,
last_progress_report: Option<Instant>,
worker_active: bool,
worker_started: bool,
}
/// Manages concurrent tasks with a max concurrency limit and FIFO queue.
pub struct TaskManager {
max_concurrent: usize,
next_id: Mutex<TaskId>,
tasks: Mutex<Vec<TaskEntry>>,
state: Mutex<TaskState>,
state_changed: Condvar,
async_changed: Notify,
}
struct TaskState {
next_id: TaskId,
tasks: HashMap<TaskId, TaskEntry>,
order: VecDeque<TaskId>,
pending: VecDeque<TaskId>,
worker_count: usize,
}
/// Capacity reservation held from asynchronous admission until the worker exits.
pub struct TaskWorker {
manager: Arc<TaskManager>,
task_id: TaskId,
}
impl Drop for TaskWorker {
fn drop(&mut self) {
self.manager.worker_exited(self.task_id);
}
}
impl TaskManager {
/// Create a new task manager with the given concurrency limit.
pub fn new(max_concurrent: usize) -> Self {
Self {
max_concurrent,
next_id: Mutex::new(1),
tasks: Mutex::new(Vec::new()),
max_concurrent: max_concurrent.max(1),
state: Mutex::new(TaskState {
next_id: 1,
tasks: HashMap::new(),
order: VecDeque::new(),
pending: VecDeque::new(),
worker_count: 0,
}),
state_changed: Condvar::new(),
async_changed: Notify::new(),
}
}
/// Submit a new task. Returns its unique identifier.
pub fn submit(&self, label: &str) -> TaskId {
let mut next = self.next_id.lock().unwrap();
let id = *next;
*next += 1;
self.submit_with_group(label, None, None)
}
let entry = TaskEntry {
fn submit_with_group(
&self,
label: &str,
group_id: Option<&str>,
group_name: Option<&str>,
) -> TaskId {
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
let id = state.next_id;
state.next_id += 1;
state.tasks.insert(
id,
label: label.to_owned(),
group_id: None,
group_name: None,
status: TaskStatus::Pending,
cancel_flag: Arc::new(AtomicBool::new(false)),
progress: None,
message: None,
created_at: Instant::now(),
finished_at: None,
last_progress_report: None,
};
let mut tasks = self.tasks.lock().unwrap();
tasks.push(entry);
// Auto-start if under capacity
let running = tasks
.iter()
.filter(|t| t.status == TaskStatus::Running)
.count();
if running < self.max_concurrent
&& let Some(t) = tasks
.iter_mut()
.find(|t| t.id == id && t.status == TaskStatus::Pending)
{
t.status = TaskStatus::Running;
}
TaskEntry {
id,
label: label.to_owned(),
group_id: group_id.map(str::to_owned),
group_name: group_name.map(str::to_owned),
status: TaskStatus::Pending,
cancel_flag: Arc::new(AtomicBool::new(false)),
progress: None,
message: None,
created_at: Instant::now(),
finished_at: None,
last_progress_report: None,
worker_active: false,
worker_started: false,
},
);
state.order.push_back(id);
state.pending.push_back(id);
Self::promote_next(&mut state, self.max_concurrent);
drop(state);
self.notify_changed();
id
}
/// Submit a new task within a group. Returns its unique identifier.
pub fn submit_grouped(&self, label: &str, group_id: &str, group_name: &str) -> TaskId {
let id = self.submit(label);
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == id) {
entry.group_id = Some(group_id.to_owned());
entry.group_name = Some(group_name.to_owned());
}
id
self.submit_with_group(label, Some(group_id), Some(group_name))
}
/// Block a worker until its task may run. Returns false if cancelled.
/// Wait synchronously for admission. Prefer [`Self::admit`] before spawning workers.
pub fn wait_until_runnable(&self, task_id: TaskId) -> bool {
let mut tasks = self.tasks.lock().unwrap();
let mut state = self.state.lock().unwrap();
loop {
match tasks
.iter()
.find(|task| task.id == task_id)
.map(|task| &task.status)
{
Some(TaskStatus::Running) => return true,
Some(TaskStatus::Pending) => tasks = self.state_changed.wait(tasks).unwrap(),
match state.tasks.get(&task_id).map(|task| &task.status) {
Some(TaskStatus::Running) => {
if let Some(task) = state.tasks.get_mut(&task_id) {
task.worker_started = true;
}
return true;
}
Some(TaskStatus::Pending) => state = self.state_changed.wait(state).unwrap(),
_ => return false,
}
}
}
/// Admit without occupying a blocking-pool thread while queued.
pub async fn admit(self: &Arc<Self>, task_id: TaskId) -> Option<TaskWorker> {
loop {
let notified = self.async_changed.notified();
{
let mut state = self.state.lock().unwrap();
match state.tasks.get_mut(&task_id) {
Some(task) if task.status == TaskStatus::Running => {
task.worker_started = true;
return Some(TaskWorker {
manager: Arc::clone(self),
task_id,
});
}
Some(task) if task.status == TaskStatus::Pending => {}
_ => return None,
}
}
notified.await;
}
}
/// Admit a non-Tokio worker while retaining capacity until its guard drops.
pub fn admit_blocking(self: &Arc<Self>, task_id: TaskId) -> Option<TaskWorker> {
if self.wait_until_runnable(task_id) {
Some(TaskWorker {
manager: Arc::clone(self),
task_id,
})
} else {
None
}
}
/// Mark a task as completed.
pub fn complete(&self, task_id: TaskId) {
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == task_id)
&& matches!(entry.status, TaskStatus::Running)
{
entry.status = TaskStatus::Completed;
entry.progress = Some(1.0);
entry.finished_at = Some(Instant::now());
}
Self::promote_next(&mut tasks, self.max_concurrent);
self.state_changed.notify_all();
let mut state = self.state.lock().unwrap();
let released = if let Some(entry) = state.tasks.get_mut(&task_id) {
if matches!(entry.status, TaskStatus::Running) {
entry.status = TaskStatus::Completed;
entry.progress = Some(1.0);
entry.finished_at = Some(Instant::now());
}
let released = entry.worker_active;
entry.worker_active = false;
entry.worker_started = false;
released
} else {
false
};
state.worker_count = state.worker_count.saturating_sub(usize::from(released));
Self::promote_next(&mut state, self.max_concurrent);
drop(state);
self.notify_changed();
}
/// Mark a task as failed with an error message.
pub fn fail(&self, task_id: TaskId, error: String) {
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == task_id)
&& matches!(entry.status, TaskStatus::Running)
{
entry.message = Some(error.clone());
entry.status = TaskStatus::Failed(error);
entry.finished_at = Some(Instant::now());
}
Self::promote_next(&mut tasks, self.max_concurrent);
self.state_changed.notify_all();
let mut state = self.state.lock().unwrap();
let released = if let Some(entry) = state.tasks.get_mut(&task_id) {
if matches!(entry.status, TaskStatus::Running) {
if entry.cancel_flag.load(Ordering::Acquire) {
entry.status = TaskStatus::Cancelled;
} else {
entry.message = Some(error.clone());
entry.status = TaskStatus::Failed(error);
}
entry.finished_at = Some(Instant::now());
}
let released = entry.worker_active;
entry.worker_active = false;
entry.worker_started = false;
released
} else {
false
};
state.worker_count = state.worker_count.saturating_sub(usize::from(released));
Self::promote_next(&mut state, self.max_concurrent);
drop(state);
self.notify_changed();
}
/// Cancel a task by setting its cancel flag and status.
pub fn cancel(&self, task_id: TaskId) {
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == task_id)
/// Cancel queued work immediately, or request a cooperative stop from a worker.
pub fn cancel(&self, task_id: TaskId) -> bool {
let mut state = self.state.lock().unwrap();
let mut cancelled = false;
let mut released = false;
if let Some(entry) = state.tasks.get_mut(&task_id)
&& matches!(entry.status, TaskStatus::Running | TaskStatus::Pending)
{
entry.cancel_flag.store(true, Ordering::Release);
entry.status = TaskStatus::Cancelled;
entry.finished_at = Some(Instant::now());
if !entry.worker_started {
if entry.worker_active {
entry.worker_active = false;
released = true;
}
entry.status = TaskStatus::Cancelled;
entry.finished_at = Some(Instant::now());
}
cancelled = true;
}
Self::promote_next(&mut tasks, self.max_concurrent);
self.state_changed.notify_all();
state.worker_count = state.worker_count.saturating_sub(usize::from(released));
Self::promote_next(&mut state, self.max_concurrent);
drop(state);
self.notify_changed();
cancelled
}
/// Cancel every active task in a group and release their workers.
pub fn cancel_group(&self, group_id: &str) {
let mut state = self.state.lock().unwrap();
let now = Instant::now();
let group_ids = state
.tasks
.values()
.filter(|task| {
task.group_id.as_deref() == Some(group_id)
&& matches!(task.status, TaskStatus::Running | TaskStatus::Pending)
})
.map(|task| task.id)
.collect::<Vec<_>>();
let mut released = 0;
for id in group_ids {
let entry = state.tasks.get_mut(&id).unwrap();
entry.cancel_flag.store(true, Ordering::Release);
if !entry.worker_started {
if entry.worker_active {
entry.worker_active = false;
released += 1;
}
entry.status = TaskStatus::Cancelled;
entry.finished_at = Some(now);
}
}
state.worker_count = state.worker_count.saturating_sub(released);
Self::promote_next(&mut state, self.max_concurrent);
drop(state);
self.notify_changed();
}
/// Return the group containing a task, if any.
pub fn group_id(&self, task_id: TaskId) -> Option<String> {
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state
.tasks
.get(&task_id)
.and_then(|task| task.group_id.clone())
}
/// Check whether a task has been cancelled.
pub fn is_cancelled(&self, task_id: TaskId) -> bool {
let tasks = self.tasks.lock().unwrap();
tasks
.iter()
.find(|t| t.id == task_id)
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state
.tasks
.get(&task_id)
.map(|t| t.cancel_flag.load(Ordering::Acquire))
.unwrap_or(false)
}
/// Shared cancellation flag for a worker owned by this task.
pub fn cancellation_flag(&self, task_id: TaskId) -> Option<Arc<AtomicBool>> {
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state
.tasks
.get(&task_id)
.map(|task| Arc::clone(&task.cancel_flag))
}
/// Return the current status of a task.
pub fn status(&self, task_id: TaskId) -> Option<TaskStatus> {
let tasks = self.tasks.lock().unwrap();
tasks
.iter()
.find(|t| t.id == task_id)
.map(|t| t.status.clone())
self.get(task_id).map(|task| task.status)
}
/// Count tasks that are still queued.
pub fn pending_count(&self) -> usize {
let tasks = self.tasks.lock().unwrap();
tasks
.iter()
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state
.tasks
.values()
.filter(|t| t.status == TaskStatus::Pending)
.count()
}
/// Count tasks that are currently running.
pub fn running_count(&self) -> usize {
let tasks = self.tasks.lock().unwrap();
tasks
.iter()
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state
.tasks
.values()
.filter(|t| t.status == TaskStatus::Running)
.count()
}
/// Remove finished tasks older than the configured retention period.
pub fn evict_expired(&self) {
let cutoff = Instant::now() - FINISHED_TASK_TTL;
let mut tasks = self.tasks.lock().unwrap();
tasks.retain(|task| {
task.finished_at
.is_none_or(|finished_at| finished_at > cutoff)
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
}
/// Remove every finished task while preserving running and queued work.
pub fn clear_completed(&self) {
let mut state = self.state.lock().unwrap();
Self::remove_where(&mut state, |task| task.status == TaskStatus::Completed);
}
/// Remove all terminal task results while preserving active work.
pub fn clear_finished(&self) {
let mut state = self.state.lock().unwrap();
Self::remove_where(&mut state, |task| {
task.finished_at.is_some() && !task.worker_active
});
}
/// Update progress for a running task. Throttled to at most once per 250ms.
pub fn report_progress(&self, task_id: TaskId, progress: Option<f32>, message: Option<String>) {
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == task_id)
let mut state = self.state.lock().unwrap();
if let Some(entry) = state.tasks.get_mut(&task_id)
&& entry.status == TaskStatus::Running
{
let now = Instant::now();
@@ -227,7 +387,7 @@ impl TaskManager {
Some(prev) => now.duration_since(prev).as_millis() >= PROGRESS_THROTTLE_MS as u128,
None => true,
};
if should_report {
if should_report || progress.is_some_and(|value| value >= 1.0) {
entry.progress = progress;
entry.message = message;
entry.last_progress_report = Some(now);
@@ -237,58 +397,166 @@ impl TaskManager {
/// Return the current progress of a task.
pub fn progress(&self, task_id: TaskId) -> Option<f32> {
let tasks = self.tasks.lock().unwrap();
tasks
.iter()
.find(|t| t.id == task_id)
.and_then(|t| t.progress)
self.get(task_id).and_then(|task| task.progress)
}
/// Return a snapshot of all tasks for UI display.
/// Return active tasks plus ten recent finished tasks for status surfaces.
pub fn snapshots(&self) -> Vec<TaskSnapshot> {
let tasks = self.tasks.lock().unwrap();
let mut snapshots = tasks
.iter()
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
let mut active = state
.tasks
.values()
.filter(|task| task.finished_at.is_none())
.chain(
tasks
.iter()
.rev()
.filter(|task| task.finished_at.is_some())
.take(RECENT_FINISHED_LIMIT),
)
.map(|task| TaskSnapshot {
id: task.id,
label: task.label.clone(),
group_id: task.group_id.clone(),
group_name: task.group_name.clone(),
status: task.status.clone(),
progress: task.progress,
message: task.message.clone(),
created_at: task.created_at,
})
.collect::<Vec<_>>();
snapshots.sort_by_key(|snapshot| snapshot.created_at);
snapshots
active.sort_by_key(|task| (task.status != TaskStatus::Running, task.created_at));
let active_groups = active
.iter()
.filter_map(|task| task.group_id.as_deref())
.collect::<HashSet<_>>();
let mut finished = state
.tasks
.values()
.filter(|task| task.finished_at.is_some())
.collect::<Vec<_>>();
finished.sort_by_key(|task| std::cmp::Reverse(task.finished_at));
let recent_ids = finished
.iter()
.take(RECENT_FINISHED_LIMIT)
.map(|task| task.id)
.collect::<HashSet<_>>();
active
.into_iter()
.chain(finished.into_iter().filter(|task| {
recent_ids.contains(&task.id)
|| task
.group_id
.as_deref()
.is_some_and(|group| active_groups.contains(group))
}))
.map(Self::snapshot)
.collect()
}
/// Promote the next queued task to running if capacity allows.
fn promote_next(tasks: &mut [TaskEntry], max_concurrent: usize) {
let running = tasks
/// Return one retained task by id.
pub fn get(&self, task_id: TaskId) -> Option<TaskSnapshot> {
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state.tasks.get(&task_id).map(Self::snapshot)
}
/// Return every retained task, newest first.
pub fn all(&self) -> Vec<TaskSnapshot> {
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state
.order
.iter()
.filter(|t| t.status == TaskStatus::Running)
.count();
if running < max_concurrent
&& let Some(t) = tasks.iter_mut().find(|t| t.status == TaskStatus::Pending)
{
t.status = TaskStatus::Running;
.rev()
.filter_map(|id| state.tasks.get(id).map(Self::snapshot))
.collect()
}
/// Return running tasks in start order.
pub fn running(&self) -> Vec<TaskSnapshot> {
let mut state = self.state.lock().unwrap();
Self::prune_expired(&mut state);
state
.order
.iter()
.filter_map(|id| state.tasks.get(id))
.filter(|task| task.status == TaskStatus::Running)
.map(Self::snapshot)
.collect()
}
fn worker_exited(&self, task_id: TaskId) {
let mut state = self.state.lock().unwrap();
let released = if let Some(task) = state.tasks.get_mut(&task_id) {
if task.cancel_flag.load(Ordering::Acquire) {
task.worker_started = false;
drop(state);
self.notify_changed();
return;
}
let released = task.worker_active;
task.worker_active = false;
task.worker_started = false;
released
} else {
false
};
state.worker_count = state.worker_count.saturating_sub(usize::from(released));
Self::promote_next(&mut state, self.max_concurrent);
drop(state);
self.notify_changed();
}
fn promote_next(state: &mut TaskState, max_concurrent: usize) {
while state.worker_count < max_concurrent {
let Some(id) = state.pending.pop_front() else {
break;
};
let Some(task) = state.tasks.get_mut(&id) else {
continue;
};
if task.status != TaskStatus::Pending {
continue;
}
task.status = TaskStatus::Running;
task.worker_active = true;
state.worker_count += 1;
}
}
fn snapshot(task: &TaskEntry) -> TaskSnapshot {
TaskSnapshot {
id: task.id,
label: task.label.clone(),
group_id: task.group_id.clone(),
group_name: task.group_name.clone(),
status: task.status.clone(),
progress: task.progress,
message: task.message.clone(),
cancellation_requested: task.cancel_flag.load(Ordering::Acquire),
created_at: task.created_at,
}
}
fn prune_expired(state: &mut TaskState) {
let cutoff = Instant::now() - FINISHED_TASK_TTL;
Self::remove_where(state, |task| {
task.finished_at
.is_some_and(|finished_at| finished_at <= cutoff)
&& !task.worker_active
});
}
fn remove_where(state: &mut TaskState, predicate: impl Fn(&TaskEntry) -> bool) {
let removed = state
.tasks
.values()
.filter(|task| predicate(task))
.map(|task| task.id)
.collect::<HashSet<_>>();
state.tasks.retain(|id, _| !removed.contains(id));
state.order.retain(|id| !removed.contains(id));
state.pending.retain(|id| !removed.contains(id));
}
fn notify_changed(&self) {
self.state_changed.notify_all();
self.async_changed.notify_waiters();
}
}
impl Default for TaskManager {
fn default() -> Self {
Self::new(3)
Self::new(
std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1),
)
}
}
@@ -309,6 +577,19 @@ mod tests {
assert_eq!(mgr.status(id), Some(TaskStatus::Running));
}
#[test]
fn default_uses_every_online_worker_like_bds2() {
let mgr = TaskManager::default();
let expected = std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1);
for index in 0..expected {
mgr.submit(&format!("task {index}"));
}
assert_eq!(mgr.running_count(), expected);
}
#[test]
fn max_concurrent_enforced() {
let mgr = TaskManager::new(3);
@@ -333,7 +614,7 @@ mod tests {
}
#[test]
fn cancel_sets_flag() {
fn cancelling_unstarted_task_settles_immediately() {
let mgr = TaskManager::default();
let id = mgr.submit("upload");
// Task is auto-started (Running)
@@ -343,6 +624,35 @@ mod tests {
assert_eq!(mgr.status(id), Some(TaskStatus::Cancelled));
}
#[tokio::test]
async fn cancelling_started_task_stays_running_until_worker_stops() {
let mgr = Arc::new(TaskManager::new(1));
let id = mgr.submit("upload");
let worker = mgr.admit(id).await.unwrap();
mgr.cancel(id);
let snapshot = mgr.get(id).unwrap();
assert_eq!(snapshot.status, TaskStatus::Running);
assert!(snapshot.cancellation_requested);
drop(worker);
mgr.fail(id, "operation cancelled".into());
assert_eq!(mgr.status(id), Some(TaskStatus::Cancelled));
}
#[tokio::test]
async fn completed_work_wins_a_late_cancellation_request() {
let mgr = Arc::new(TaskManager::new(1));
let id = mgr.submit("atomic update");
let worker = mgr.admit(id).await.unwrap();
mgr.cancel(id);
drop(worker);
mgr.complete(id);
assert_eq!(mgr.status(id), Some(TaskStatus::Completed));
}
#[test]
fn complete_and_fail() {
let mgr = TaskManager::default();
@@ -378,8 +688,12 @@ mod tests {
// After b fails: d promoted to running
{
let mut tasks = mgr.tasks.lock().unwrap();
for task in tasks.iter_mut().filter(|task| task.finished_at.is_some()) {
let mut state = mgr.state.lock().unwrap();
for task in state
.tasks
.values_mut()
.filter(|task| task.finished_at.is_some())
{
task.finished_at =
Some(Instant::now() - FINISHED_TASK_TTL - Duration::from_secs(1));
}
@@ -403,6 +717,45 @@ mod tests {
assert_eq!(mgr.snapshots().len(), 10);
}
#[test]
fn snapshots_retain_every_active_task_and_complete_active_groups() {
let mgr = TaskManager::new(20);
let finished_group_member = mgr.submit_grouped("finished", "group", "Group");
mgr.complete(finished_group_member);
for index in 0..12 {
mgr.submit_grouped(&format!("active {index}"), "group", "Group");
}
for index in 0..12 {
let id = mgr.submit(&format!("history {index}"));
mgr.complete(id);
}
let snapshots = mgr.snapshots();
assert_eq!(
snapshots
.iter()
.filter(|task| matches!(task.status, TaskStatus::Pending | TaskStatus::Running))
.count(),
12
);
assert!(
snapshots
.iter()
.any(|task| task.id == finished_group_member)
);
}
#[test]
fn clear_completed_preserves_active_tasks() {
let mgr = TaskManager::new(2);
let done = mgr.submit("done");
let running = mgr.submit("running");
mgr.complete(done);
mgr.clear_completed();
assert_eq!(mgr.status(done), None);
assert_eq!(mgr.status(running), Some(TaskStatus::Running));
}
#[test]
fn completing_task_starts_next_queued() {
let mgr = TaskManager::new(1);
@@ -454,6 +807,22 @@ mod tests {
assert_eq!(mgr.status(id), Some(TaskStatus::Completed));
}
#[test]
fn cancelling_group_settles_every_active_task() {
let mgr = TaskManager::new(2);
let first = mgr.submit_grouped("first", "generation-1", "Render Site");
let second = mgr.submit_grouped("second", "generation-1", "Render Site");
let third = mgr.submit_grouped("third", "generation-1", "Render Site");
let unrelated = mgr.submit("unrelated");
mgr.cancel_group("generation-1");
assert_eq!(mgr.status(first), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(second), Some(TaskStatus::Cancelled));
assert_eq!(mgr.status(third), Some(TaskStatus::Cancelled));
assert_ne!(mgr.status(unrelated), Some(TaskStatus::Cancelled));
}
#[test]
fn report_progress_updates_task() {
let mgr = TaskManager::default();
@@ -462,4 +831,70 @@ mod tests {
assert_eq!(mgr.progress(id), Some(0.5));
assert_eq!(mgr.snapshots()[0].message.as_deref(), Some("halfway"));
}
#[tokio::test]
async fn cancellation_holds_capacity_until_worker_exits() {
let mgr = Arc::new(TaskManager::new(1));
let running = mgr.submit("running");
let queued = mgr.submit("queued");
let worker = mgr.admit(running).await.unwrap();
mgr.cancel(running);
assert_eq!(mgr.status(running), Some(TaskStatus::Running));
assert!(mgr.get(running).unwrap().cancellation_requested);
assert_eq!(mgr.status(queued), Some(TaskStatus::Pending));
drop(worker);
assert_eq!(mgr.status(queued), Some(TaskStatus::Pending));
mgr.fail(running, "operation cancelled".into());
assert_eq!(mgr.status(queued), Some(TaskStatus::Running));
}
#[tokio::test]
async fn panicking_worker_guard_releases_capacity() {
let mgr = Arc::new(TaskManager::new(1));
let panicking = mgr.submit("panicking");
let queued = mgr.submit("queued");
let worker = mgr.admit(panicking).await.unwrap();
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _worker = worker;
panic!("boom");
}));
assert_eq!(mgr.status(queued), Some(TaskStatus::Running));
}
#[test]
fn terminal_progress_bypasses_throttle() {
let mgr = TaskManager::new(1);
let id = mgr.submit("work");
mgr.report_progress(id, Some(0.5), Some("working".into()));
mgr.report_progress(id, Some(1.0), Some("done".into()));
let task = mgr
.snapshots()
.into_iter()
.find(|task| task.id == id)
.unwrap();
assert_eq!(task.progress, Some(1.0));
assert_eq!(task.message.as_deref(), Some("done"));
}
#[test]
fn clear_completed_keeps_other_terminal_results() {
let mgr = TaskManager::new(3);
let completed = mgr.submit("completed");
let failed = mgr.submit("failed");
let cancelled = mgr.submit("cancelled");
mgr.complete(completed);
mgr.fail(failed, "failed".into());
mgr.cancel(cancelled);
mgr.clear_completed();
assert_eq!(mgr.status(completed), None);
assert!(matches!(mgr.status(failed), Some(TaskStatus::Failed(_))));
assert_eq!(mgr.status(cancelled), Some(TaskStatus::Cancelled));
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -8,7 +8,6 @@ use crate::db::queries::template as qt;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Template, TemplateStatus};
use crate::util::frontmatter::read_template_file;
use crate::util::now_unix_ms;
/// Report returned by `rebuild_templates_from_filesystem`.
#[derive(Debug, Default)]
@@ -18,6 +17,8 @@ pub struct TemplateRebuildReport {
pub errors: Vec<String>,
}
pub type ItemProgressFn = Box<dyn Fn(usize, usize, &str) -> bool + Send>;
/// Rebuild templates from the filesystem into the database.
///
/// Walks the `templates/` directory for `*.liquid` files, parses each via
@@ -27,6 +28,15 @@ pub fn rebuild_templates_from_filesystem(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<TemplateRebuildReport> {
rebuild_templates_from_filesystem_with_progress(conn, data_dir, project_id, None)
}
pub fn rebuild_templates_from_filesystem_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
on_item: Option<ItemProgressFn>,
) -> EngineResult<TemplateRebuildReport> {
let mut report = TemplateRebuildReport::default();
let templates_dir = data_dir.join("templates");
@@ -35,17 +45,23 @@ pub fn rebuild_templates_from_filesystem(
return Ok(report);
}
for entry in WalkDir::new(&templates_dir)
let files = WalkDir::new(&templates_dir)
.into_iter()
.filter_map(|e| e.ok())
{
.filter(|entry| entry.path().is_file())
.filter(|entry| entry.path().extension().and_then(|ext| ext.to_str()) == Some("liquid"))
.collect::<Vec<_>>();
for (index, entry) in files.iter().enumerate() {
let path = entry.path();
if !path.is_file() {
continue;
}
let ext = path.extension().and_then(|e| e.to_str());
if ext != Some("liquid") {
continue;
let name = path
.file_stem()
.and_then(|stem| stem.to_str())
.unwrap_or("?");
if let Some(ref callback) = on_item
&& !callback(index + 1, files.len(), name)
{
return Err(EngineError::Cancelled);
}
match rebuild_single_template(conn, data_dir, project_id, path) {
@@ -83,8 +99,6 @@ pub(crate) fn rebuild_single_template(
.to_string();
let kind = fm.kind.parse().map_err(EngineError::Parse)?;
let now = now_unix_ms();
// File exists on disk -> Published; content is None in DB
let status = TemplateStatus::Published;
@@ -100,7 +114,7 @@ pub(crate) fn rebuild_single_template(
tpl.status = status;
tpl.content = None;
tpl.created_at = fm.created_at;
tpl.updated_at = now;
tpl.updated_at = fm.updated_at;
qt::update_template(conn, &tpl)?;
Ok(false)
}
@@ -117,7 +131,7 @@ pub(crate) fn rebuild_single_template(
status,
content: None,
created_at: fm.created_at,
updated_at: now,
updated_at: fm.updated_at,
};
qt::insert_template(conn, &tpl)?;
Ok(true)
@@ -236,6 +250,7 @@ updatedAt: \"2024-01-01T00:00:00.000Z\"
assert_eq!(tpl.version, 3);
assert_eq!(tpl.status, TemplateStatus::Published);
assert!(tpl.content.is_none());
assert_eq!(tpl.updated_at, 1_704_067_200_000);
}
#[test]

View File

@@ -0,0 +1,146 @@
use serde::{Deserialize, Serialize};
use crate::db::DbConnection as Connection;
use crate::db::queries::{project, setting};
use crate::engine::{EngineError, EngineResult};
use crate::util::now_unix_ms;
const KEY_SUFFIX: &str = "ui_state";
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct ProjectUiState {
pub sidebar_view: String,
pub sidebar_visible: bool,
pub sidebar_width: f32,
pub panel_visible: bool,
pub panel_tab: String,
pub tabs: Vec<PersistedTab>,
pub active_tab: Option<String>,
}
impl Default for ProjectUiState {
fn default() -> Self {
Self {
sidebar_view: "posts".to_string(),
sidebar_visible: true,
sidebar_width: 280.0,
panel_visible: false,
panel_tab: "tasks".to_string(),
tabs: Vec::new(),
active_tab: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PersistedTab {
pub tab_type: String,
pub id: String,
pub title: String,
pub is_transient: bool,
}
pub fn load(conn: &Connection, project_id: &str) -> EngineResult<Option<ProjectUiState>> {
match setting::get_setting_by_key(conn, &key(project_id)) {
Ok(setting) => Ok(Some(serde_json::from_str(&setting.value)?)),
Err(diesel::result::Error::NotFound) => Ok(None),
Err(error) => Err(EngineError::Db(error)),
}
}
pub fn save(conn: &Connection, project_id: &str, state: &ProjectUiState) -> EngineResult<()> {
match project::get_project_by_id(conn, project_id) {
Ok(_) => {}
Err(diesel::result::Error::NotFound) => {
return Err(EngineError::NotFound(format!("project {project_id}")));
}
Err(error) => return Err(EngineError::Db(error)),
}
let serialized = serde_json::to_string(state)?;
setting::set_setting_value(conn, &key(project_id), &serialized, now_unix_ms())?;
Ok(())
}
fn key(project_id: &str) -> String {
format!("project:{project_id}:{KEY_SUFFIX}")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::db::queries::project::{insert_project, make_test_project};
fn setup() -> Database {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
insert_project(db.conn(), &make_test_project("p1", "one")).unwrap();
insert_project(db.conn(), &make_test_project("p2", "two")).unwrap();
db
}
#[test]
fn round_trips_independent_project_sessions() {
let db = setup();
let one = ProjectUiState {
sidebar_view: "media".into(),
sidebar_visible: false,
sidebar_width: 412.0,
panel_visible: true,
panel_tab: "output".into(),
tabs: vec![PersistedTab {
tab_type: "post".into(),
id: "post-1".into(),
title: "Post one".into(),
is_transient: false,
}],
active_tab: Some("post-1".into()),
};
let two = ProjectUiState {
sidebar_view: "scripts".into(),
tabs: vec![PersistedTab {
tab_type: "scripts".into(),
id: "script-2".into(),
title: "Script two".into(),
is_transient: true,
}],
active_tab: Some("script-2".into()),
..ProjectUiState::default()
};
save(db.conn(), "p1", &one).unwrap();
save(db.conn(), "p2", &two).unwrap();
assert_eq!(load(db.conn(), "p1").unwrap(), Some(one));
assert_eq!(load(db.conn(), "p2").unwrap(), Some(two));
assert_eq!(load(db.conn(), "missing").unwrap(), None);
}
#[test]
fn missing_newer_fields_use_safe_defaults() {
let db = setup();
setting::set_setting_value(db.conn(), &key("p1"), r#"{"sidebar_view":"media"}"#, 1)
.unwrap();
let state = load(db.conn(), "p1").unwrap().unwrap();
assert_eq!(state.sidebar_view, "media");
assert!(state.sidebar_visible);
assert_eq!(state.sidebar_width, 280.0);
assert!(state.tabs.is_empty());
}
#[test]
fn deleting_a_project_removes_its_session() {
let db = setup();
save(db.conn(), "p1", &ProjectUiState::default()).unwrap();
crate::engine::project::delete_project(db.conn(), "p1", None).unwrap();
assert_eq!(load(db.conn(), "p1").unwrap(), None);
assert!(matches!(
save(db.conn(), "p1", &ProjectUiState::default()),
Err(EngineError::NotFound(_))
));
}
}

View File

@@ -10,7 +10,12 @@ use crate::model::Media;
use crate::util::{media_sidecar_path, thumbnail_path};
/// Thumbnail sizes per media_processing.allium.
const THUMBNAIL_SIZES: &[&str] = &["small", "medium", "large", "ai"];
const THUMBNAIL_VARIANTS: &[(&str, &str)] = &[
("small", "webp"),
("medium", "webp"),
("large", "webp"),
("ai", "jpg"),
];
/// Types of media validation issues.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -125,8 +130,7 @@ fn check_media_item(
// 3. Missing thumbnails — only for image types
if is_image_mime(&media.mime_type) {
let ext = thumbnail_extension(&media.mime_type);
for size in THUMBNAIL_SIZES {
for (size, ext) in THUMBNAIL_VARIANTS {
let thumb_rel = thumbnail_path(&media.id, size, ext);
let thumb_path = data_dir.join(&thumb_rel);
if !thumb_path.exists() {
@@ -160,15 +164,6 @@ fn is_image_mime(mime: &str) -> bool {
mime.starts_with("image/")
}
fn thumbnail_extension(mime: &str) -> &str {
match mime {
"image/png" => "png",
"image/gif" => "gif",
"image/webp" => "webp",
_ => "jpg",
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -182,10 +177,15 @@ mod tests {
}
#[test]
fn thumbnail_ext_defaults_to_jpg() {
assert_eq!(thumbnail_extension("image/jpeg"), "jpg");
assert_eq!(thumbnail_extension("image/png"), "png");
assert_eq!(thumbnail_extension("image/webp"), "webp");
assert_eq!(thumbnail_extension("image/tiff"), "jpg"); // fallback
fn thumbnail_variants_match_the_generator_formats() {
assert_eq!(
THUMBNAIL_VARIANTS,
&[
("small", "webp"),
("medium", "webp"),
("large", "webp"),
("ai", "jpg")
]
);
}
}

View File

@@ -1,14 +1,17 @@
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use crate::db::DbConnection as Connection;
use walkdir::WalkDir;
use crate::db::queries;
use crate::engine::generation::has_published_snapshot;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Post, PostStatus};
use crate::render::build_site_render_artifacts;
use crate::util::file_hash;
use crate::model::Post;
use crate::render::{build_canonical_post_path, build_site_route_manifest};
const MTIME_GRANULARITY_TOLERANCE_MS: i64 = 1_000;
#[derive(Debug, Clone, Default)]
pub struct SiteValidationReport {
@@ -22,82 +25,89 @@ pub fn validate_site(
data_dir: &Path,
project_id: &str,
) -> EngineResult<SiteValidationReport> {
validate_site_with_progress(conn, data_dir, project_id, |_, _| true)
}
pub fn validate_site_with_progress(
conn: &Connection,
data_dir: &Path,
project_id: &str,
mut on_progress: impl FnMut(usize, usize) -> bool,
) -> EngineResult<SiteValidationReport> {
const PHASES: usize = 4;
if !on_progress(0, PHASES) {
return Err(EngineError::Cancelled);
}
let metadata = crate::engine::meta::read_project_json(data_dir)?;
let output_dir = generated_output_dir(data_dir);
let published_posts = load_published_posts(data_dir, conn, project_id)?;
let artifacts =
build_site_render_artifacts(conn, data_dir, project_id, &metadata, &published_posts)
.map_err(|error| EngineError::Parse(error.to_string()))?;
let mut expected = artifacts
.pages
.iter()
.map(|page| page.relative_path.clone())
let published_posts = load_published_posts(conn, project_id)?;
let route_manifest = build_site_route_manifest(data_dir, &metadata, &published_posts)
.map_err(|error| crate::engine::EngineError::Parse(error.to_string()))?;
if !on_progress(1, PHASES) {
return Err(EngineError::Cancelled);
}
crate::engine::generation::refresh_validation_sitemap(
conn,
&output_dir,
project_id,
data_dir,
&metadata,
&published_posts,
&route_manifest,
)?;
let expected = route_manifest
.into_iter()
.map(|page| page.relative_path)
.collect::<HashSet<_>>();
expected.insert("calendar.json".to_string());
expected.insert("rss.xml".to_string());
for language in render_languages(&metadata) {
let prefix = if language
== metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string())
{
String::new()
} else {
format!("{language}/")
};
expected.insert(format!("{prefix}feed.xml"));
expected.insert(format!("{prefix}atom.xml"));
expected.insert(format!("{prefix}sitemap.xml"));
if !on_progress(2, PHASES) {
return Err(EngineError::Cancelled);
}
let mut actual = HashSet::new();
let mut zero_byte = HashSet::new();
if output_dir.exists() {
for entry in WalkDir::new(&output_dir).into_iter().filter_map(Result::ok) {
if !entry.file_type().is_file() {
if !entry.file_type().is_file() || entry.file_name() != "index.html" {
continue;
}
let rel = entry
.path()
.strip_prefix(&output_dir)
.unwrap_or(entry.path())
.to_string_lossy()
.replace('\\', "/");
if rel.starts_with("meta/")
|| rel.starts_with("posts/")
|| rel.starts_with("media/")
|| rel.starts_with("assets/")
{
continue;
}
if rel.starts_with("pagefind") || rel.contains("/pagefind/") {
continue;
}
if rel.ends_with(".html") || rel.ends_with(".xml") || rel.ends_with(".json") {
actual.insert(rel);
let path = relative_path(&output_dir, entry.path());
if entry.metadata().is_ok_and(|metadata| metadata.len() > 0) {
actual.insert(path);
} else {
zero_byte.insert(path);
}
}
}
let mut missing_pages = expected.difference(&actual).cloned().collect::<Vec<_>>();
let mut extra_pages = actual.difference(&expected).cloned().collect::<Vec<_>>();
let mut stale_pages = Vec::new();
for rel in expected.intersection(&actual) {
if let Ok(stored) =
queries::generated_file_hash::get_generated_file_hash(conn, project_id, rel)
{
let actual_hash = file_hash(&output_dir.join(rel))?;
if actual_hash != stored.content_hash {
stale_pages.push(rel.clone());
}
}
extra_pages.extend(zero_byte.difference(&expected).cloned());
let generated_at = queries::generated_file_hash::list_generated_file_hashes(conn, project_id)?
.into_iter()
.map(|file| (file.relative_path, file.updated_at))
.collect::<HashMap<_, _>>();
if !on_progress(3, PHASES) {
return Err(EngineError::Cancelled);
}
let mut stale_pages = stale_post_paths(
data_dir,
&output_dir,
&metadata,
&published_posts,
&expected,
&actual,
&generated_at,
);
missing_pages.sort();
extra_pages.sort();
extra_pages.dedup();
stale_pages.sort();
stale_pages.dedup();
if !on_progress(PHASES, PHASES) {
return Err(EngineError::Cancelled);
}
Ok(SiteValidationReport {
missing_pages,
@@ -106,6 +116,77 @@ pub fn validate_site(
})
}
fn stale_post_paths(
data_dir: &Path,
output_dir: &Path,
metadata: &crate::model::ProjectMetadata,
published_posts: &[Post],
expected: &HashSet<String>,
actual: &HashSet<String>,
generated_at: &HashMap<String, i64>,
) -> Vec<String> {
let main_language = metadata.main_language.as_deref().unwrap_or("en");
let mut languages = vec![main_language.to_string()];
for language in &metadata.blog_languages {
if !languages
.iter()
.any(|known| known.eq_ignore_ascii_case(language))
{
languages.push(language.clone());
}
}
let mut stale = Vec::new();
for post in published_posts {
let Some(source_modified) = modified_ms(&data_dir.join(&post.file_path)) else {
continue;
};
for language in &languages {
if language != main_language && post.do_not_translate {
continue;
}
let relative_path = format!(
"{}/index.html",
build_canonical_post_path(post, language, main_language).trim_start_matches('/')
);
if !expected.contains(&relative_path) || !actual.contains(&relative_path) {
continue;
}
let Some(output_modified) = modified_ms(&output_dir.join(&relative_path)) else {
continue;
};
let effective_generated = output_modified.max(
generated_at
.get(&relative_path)
.copied()
.unwrap_or_default(),
);
if source_modified > effective_generated + MTIME_GRANULARITY_TOLERANCE_MS {
stale.push(relative_path);
}
}
}
stale
}
fn modified_ms(path: &Path) -> Option<i64> {
let modified = path.metadata().ok()?.modified().ok()?;
Some(system_time_ms(modified))
}
fn system_time_ms(time: SystemTime) -> i64 {
time.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_millis() as i64)
.unwrap_or_default()
}
fn relative_path(root: &Path, path: &Path) -> String {
path.strip_prefix(root)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/")
}
fn generated_output_dir(data_dir: &Path) -> std::path::PathBuf {
let html_dir = data_dir.join("html");
if html_dir.exists() {
@@ -115,46 +196,9 @@ fn generated_output_dir(data_dir: &Path) -> std::path::PathBuf {
}
}
fn load_published_posts(
data_dir: &Path,
conn: &Connection,
project_id: &str,
) -> EngineResult<Vec<(Post, String)>> {
let posts = queries::post::list_posts_by_project(conn, project_id)?;
let mut published = Vec::new();
for post in posts
fn load_published_posts(conn: &Connection, project_id: &str) -> EngineResult<Vec<Post>> {
Ok(queries::post::list_posts_by_project(conn, project_id)?
.into_iter()
.filter(|post| post.status == PostStatus::Published)
{
let body = if let Some(content) = &post.content {
content.clone()
} else if let Some(content) = &post.published_content {
content.clone()
} else {
let raw =
std::fs::read_to_string(data_dir.join(post.file_path.trim_start_matches('/')))?;
crate::util::frontmatter::read_post_file(&raw)
.map(|(_, body)| body)
.map_err(EngineError::Parse)?
};
published.push((post, body));
}
Ok(published)
}
fn render_languages(metadata: &crate::model::ProjectMetadata) -> Vec<String> {
let main = metadata
.main_language
.clone()
.unwrap_or_else(|| "en".to_string());
let mut languages = vec![main.clone()];
for language in &metadata.blog_languages {
if !languages
.iter()
.any(|existing| existing.eq_ignore_ascii_case(language))
{
languages.push(language.clone());
}
}
languages
.filter(has_published_snapshot)
.collect())
}

Some files were not shown because too many files have changed in this diff Show More