diff --git a/Cargo.lock b/Cargo.lock index d2e3f03..aaac98c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -621,6 +621,20 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clipboard-win" version = "5.4.1" @@ -1059,6 +1073,27 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "dispatch" version = "0.2.0" @@ -1144,6 +1179,7 @@ dependencies = [ "serde_norway", "sha2", "time", + "turbovault-parser", "ureq", "url", ] @@ -2856,6 +2892,30 @@ dependencies = [ "typenum", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.62.2", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "iced" version = "0.14.0" @@ -3034,7 +3094,7 @@ dependencies = [ "iced_renderer", "log", "num-traits", - "pulldown-cmark", + "pulldown-cmark 0.12.2", "rustc-hash 2.1.3", "thiserror 2.0.19", "unicode-segmentation", @@ -3471,6 +3531,12 @@ dependencies = [ "x11", ] +[[package]] +name = "libyaml-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" + [[package]] name = "lilt" version = "0.8.1" @@ -4142,6 +4208,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "orbclient" version = "0.3.55" @@ -4501,6 +4573,19 @@ dependencies = [ "unicase", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags 2.13.1", + "getopts", + "memchr", + "pulldown-cmark-escape", + "unicase", +] + [[package]] name = "pulldown-cmark-escape" version = "0.11.0" @@ -4654,6 +4739,17 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.19", +] + [[package]] name = "regex" version = "1.13.1" @@ -5051,6 +5147,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "dirs", +] + [[package]] name = "shlex" version = "2.0.1" @@ -5583,7 +5688,20 @@ version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" dependencies = [ + "bytes", "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -5724,6 +5842,41 @@ dependencies = [ "core_maths", ] +[[package]] +name = "turbovault-core" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c68b413757ee3845f5b5182c8643e16430de683af4588d005ed2f9b21f84be" +dependencies = [ + "chrono", + "log", + "parking_lot", + "serde", + "serde_json", + "sha2", + "shellexpand", + "thiserror 2.0.19", + "tokio", + "uuid", + "winnow 1.0.4", + "yaml_serde", +] + +[[package]] +name = "turbovault-parser" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c153db2ca08515d7bfb1b2c31479710f41d39f0c0e1ddfe75d2abc32cd9c37fa" +dependencies = [ + "log", + "pulldown-cmark 0.13.4", + "regex", + "serde", + "serde_json", + "turbovault-core", + "yaml_serde", +] + [[package]] name = "two-face" version = "0.4.5" @@ -5946,6 +6099,7 @@ version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -6870,6 +7024,19 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "yaml_serde" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c7c1b1a6a7c8a6b2741a6c21a4f8918e51899b111cfa08d1288202656e3975" +dependencies = [ + "indexmap", + "itoa", + "libyaml-rs", + "ryu", + "serde", +] + [[package]] name = "yazi" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 908d050..86ab345 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ serde_json = { version = "1.0.149", features = ["preserve_order", "raw_value"] } serde_norway = "0.9.42" sha2 = "0.11.0" time = { version = "0.3.54", features = ["formatting", "parsing"] } +turbovault-parser = "1.6.0" ureq = { version = "3.3.0", default-features = false, features = ["rustls"] } url = "2.5.8" diff --git a/PLAN.md b/PLAN.md index 066d58c..d53fe4a 100644 --- a/PLAN.md +++ b/PLAN.md @@ -3,129 +3,3 @@ Only unfinished implementation work belongs here. DwarfStar remains the behavioral oracle for model execution, token processing, context accounting, KV-cache behavior, the HTTP API, and the built-in agent loop. - -## Dev Brain: a project-backed LLM Wiki - -Dev Brain implements the LLM Wiki idea described by Andrej Karpathy. The -registered projects remain the authoritative, changing source material. A -user-selected Obsidian vault is the derived, human-readable body of knowledge -that the agent maintains from those projects. Its purpose is to compile durable -knowledge about architecture, behavior, decisions, invariants, workflows, and -relationships once, keep it aligned as the projects change, and make it cheap -for later agents to retrieve the relevant context. - -This is not a general Markdown memory store. The vault contains coherent topic -pages with explicit provenance, links, and freshness state. Queries use the -compiled wiki when it is verified and return to project sources when knowledge -is missing or stale. The vault remains ordinary Obsidian Markdown so that the -user can inspect, navigate, edit, and version it without DS4Server. - -### Configuration and ownership - -- Preferences provide an enable switch and the path to one dedicated Obsidian - vault. When disabled, Dev Brain contributes neither tools nor instructions to - the agent prompt. -- A vault must be an existing directory containing `.obsidian`. DS4Server may - change only its declared wiki files and must never modify Obsidian settings, - attachments, trash, hidden files, or unrelated user notes. -- The existing configuration store persists the setting. Changing the vault or - enable state invalidates the current Dev Brain context and derived index. - -### Vault contract - -The vault is self-describing and contains: - -- `purpose.md`, maintained with the user, defining the wiki's scope, priorities, - and recurring questions; -- `schema.md`, defining page types, naming and linking conventions, provenance, - freshness states, and the rules for compilation, querying, and validation; -- `index.md`, a concise catalog of wiki pages with one-line descriptions; -- `log.md`, an append-only record of material wiki updates and their source - revisions; and -- topic pages grouped under `projects/`, `subsystems/`, `concepts/`, - `decisions/`, `invariants/`, and `workflows/` as the content requires. - -Every managed topic page has YAML frontmatter containing its page type, project -identity, `verified`, `stale`, or `needs-review` status, verification time, and -source records. A source record identifies the registered project, a -repo-relative path, an optional symbol, and the exact evidence version: the Git -revision for a clean worktree and a content hash for the files actually read. -The body uses normal Obsidian links and embeds to connect related knowledge. -Pages model useful concepts rather than mirroring every source file. - -### Compilation and maintenance - -Initial compilation examines high-signal project material such as manifests, -documentation, schemas, entry points, public interfaces, and tests, then writes -the smallest set of durable topic pages that answers the purpose of the wiki. -It does not attempt to summarize every file. - -Refresh compares each page's recorded evidence with the current project state. -Changed source paths and hashes select the dependent pages to re-read and -revalidate. The agent may update, split, merge, or retire those pages and keeps -`index.md` and `log.md` consistent with the result. Knowledge crystallized from -a development session must still cite project evidence; unsupported conclusions -are marked `needs-review` rather than presented as fact. - -Wiki changes are prepared as one candidate batch in a temporary location. The -complete candidate is validated before managed files are replaced. A failed or -ambiguous update leaves the last valid vault untouched, and publication must -not expose a partially updated wiki. - -### Parsing, indexing, and querying - -Use `turbovault-parser` to parse Obsidian-flavored Markdown, including YAML -frontmatter, headings, wikilinks, embeds, tags, and source positions without -treating link-like text inside code as graph edges. Keep the Markdown files as -the source of truth and build a disposable SQLite index using the database -support already present in DS4Server: - -- page path, title, type, status, headings, tags, and searchable body text; -- resolved wikilink and embed edges; -- project-source-to-page provenance edges; and -- an FTS5 index for ranked full-text search. - -The index is rebuilt when a vault is enabled and refreshed from file metadata -and content hashes before use. Querying starts with `index.md` or FTS, follows -relevant links and backlinks, and returns both wiki pages and their project -evidence. Stale or unverified pages are labeled and excluded from authoritative -answers until the underlying sources have been checked. - -The first implementation deliberately uses no filesystem watcher, embeddings, -vector database, or separate search service. Explicit refresh plus SQLite FTS5 -is sufficient until vault size or measured retrieval quality proves otherwise. - -### Validation - -Validation prevents the wiki from silently becoming a confident but obsolete -source: - -- structural validation checks frontmatter, page types, required fields, and - the managed path boundary; -- graph validation checks that links resolve, page identities are unique, and - `index.md` matches the managed pages; -- provenance validation checks that every cited project and source exists and - that paths remain inside the registered project; -- freshness validation checks recorded revisions and hashes against the current - files; and -- semantic validation has the agent re-check affected claims against changed - code, documentation, and tests before marking a page verified. - -Semantic validation cannot prove every statement correct, so uncertainty must -remain visible as `needs-review`. Agents may rely on `verified` pages, may use -`stale` pages only as leads, and must not turn unsupported wiki text into project -truth. - -### Agent integration and acceptance - -Dev Brain is a separate wiki maintenance and retrieval capability, not generic -filesystem access outside the active project. Its operations expose search and -read over the derived index plus validated batch publication of managed wiki -content. Detailed operating instructions live in `schema.md`; the system prompt -only announces Dev Brain when enabled and directs the agent to that contract. - -The implementation is complete when tests cover vault path confinement, -Obsidian parsing and index rebuilding, detection of source-hash drift, broken -links and invalid provenance, exclusion of stale pages from authoritative -queries, all-or-nothing publication, and prompt gating when Dev Brain is -disabled. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index f475526..4feafca 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -78,6 +78,55 @@ Tool calls and results appear in the transcript. Use their copy actions for the complete, untruncated text; large outputs can also be opened from their saved file. +## Dev Brain + +Dev Brain is a project-backed wiki for durable engineering knowledge. It lets +the coding agent compile architecture, behavior, decisions, invariants, +workflows, and relationships once, then retrieve that verified context in later +chats. Registered projects remain authoritative; Dev Brain is not a general +memory store and never replaces checking changed source code. + +To set it up: + +1. Create or choose a dedicated Obsidian vault. The folder must already contain + **.obsidian**. +2. Open **Preferences > Dev Brain**, choose the vault, enable Dev Brain, and + save. +3. In a project chat, ask the agent to build or refresh Dev Brain. Tell it the + questions and areas that matter most so it can maintain **purpose.md** and + compile a small set of useful topic pages instead of summarizing every file. + +Useful requests include “Build Dev Brain for these projects,” “Refresh Dev +Brain after my recent changes,” “What does Dev Brain know about the cache +invariants?”, and “Check the wiki, then verify this against the source.” Search +and reads use a disposable local SQLite full-text index. The index checks vault +files and cited project evidence before use, so no separate indexing service or +manual refresh button is needed. + +The vault remains ordinary Obsidian Markdown: + +- **purpose.md** records its scope, priorities, and recurring questions. +- **schema.md** is the detailed contract the agent follows when maintaining it. +- **index.md** catalogs every managed topic page. +- **log.md** is an append-only history of material updates. +- Topic pages live under **projects/**, **subsystems/**, **concepts/**, + **decisions/**, **invariants/**, and **workflows/** as needed. + +Every topic cites exact project files and a Git revision or content hash. +**Verified** pages may support answers. **Stale** pages are navigation leads +whose evidence changed, and **needs-review** pages contain visible uncertainty; +the agent must recheck project sources before relying on either. Wiki updates +are validated as a complete batch before publication, so a broken link, +out-of-bounds path, invalid citation, or stale verified claim leaves the last +valid wiki untouched. + +DS4Server may modify only the four contract files and topic pages explicitly +marked as Dev Brain-managed. It does not modify Obsidian settings, attachments, +trash, hidden files, or unrelated notes. You can inspect, edit, link, and +version the managed Markdown normally; invalid manual edits are reported rather +than silently trusted. Disabling Dev Brain removes its tools and instructions +from the built-in coding agent without deleting the vault. + ## Views and sidebar Use **View > Show Sidebar** (`⌘B`) to hide or restore the project sidebar. The diff --git a/src/a2ui_validation.rs b/src/a2ui_validation.rs index 6e34344..fc19cd9 100644 --- a/src/a2ui_validation.rs +++ b/src/a2ui_validation.rs @@ -94,7 +94,7 @@ pub(crate) fn run(args: impl Iterator) -> Result<(), String> { .timeout_recv_body(Some(Duration::from_secs(30))) .build() .into(); - let mut system = crate::agent::system_prompt(model, &config.generation.system_prompt); + let mut system = crate::agent::system_prompt(model, &config.generation.system_prompt, false); system.push_str("\n\n"); system.push_str(crate::a2ui::SYSTEM_PROMPT); let metadata = Store::default().client_metadata(); diff --git a/src/agent.rs b/src/agent.rs index 749a95b..8741b53 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -302,6 +302,7 @@ pub(crate) struct Tools { jobs: HashMap, next_job: u32, browser: Browser, + dev_brain: Option, } impl Tools { @@ -316,9 +317,19 @@ impl Tools { jobs: HashMap::new(), next_job: 1, browser: Browser::new()?, + dev_brain: None, }) } + pub(crate) fn enable_dev_brain( + &mut self, + config: &crate::config::DevBrainConfig, + projects: &[crate::database::Project], + ) -> Result<(), String> { + self.dev_brain = Some(crate::dev_brain::DevBrain::open(config, projects)?); + Ok(()) + } + fn execute(&mut self, call: &ToolCall, cancel: &AtomicBool) -> String { let result = match call.name.as_str() { "read" => self.read(call), @@ -332,6 +343,9 @@ impl Tools { "bash_stop" => self.bash_observe(call, true, cancel), "google_search" => self.google_search(call, cancel), "visit_page" => self.visit_page(call, cancel), + "dev_brain_search" => self.dev_brain_search(call), + "dev_brain_read" => self.dev_brain_read(call), + "dev_brain_publish" => self.dev_brain_publish(call), name => Err(format!("unknown tool: {name}")), }; match result { @@ -349,6 +363,54 @@ impl Tools { } } + fn dev_brain_search(&mut self, call: &ToolCall) -> Result { + let query = required_string(call, "query")?; + let limit = integer(call, "limit", 8, 1, 50); + let authoritative = boolean(call, "authoritative", true); + self.dev_brain + .as_mut() + .ok_or_else(|| "Dev Brain is disabled for this session.".to_owned())? + .search(query, limit, authoritative) + } + + fn dev_brain_read(&mut self, call: &ToolCall) -> Result { + let path = required_string(call, "path")?; + let authoritative = boolean(call, "authoritative", false); + self.dev_brain + .as_mut() + .ok_or_else(|| "Dev Brain is disabled for this session.".to_owned())? + .read(path, authoritative) + } + + fn dev_brain_publish(&mut self, call: &ToolCall) -> Result { + let files = call + .arguments + .get("files") + .and_then(Value::as_object) + .ok_or_else(|| "dev_brain_publish requires a files object.".to_owned())?; + let remove = call + .arguments + .get("remove") + .map(|value| { + value + .as_array() + .ok_or_else(|| "remove must be an array of paths.".to_owned())? + .iter() + .map(|path| { + path.as_str() + .map(str::to_owned) + .ok_or_else(|| "remove paths must be strings.".to_owned()) + }) + .collect::, String>>() + }) + .transpose()? + .unwrap_or_default(); + self.dev_brain + .as_mut() + .ok_or_else(|| "Dev Brain is disabled for this session.".to_owned())? + .publish(files, &remove) + } + fn result_limit(&self) -> usize { (self.context_tokens.max(4096) as usize * 2).min(512 * 1024) } @@ -1229,16 +1291,26 @@ pub(crate) fn parse_tool_calls( .map(|calls| (content, calls)) } -pub(crate) fn system_prompt(model: ModelChoice, extra: &str) -> String { +pub(crate) fn system_prompt(model: ModelChoice, extra: &str, dev_brain: bool) -> String { + let schemas = if dev_brain { + format!("{TOOL_SCHEMAS}\n{}", crate::dev_brain::TOOL_SCHEMAS) + } else { + TOOL_SCHEMAS.to_owned() + }; let tools = if model == ModelChoice::Glm52 { format!( - "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n# Tools\n\nYou are provided with function signatures within XML tags:\n\n{TOOL_SCHEMAS}\n\n\nFor a function call, output exactly: function-namekeyvalue\nTool calls are not allowed inside . Use read/search for focused context, edit with exact unique old text, and [upto] only between unique head and tail anchors. Use refresh_sec for long bash jobs and poll with bash_status or stop with bash_stop. Preserve the current system configuration unless the user explicitly asks otherwise." + "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n# Tools\n\nYou are provided with function signatures within XML tags:\n\n{schemas}\n\n\nFor a function call, output exactly: function-namekeyvalue\nTool calls are not allowed inside . Use read/search for focused context, edit with exact unique old text, and [upto] only between unique head and tail anchors. Use refresh_sec for long bash jobs and poll with bash_status or stop with bash_stop. Preserve the current system configuration unless the user explicitly asks otherwise." ) } else { format!( - "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n## Tools\n\nInvoke native DSML tools exactly as:\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n\n\n\nTool calls are not allowed inside . String parameters use raw text and string=\"true\"; numbers and booleans use JSON text and string=\"false\". Read defaults to a bounded chunk; use more to continue and whole=true only when needed. Use write for new files or whole-file replacement. Use edit with path first and exact unique old text; old may contain one [upto] marker between unique head and tail anchors. For long bash commands pass refresh_sec, then use bash_status or bash_stop. The first web call asks permission to start visible Chrome.\n\n### Available Tool Schemas\n\n{TOOL_SCHEMAS}\n\n# Rules\n- Always use strict DSML syntax.\n- Use read/search to get anchors before editing.\n- Preserve the current system configuration unless explicitly asked otherwise." + "You are a coding agent running in a local workspace. Use tools for local file and system work. Avoid printing large file contents or large code blocks as answers; create or edit files with tools, then summarize results briefly.\n\n## Tools\n\nInvoke native DSML tools exactly as:\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n\n\n\nTool calls are not allowed inside . String parameters use raw text and string=\"true\"; numbers and booleans use JSON text and string=\"false\". Read defaults to a bounded chunk; use more to continue and whole=true only when needed. Use write for new files or whole-file replacement. Use edit with path first and exact unique old text; old may contain one [upto] marker between unique head and tail anchors. For long bash commands pass refresh_sec, then use bash_status or bash_stop. The first web call asks permission to start visible Chrome.\n\n### Available Tool Schemas\n\n{schemas}\n\n# Rules\n- Always use strict DSML syntax.\n- Use read/search to get anchors before editing.\n- Preserve the current system configuration unless explicitly asked otherwise." ) }; + let tools = if dev_brain { + format!("{tools}\n\n{}", crate::dev_brain::PROMPT) + } else { + tools + }; if extra.trim().is_empty() { tools } else { @@ -1246,10 +1318,10 @@ pub(crate) fn system_prompt(model: ModelChoice, extra: &str) -> String { } } -pub(crate) fn system_prompt_reminder(model: ModelChoice) -> String { +pub(crate) fn system_prompt_reminder(model: ModelChoice, dev_brain: bool) -> String { format!( "[System prompt reminder follows.]\n{}\n[End system prompt reminder.]", - system_prompt(model, "") + system_prompt(model, "", dev_brain) ) } @@ -1654,7 +1726,7 @@ mod tests { #[test] fn prompts_and_parsers_expose_the_reference_tool_set() { - let prompt = system_prompt(ModelChoice::DeepSeekV4Flash, "extra"); + let prompt = system_prompt(ModelChoice::DeepSeekV4Flash, "extra", false); for name in [ "google_search", "visit_page", @@ -1672,9 +1744,11 @@ mod tests { } assert!(prompt.ends_with("extra")); assert!( - system_prompt_reminder(ModelChoice::DeepSeekV4Flash) + system_prompt_reminder(ModelChoice::DeepSeekV4Flash, false) .contains("[System prompt reminder follows.]") ); + assert!(!prompt.contains("dev_brain_search")); + assert!(system_prompt(ModelChoice::DeepSeekV4Flash, "", true).contains("dev_brain_search")); assert!(datetime_context().starts_with("Current local date and time at session start:")); assert!(!prompt_reminder_due(49_999, 0)); assert!(prompt_reminder_due(50_000, 0)); diff --git a/src/app.rs b/src/app.rs index 839f5f7..1d031e6 100644 --- a/src/app.rs +++ b/src/app.rs @@ -14,7 +14,7 @@ use preferences::PreferenceDraft; #[cfg(test)] use preferences::{parse_optional_gib, parse_streaming_cache}; -use crate::config::{Config, EndpointConfig}; +use crate::config::{Config, DevBrainConfig, EndpointConfig}; use crate::database::{Database, ProjectWithSessions, SessionState, StoredMessage}; #[cfg(target_os = "macos")] use crate::engine::ChatTurn; @@ -247,6 +247,7 @@ pub(super) enum DetailTab { pub(super) enum PreferenceSection { Model, Endpoint, + DevBrain, Generation, Execution, Acceleration, @@ -255,9 +256,10 @@ pub(super) enum PreferenceSection { } impl PreferenceSection { - const ALL: [Self; 7] = [ + const ALL: [Self; 8] = [ Self::Model, Self::Endpoint, + Self::DevBrain, Self::Generation, Self::Execution, Self::Acceleration, @@ -269,6 +271,7 @@ impl PreferenceSection { match self { Self::Model => "preferences-model", Self::Endpoint => "preferences-endpoint", + Self::DevBrain => "preferences-dev-brain", Self::Generation => "preferences-generation", Self::Execution => "preferences-execution", Self::Acceleration => "preferences-acceleration", @@ -281,6 +284,7 @@ impl PreferenceSection { match self { Self::Model => "Model & lifecycle", Self::Endpoint => "Local endpoint", + Self::DevBrain => "Dev Brain", Self::Generation => "Generation", Self::Execution => "Execution", Self::Acceleration => "Acceleration & memory", @@ -338,6 +342,10 @@ pub(crate) enum Message { PreferenceEndpointPortChanged(String), PreferenceEndpointEnabledChanged(bool), PreferenceEndpointCorsChanged(bool), + PreferenceDevBrainEnabledChanged(bool), + PreferenceDevBrainVaultChanged(String), + ChooseDevBrainVault, + DevBrainVaultPicked(Option), PreferenceContextChanged(String), PreferenceMaxTokensChanged(String), PreferenceSystemPromptAction(text_editor::Action), @@ -1157,6 +1165,33 @@ impl App { self.preference_draft.endpoint_cors = enabled; self.preference_error = None; } + Message::PreferenceDevBrainEnabledChanged(enabled) => { + self.preference_draft.dev_brain_enabled = enabled; + self.preference_error = None; + } + Message::PreferenceDevBrainVaultChanged(value) => { + self.preference_draft.dev_brain_vault_path = value; + self.preference_error = None; + } + Message::ChooseDevBrainVault => { + return Task::perform( + async { + AsyncFileDialog::new() + .set_title("Choose an Obsidian vault") + .pick_folder() + .await + .map(|folder| folder.path().to_path_buf()) + }, + Message::DevBrainVaultPicked, + ); + } + Message::DevBrainVaultPicked(path) => { + if let Some(path) = path { + self.preference_draft.dev_brain_vault_path = + path.to_string_lossy().into_owned(); + self.preference_error = None; + } + } Message::PreferenceContextChanged(value) => { self.preference_draft.context_tokens = value; self.preference_error = None; @@ -1702,6 +1737,7 @@ impl App { self.tokens_per_second = None; } self.reload_projects(); + self.invalidate_dev_brain_context(); self.finish_cache_change(); } Err(error) => self.error = Some(error), diff --git a/src/app/generation.rs b/src/app/generation.rs index 02a5582..61cc910 100644 --- a/src/app/generation.rs +++ b/src/app/generation.rs @@ -382,7 +382,7 @@ fn title_context(messages: impl IntoIterator) -> Vec impl App { fn chat_system_prompt(&self, model: ModelChoice, prompt: &str, agents: Option<&str>) -> String { - let mut prompt = crate::agent::system_prompt(model, prompt); + let mut prompt = crate::agent::system_prompt(model, prompt, self.config.dev_brain.enabled); if self.config.a2ui_enabled { prompt.push_str("\n\n"); prompt.push_str(crate::a2ui::SYSTEM_PROMPT); @@ -668,7 +668,10 @@ impl App { } fn system_prompt_reminders(&self, model: ModelChoice) -> Vec { - let mut reminders = vec![crate::agent::system_prompt_reminder(model)]; + let mut reminders = vec![crate::agent::system_prompt_reminder( + model, + self.config.dev_brain.enabled, + )]; if self.config.a2ui_enabled { reminders.push(crate::a2ui::SYSTEM_PROMPT.to_owned()); } @@ -1163,7 +1166,15 @@ impl App { .find(|project| project.project.id == project_id) .map(|project| PathBuf::from(&project.project.path)) .ok_or_else(|| "The active project is unavailable.".to_owned())?; - let tools = crate::agent::Tools::new(&root, self.config.generation.context_tokens)?; + let mut tools = crate::agent::Tools::new(&root, self.config.generation.context_tokens)?; + if self.config.dev_brain.enabled { + let projects = self + .projects + .iter() + .map(|project| project.project.clone()) + .collect::>(); + tools.enable_dev_brain(&self.config.dev_brain, &projects)?; + } self.agent_tools = Some((session_id, Arc::new(Mutex::new(tools)))); } let tools = Arc::clone(&self.agent_tools.as_ref().unwrap().1); diff --git a/src/app/preferences.rs b/src/app/preferences.rs index 513c2ba..f108f7b 100644 --- a/src/app/preferences.rs +++ b/src/app/preferences.rs @@ -10,6 +10,8 @@ pub(super) struct PreferenceDraft { pub(super) endpoint_port: String, pub(super) endpoint_enabled: bool, pub(super) endpoint_cors: bool, + pub(super) dev_brain_enabled: bool, + pub(super) dev_brain_vault_path: String, pub(super) context_tokens: String, pub(super) max_generated_tokens: String, pub(super) system_prompt: text_editor::Content, @@ -60,6 +62,8 @@ impl PreferenceDraft { endpoint_port: config.endpoint.port.to_string(), endpoint_enabled: config.endpoint.enabled, endpoint_cors: config.endpoint.cors, + dev_brain_enabled: config.dev_brain.enabled, + dev_brain_vault_path: config.dev_brain.vault_path.clone().unwrap_or_default(), context_tokens: generation.context_tokens.to_string(), max_generated_tokens: generation.max_generated_tokens.to_string(), system_prompt: text_editor::Content::with_text(&generation.system_prompt), @@ -368,6 +372,10 @@ impl App { enabled: self.preference_draft.endpoint_enabled, cors: self.preference_draft.endpoint_cors, }, + dev_brain: DevBrainConfig { + enabled: self.preference_draft.dev_brain_enabled, + vault_path: optional_text(&self.preference_draft.dev_brain_vault_path), + }, generation, runtime, interface: self.config.interface.clone(), @@ -376,6 +384,18 @@ impl App { self.preference_error = Some(error); return; } + if config.dev_brain.enabled { + let projects = self + .projects + .iter() + .map(|project| project.project.clone()) + .collect::>(); + if let Err(error) = crate::dev_brain::DevBrain::open(&config.dev_brain, &projects) { + self.preference_error = Some(error); + return; + } + } + let dev_brain_changed = self.config.dev_brain != config.dev_brain; #[cfg(target_os = "macos")] let endpoint_changed = self.config.endpoint != config.endpoint; #[cfg(target_os = "macos")] @@ -415,6 +435,10 @@ impl App { } self.config = config; #[cfg(target_os = "macos")] + if dev_brain_changed { + self.invalidate_dev_brain_context(); + } + #[cfg(target_os = "macos")] update_runtime_config(&self.runtime_config, &self.config); #[cfg(target_os = "macos")] if endpoint_changed { diff --git a/src/app/projects.rs b/src/app/projects.rs index 0ca2369..f83892d 100644 --- a/src/app/projects.rs +++ b/src/app/projects.rs @@ -152,6 +152,7 @@ impl App { self.project_name_input.clear(); self.error = None; self.reload_projects(); + self.invalidate_dev_brain_context(); self.refresh_git_state(); } Err(error) => self.error = Some(error), @@ -349,6 +350,21 @@ impl App { } } } + + pub(super) fn invalidate_dev_brain_context(&mut self) { + if !self.config.dev_brain.enabled { + return; + } + #[cfg(target_os = "macos")] + { + self.agent_tools = None; + for chat in self.background_chats.values_mut() { + chat.agent_tools = None; + chat.system_prompt_seen_at = 0; + } + } + self.system_prompt_seen_at = 0; + } } fn read_git_state(path: &Path) -> Option { diff --git a/src/app/view/preferences.rs b/src/app/view/preferences.rs index 57a635e..c0fa7c7 100644 --- a/src/app/view/preferences.rs +++ b/src/app/view/preferences.rs @@ -156,6 +156,33 @@ impl App { ] .spacing(10), ); + let dev_brain_group = preference_group( + PreferenceSection::DevBrain, + "DEV BRAIN", + column![ + hint( + checkbox(self.preference_draft.dev_brain_enabled) + .label("Enable project-backed LLM wiki") + .on_toggle(Message::PreferenceDevBrainEnabledChanged), + "Adds validated search, read, and batch-publication tools for a dedicated Obsidian vault. Disabled means no Dev Brain tools or prompt instructions.", + ), + row![ + text_input( + "/path/to/Obsidian vault", + &self.preference_draft.dev_brain_vault_path, + ) + .on_input(Message::PreferenceDevBrainVaultChanged) + .padding(9) + .width(Length::Fill), + action_button("Choose…").on_press(Message::ChooseDevBrainVault), + ] + .spacing(8) + .align_y(Alignment::Center), + text("The folder must already contain .obsidian. DS4Server manages only its declared wiki pages and leaves settings, attachments, hidden files, and unrelated notes untouched.") + .size(12), + ] + .spacing(10), + ); let generation_group = preference_group( PreferenceSection::Generation, "GENERATION", @@ -552,6 +579,7 @@ impl App { let mut fields = column![ model_group, endpoint_group, + dev_brain_group, generation_group, execution_group, acceleration_group, diff --git a/src/config.rs b/src/config.rs index 3888f21..561f982 100644 --- a/src/config.rs +++ b/src/config.rs @@ -15,6 +15,7 @@ pub struct Config { pub model: ModelChoice, pub idle_timeout_minutes: i32, pub a2ui_enabled: bool, + pub dev_brain: DevBrainConfig, pub endpoint: EndpointConfig, pub generation: GenerationPreferences, pub runtime: RuntimePreferences, @@ -27,6 +28,7 @@ impl Default for Config { model: ModelChoice::default(), idle_timeout_minutes: 10, a2ui_enabled: true, + dev_brain: DevBrainConfig::default(), endpoint: EndpointConfig::default(), generation: GenerationPreferences::default(), runtime: RuntimePreferences::default(), @@ -35,6 +37,35 @@ impl Default for Config { } } +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct DevBrainConfig { + pub enabled: bool, + pub vault_path: Option, +} + +impl DevBrainConfig { + pub fn vault(&self) -> Result { + if !self.enabled { + return Err("Dev Brain is disabled.".into()); + } + let path = self + .vault_path + .as_deref() + .filter(|path| !path.trim().is_empty()) + .ok_or_else(|| "Choose an Obsidian vault before enabling Dev Brain.".to_owned())?; + crate::dev_brain::validate_vault(Path::new(path)) + } + + fn validate(&self) -> Result<(), String> { + if self.enabled { + self.vault().map(|_| ()) + } else { + Ok(()) + } + } +} + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(default, deny_unknown_fields)] pub struct EndpointConfig { @@ -113,7 +144,8 @@ impl Config { return Err("Endpoint port must be between 1 and 65535.".into()); } self.generation.validate()?; - self.runtime.validate(self.model) + self.runtime.validate(self.model)?; + self.dev_brain.validate() } } diff --git a/src/dev_brain.rs b/src/dev_brain.rs new file mode 100644 index 0000000..e3710e8 --- /dev/null +++ b/src/dev_brain.rs @@ -0,0 +1,1665 @@ +use diesel::connection::SimpleConnection; +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Double, Text}; +use serde::Deserialize; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; +use std::sync::RwLock; +use std::time::{SystemTime, UNIX_EPOCH}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use turbovault_parser::{LinkType, Parser, to_plain_text}; + +use crate::config::DevBrainConfig; +use crate::database::Project; + +const ROOT_PAGES: [&str; 4] = ["purpose.md", "schema.md", "index.md", "log.md"]; +const TOPIC_DIRS: [&str; 6] = [ + "projects", + "subsystems", + "concepts", + "decisions", + "invariants", + "workflows", +]; +// ponytail: one process-wide lock keeps publication invisible to all in-app +// readers; use per-vault locks only if concurrent vault throughput matters. +static VAULT_LOCK: RwLock<()> = RwLock::new(()); + +pub(crate) const PROMPT: &str = "Dev Brain is enabled for this session. It is a project-backed Obsidian wiki, not generic memory. Read dev_brain_read(path=\"schema.md\") before maintaining it. Search verified knowledge with dev_brain_search before broad source exploration; stale and needs-review pages are leads only and must be checked against project sources. Publish wiki changes only through dev_brain_publish so the complete candidate is validated and applied as one batch."; + +pub(crate) const TOOL_SCHEMAS: &str = r#"{"type":"function","function":{"name":"dev_brain_search","description":"Search the validated Dev Brain wiki and return ranked pages, graph context, and project evidence. Authoritative search excludes stale and needs-review pages.","parameters":{"type":"object","properties":{"query":{"type":"string"},"limit":{"type":"number"},"authoritative":{"type":"boolean"}},"required":["query"]}}} +{"type":"function","function":{"name":"dev_brain_read","description":"Read one indexed Dev Brain page with freshness status and project evidence.","parameters":{"type":"object","properties":{"path":{"type":"string"},"authoritative":{"type":"boolean"}},"required":["path"]}}} +{"type":"function","function":{"name":"dev_brain_publish","description":"Validate and publish one candidate batch of managed wiki changes. Files overlays the current managed snapshot; remove retires managed topic pages. The whole resulting wiki is validated before any live file changes.","parameters":{"type":"object","properties":{"files":{"type":"object","additionalProperties":{"type":"string"}},"remove":{"type":"array","items":{"type":"string"}}},"required":["files"]}}}"#; + +const DEFAULT_PURPOSE: &str = "# Dev Brain purpose\n\n\ +Dev Brain compiles durable, source-backed knowledge from the registered projects.\n\n\ +## Priorities\n\n\ +- Architecture, behavior, decisions, invariants, workflows, and relationships.\n\ +- Small topic pages that answer recurring development questions.\n\ +- Visible uncertainty and exact project provenance.\n\n\ +## Recurring questions\n\n\ +- Where does a behavior live, and what must remain invariant when it changes?\n\ +- Which decisions constrain the current implementation?\n\ +- Which project sources must be rechecked before relying on this page?\n"; + +const DEFAULT_INDEX: &str = "# Dev Brain index\n\nNo topic pages have been compiled yet.\n"; +const DEFAULT_LOG: &str = + "# Dev Brain log\n\n\n"; +const DEFAULT_SCHEMA: &str = r#"# Dev Brain schema + +The registered projects are authoritative. This vault is a derived, human-readable wiki. + +## Managed paths + +DS4Server manages `purpose.md`, `schema.md`, `index.md`, `log.md`, and topic pages below `projects/`, `subsystems/`, `concepts/`, `decisions/`, `invariants/`, and `workflows/`. Topic pages must opt in with `dev_brain: true`. Other notes, hidden files, Obsidian settings, attachments, and trash are never modified. + +## Topic frontmatter + +```yaml +--- +dev_brain: true +type: subsystem # project, subsystem, concept, decision, invariant, or workflow +project: Registered project name +status: verified # verified, stale, or needs-review +verified_at: 2026-07-27T12:00:00Z +sources: + - project: Registered project name + path: src/example.rs + symbol: optional_symbol + revision: exact-clean-git-revision + # Use hash instead of revision when the worktree is dirty or is not Git. +--- +``` + +Each source has exactly one evidence version: `revision` or a lowercase SHA-256 `hash`. Paths are project-relative and may not escape the registered project. + +## Compilation + +Read the purpose first. Examine high-signal manifests, documentation, schemas, entry points, public interfaces, and tests. Create the smallest coherent topic set that answers the purpose; do not mirror every source file. Use ordinary Obsidian wikilinks and embeds to connect topics. + +## Querying + +Start with `index.md` or ranked search, then follow links and backlinks. A `verified` page is authoritative only while all recorded evidence is current. Treat `stale` pages as navigation leads and `needs-review` pages as explicit uncertainty; check project sources before making claims from either. + +## Refresh and semantic validation + +When evidence changes, re-read the affected code, documentation, and tests. Update, split, merge, or retire dependent pages, keep `index.md` exact, and append a material update entry to `log.md` with source revisions. Only mark a page `verified` after its claims have been checked against its current evidence. Unsupported conclusions stay `needs-review`. + +## Publication + +Use `dev_brain_publish` for every managed change. It builds a candidate snapshot, checks structure, paths, graph resolution, unique identities, index coverage, provenance, and freshness, then publishes the batch. `log.md` is append-only. +"#; + +#[derive(Clone)] +struct RegisteredProject { + name: String, + root: PathBuf, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SourceRecord { + project: String, + path: String, + #[serde(default)] + symbol: Option, + #[serde(default)] + revision: Option, + #[serde(default)] + hash: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TopicFrontmatter { + dev_brain: bool, + #[serde(rename = "type")] + page_type: String, + project: String, + status: String, + verified_at: String, + sources: Vec, +} + +#[derive(Clone)] +struct Page { + path: String, + title: String, + page_type: String, + status: String, + verified_at: String, + headings: Vec, + tags: Vec, + links: Vec, + sources: Vec, + body: String, + content: String, +} + +#[derive(Clone)] +struct RawLink { + target: String, + kind: LinkType, + line: usize, +} + +#[derive(Clone, Eq, PartialEq)] +struct Fingerprint { + path: String, + modified_nanos: u128, + size: u64, + hash: String, +} + +#[derive(QueryableByName)] +struct SearchRow { + #[diesel(sql_type = Text)] + path: String, + #[diesel(sql_type = Text)] + title: String, + #[diesel(sql_type = Text)] + page_type: String, + #[diesel(sql_type = Text)] + status: String, + #[diesel(sql_type = Text)] + excerpt: String, + #[diesel(sql_type = Double)] + rank: f64, +} + +#[derive(QueryableByName)] +struct PageRow { + #[diesel(sql_type = Text)] + path: String, + #[diesel(sql_type = Text)] + title: String, + #[diesel(sql_type = Text)] + page_type: String, + #[diesel(sql_type = Text)] + status: String, + #[diesel(sql_type = Text)] + verified_at: String, + #[diesel(sql_type = Text)] + content: String, +} + +#[derive(QueryableByName)] +struct EvidenceRow { + #[diesel(sql_type = Text)] + project: String, + #[diesel(sql_type = Text)] + source_path: String, + #[diesel(sql_type = Text)] + symbol: String, + #[diesel(sql_type = Text)] + version: String, +} + +#[derive(QueryableByName)] +struct PathRow { + #[diesel(sql_type = Text)] + path: String, +} + +#[derive(QueryableByName)] +struct SourceKeyRow { + #[diesel(sql_type = Text)] + project: String, + #[diesel(sql_type = Text)] + source_path: String, +} + +pub(crate) struct DevBrain { + vault: PathBuf, + projects: Vec, + connection: SqliteConnection, + fingerprint: Vec, +} + +impl DevBrain { + pub(crate) fn open(config: &DevBrainConfig, projects: &[Project]) -> Result { + let _guard = VAULT_LOCK + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let vault = config.vault()?; + ensure_contract(&vault)?; + let projects = projects + .iter() + .map(|project| { + Ok(RegisteredProject { + name: project.name.clone(), + root: Path::new(&project.path).canonicalize().map_err(|error| { + format!( + "Could not open registered project {}: {error}", + project.name + ) + })?, + }) + }) + .collect::, String>>()?; + let mut names = HashSet::new(); + if let Some(duplicate) = projects + .iter() + .map(|project| project.name.as_str()) + .find(|name| !names.insert((*name).to_owned())) + { + return Err(format!( + "Registered project names must be unique for Dev Brain provenance: {duplicate}" + )); + } + let connection = + SqliteConnection::establish(":memory:").map_err(|error| error.to_string())?; + let mut brain = Self { + vault, + projects, + connection, + fingerprint: Vec::new(), + }; + brain.rebuild()?; + Ok(brain) + } + + pub(crate) fn search( + &mut self, + query: &str, + limit: usize, + authoritative: bool, + ) -> Result { + let _guard = VAULT_LOCK + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + self.refresh()?; + let query = fts_query(query); + if query.is_empty() { + return self.read_indexed("index.md", authoritative); + } + let sql = if authoritative { + "SELECT p.path, p.title, p.page_type, p.status, \ + snippet(page_fts, 2, '', '', ' … ', 18) AS excerpt, \ + bm25(page_fts) AS rank FROM page_fts \ + JOIN pages p ON p.path = page_fts.path \ + WHERE page_fts MATCH ? AND p.status = 'verified' \ + ORDER BY rank LIMIT ?" + } else { + "SELECT p.path, p.title, p.page_type, p.status, \ + snippet(page_fts, 2, '', '', ' … ', 18) AS excerpt, \ + bm25(page_fts) AS rank FROM page_fts \ + JOIN pages p ON p.path = page_fts.path \ + WHERE page_fts MATCH ? ORDER BY rank LIMIT ?" + }; + let rows = diesel::sql_query(sql) + .bind::(&query) + .bind::(i64::try_from(limit.clamp(1, 50)).unwrap_or(50)) + .load::(&mut self.connection) + .map_err(|error| format!("Could not search Dev Brain: {error}"))?; + if rows.is_empty() { + return Ok(if authoritative { + "No verified Dev Brain pages matched. Search with authoritative=false for stale leads, then check project sources.\n".into() + } else { + "No Dev Brain pages matched.\n".into() + }); + } + let mut output = String::new(); + for row in &rows { + output.push_str(&format!( + "## {} ({})\npath: {} | status: {} | rank: {:.3}\n{}\n", + row.title, row.page_type, row.path, row.status, row.rank, row.excerpt + )); + self.append_evidence(&row.path, &mut output)?; + let related = self.related_paths(&row.path, authoritative)?; + if !related.is_empty() { + output.push_str(&format!("related: {}\n", related.join(", "))); + } + output.push('\n'); + } + Ok(output) + } + + pub(crate) fn read(&mut self, value: &str, authoritative: bool) -> Result { + let _guard = VAULT_LOCK + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + self.refresh()?; + self.read_indexed(value, authoritative) + } + + fn read_indexed(&mut self, value: &str, authoritative: bool) -> Result { + let normalized = normalize_managed_path(value) + .map(|path| path_to_string(&path)) + .unwrap_or_else(|_| value.to_owned()); + let sql = if authoritative { + "SELECT path, title, page_type, status, verified_at, content FROM pages \ + WHERE (path = ? OR lower(title) = lower(?)) AND status = 'verified' LIMIT 1" + } else { + "SELECT path, title, page_type, status, verified_at, content FROM pages \ + WHERE path = ? OR lower(title) = lower(?) ORDER BY path LIMIT 1" + }; + let row = diesel::sql_query(sql) + .bind::(&normalized) + .bind::(value) + .get_result::(&mut self.connection) + .optional() + .map_err(|error| format!("Could not read Dev Brain: {error}"))?; + let Some(row) = row else { + return Ok(if authoritative { + format!( + "{value} is missing, stale, or needs review and was excluded from authoritative reading. Read with authoritative=false only as a lead.\n" + ) + } else { + format!("Dev Brain page not found: {value}\n") + }); + }; + let mut output = format!( + "# {} ({})\npath: {} | status: {} | verified_at: {}\n", + row.title, row.page_type, row.path, row.status, row.verified_at + ); + self.append_evidence(&row.path, &mut output)?; + output.push('\n'); + output.push_str(&row.content); + if !output.ends_with('\n') { + output.push('\n'); + } + Ok(output) + } + + pub(crate) fn publish( + &mut self, + files: &Map, + remove: &[String], + ) -> Result { + let _guard = VAULT_LOCK + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if files.is_empty() && remove.is_empty() { + return Err("Dev Brain publication batch is empty.".into()); + } + let current = managed_files(&self.vault)?; + let old_log = fs::read_to_string(self.vault.join("log.md")).unwrap_or_default(); + let staging = temporary_sibling(&self.vault, "candidate")?; + fs::create_dir(&staging).map_err(|error| error.to_string())?; + let result = (|| { + for path in ¤t { + let target = staging.join(path); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|error| error.to_string())?; + } + fs::copy(self.vault.join(path), target).map_err(|error| error.to_string())?; + } + let mut changed = BTreeSet::new(); + for (raw_path, value) in files { + let path = normalize_managed_path(raw_path)?; + let content = value.as_str().ok_or_else(|| { + format!("Dev Brain file content must be a string: {raw_path}") + })?; + if content.len() > 16 * 1024 * 1024 { + return Err(format!("Dev Brain file is too large: {raw_path}")); + } + if is_topic_path(&path) && !is_managed_topic(content) { + return Err(format!( + "Managed topic {raw_path} must have YAML frontmatter with dev_brain: true." + )); + } + let target = staging.join(&path); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|error| error.to_string())?; + } + fs::write(target, content).map_err(|error| error.to_string())?; + changed.insert(path); + } + for raw_path in remove { + let path = normalize_managed_path(raw_path)?; + if ROOT_PAGES.contains(&path_to_string(&path).as_str()) { + return Err(format!( + "Required Dev Brain page cannot be removed: {raw_path}" + )); + } + if !current.contains(&path) { + return Err(format!("Refusing to remove an unmanaged note: {raw_path}")); + } + match fs::remove_file(staging.join(&path)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.to_string()), + } + changed.insert(path); + } + let new_log = fs::read_to_string(staging.join("log.md")) + .map_err(|error| format!("Could not read candidate log.md: {error}"))?; + if !new_log.starts_with(&old_log) { + return Err( + "log.md is append-only; the candidate removed or changed history.".into(), + ); + } + let pages = load_and_validate_pages(&staging, &self.vault, &self.projects, true)?; + validate_graph(&pages, &staging, &self.vault)?; + commit_batch(&self.vault, &staging, &changed)?; + Ok(changed.len()) + })(); + let _ = fs::remove_dir_all(&staging); + let count = result?; + self.rebuild()?; + Ok(format!( + "Published {count} managed Dev Brain file change{} after full candidate validation.\n", + if count == 1 { "" } else { "s" } + )) + } + + fn refresh(&mut self) -> Result<(), String> { + if self.current_fingerprint()? != self.fingerprint { + self.rebuild()?; + } + Ok(()) + } + + fn rebuild(&mut self) -> Result<(), String> { + let pages = load_and_validate_pages(&self.vault, &self.vault, &self.projects, false)?; + let resolved = validate_graph(&pages, &self.vault, &self.vault)?; + self.connection + .batch_execute( + "DROP TABLE IF EXISTS page_fts; + DROP TABLE IF EXISTS sources; + DROP TABLE IF EXISTS links; + DROP TABLE IF EXISTS tags; + DROP TABLE IF EXISTS headings; + DROP TABLE IF EXISTS pages; + CREATE TABLE pages ( + path TEXT PRIMARY KEY, + title TEXT NOT NULL, + page_type TEXT NOT NULL, + status TEXT NOT NULL, + verified_at TEXT NOT NULL, + body TEXT NOT NULL, + content TEXT NOT NULL + ); + CREATE TABLE headings (page_path TEXT NOT NULL, heading TEXT NOT NULL); + CREATE TABLE tags (page_path TEXT NOT NULL, tag TEXT NOT NULL); + CREATE TABLE links ( + source_path TEXT NOT NULL, + target_path TEXT NOT NULL, + kind TEXT NOT NULL + ); + CREATE TABLE sources ( + page_path TEXT NOT NULL, + project TEXT NOT NULL, + source_path TEXT NOT NULL, + symbol TEXT NOT NULL, + version TEXT NOT NULL + ); + CREATE VIRTUAL TABLE page_fts USING fts5( + path UNINDEXED, title, body, tags, tokenize='unicode61' + );", + ) + .map_err(|error| format!("Could not create Dev Brain index: {error}"))?; + self.connection + .transaction::<_, diesel::result::Error, _>(|connection| { + for page in &pages { + diesel::sql_query( + "INSERT INTO pages \ + (path, title, page_type, status, verified_at, body, content) \ + VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind::(&page.path) + .bind::(&page.title) + .bind::(&page.page_type) + .bind::(&page.status) + .bind::(&page.verified_at) + .bind::(&page.body) + .bind::(&page.content) + .execute(connection)?; + diesel::sql_query( + "INSERT INTO page_fts (path, title, body, tags) VALUES (?, ?, ?, ?)", + ) + .bind::(&page.path) + .bind::(&page.title) + .bind::(&page.body) + .bind::(page.tags.join(" ")) + .execute(connection)?; + for heading in &page.headings { + diesel::sql_query( + "INSERT INTO headings (page_path, heading) VALUES (?, ?)", + ) + .bind::(&page.path) + .bind::(heading) + .execute(connection)?; + } + for tag in &page.tags { + diesel::sql_query("INSERT INTO tags (page_path, tag) VALUES (?, ?)") + .bind::(&page.path) + .bind::(tag) + .execute(connection)?; + } + for source in &page.sources { + let version = source.revision.as_ref().map_or_else( + || format!("sha256:{}", source.hash.as_deref().unwrap_or_default()), + |revision| format!("git:{revision}"), + ); + diesel::sql_query( + "INSERT INTO sources \ + (page_path, project, source_path, symbol, version) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind::(&page.path) + .bind::(&source.project) + .bind::(&source.path) + .bind::(source.symbol.as_deref().unwrap_or("")) + .bind::(version) + .execute(connection)?; + } + } + for (source, target, kind) in &resolved { + diesel::sql_query( + "INSERT INTO links (source_path, target_path, kind) VALUES (?, ?, ?)", + ) + .bind::(source) + .bind::(target) + .bind::(kind) + .execute(connection)?; + } + Ok(()) + }) + .map_err(|error| format!("Could not build Dev Brain index: {error}"))?; + self.fingerprint = self.current_fingerprint()?; + Ok(()) + } + + fn current_fingerprint(&mut self) -> Result, String> { + let mut fingerprints = vault_fingerprint(&self.vault)?; + let sources = diesel::sql_query( + "SELECT DISTINCT project, source_path FROM sources ORDER BY project, source_path", + ) + .load::(&mut self.connection) + .map_err(|error| error.to_string())?; + let mut project_names = BTreeSet::new(); + for source in sources { + let project = self + .projects + .iter() + .find(|project| project.name == source.project) + .ok_or_else(|| format!("Unregistered Dev Brain project: {}", source.project))?; + let full = project + .root + .join(normalize_source_path(&source.source_path)?); + let metadata = full.metadata().map_err(|error| { + format!( + "Could not inspect Dev Brain source {}:{}: {error}", + source.project, source.source_path + ) + })?; + let modified_nanos = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_nanos()); + fingerprints.push(Fingerprint { + path: format!("source:{}:{}", source.project, source.source_path), + modified_nanos, + size: metadata.len(), + hash: hash_file(&full)?, + }); + project_names.insert(source.project); + } + for name in project_names { + let project = self + .projects + .iter() + .find(|project| project.name == name) + .unwrap(); + let state = git_state(&project.root) + .map(|(revision, clean)| format!("{revision}:{clean}")) + .unwrap_or_default(); + fingerprints.push(Fingerprint { + path: format!("git:{name}"), + modified_nanos: 0, + size: 0, + hash: state, + }); + } + fingerprints.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(fingerprints) + } + + fn append_evidence(&mut self, path: &str, output: &mut String) -> Result<(), String> { + let rows = diesel::sql_query( + "SELECT project, source_path, symbol, version FROM sources \ + WHERE page_path = ? ORDER BY project, source_path, symbol", + ) + .bind::(path) + .load::(&mut self.connection) + .map_err(|error| error.to_string())?; + for row in rows { + output.push_str(&format!( + "evidence: {}:{}{} @ {}\n", + row.project, + row.source_path, + if row.symbol.is_empty() { + String::new() + } else { + format!("#{}", row.symbol) + }, + row.version + )); + } + Ok(()) + } + + fn related_paths(&mut self, path: &str, authoritative: bool) -> Result, String> { + let sql = if authoritative { + "SELECT DISTINCT p.path AS path FROM pages p JOIN links l \ + ON p.path = l.target_path OR p.path = l.source_path \ + WHERE (l.source_path = ? OR l.target_path = ?) \ + AND p.path != ? AND p.status = 'verified' ORDER BY p.path LIMIT 12" + } else { + "SELECT DISTINCT p.path AS path FROM pages p JOIN links l \ + ON p.path = l.target_path OR p.path = l.source_path \ + WHERE (l.source_path = ? OR l.target_path = ?) \ + AND p.path != ? ORDER BY p.path LIMIT 12" + }; + diesel::sql_query(sql) + .bind::(path) + .bind::(path) + .bind::(path) + .load::(&mut self.connection) + .map(|rows| rows.into_iter().map(|row| row.path).collect()) + .map_err(|error| error.to_string()) + } +} + +fn ensure_contract(vault: &Path) -> Result<(), String> { + validate_vault(vault)?; + let present = ROOT_PAGES + .iter() + .filter(|path| vault.join(path).is_file()) + .count(); + if present == ROOT_PAGES.len() { + return Ok(()); + } + if present != 0 { + return Err( + "The selected vault has a partial Dev Brain contract. Provide all of purpose.md, schema.md, index.md, and log.md, or remove the conflicting files." + .into(), + ); + } + let staging = temporary_sibling(vault, "contract")?; + fs::create_dir(&staging).map_err(|error| error.to_string())?; + let result = (|| { + for (path, content) in [ + ("purpose.md", DEFAULT_PURPOSE), + ("schema.md", DEFAULT_SCHEMA), + ("index.md", DEFAULT_INDEX), + ("log.md", DEFAULT_LOG), + ] { + fs::write(staging.join(path), content).map_err(|error| error.to_string())?; + } + let changed = ROOT_PAGES + .iter() + .map(PathBuf::from) + .collect::>(); + commit_batch(vault, &staging, &changed) + })(); + let _ = fs::remove_dir_all(staging); + result +} + +pub(crate) fn validate_vault(path: &Path) -> Result { + let vault = path + .canonicalize() + .map_err(|error| format!("Could not open Dev Brain vault {}: {error}", path.display()))?; + if !vault.is_dir() || !vault.join(".obsidian").is_dir() { + return Err("Dev Brain vault must be an existing directory containing .obsidian.".into()); + } + Ok(vault) +} + +fn load_and_validate_pages( + content_root: &Path, + vault_root: &Path, + projects: &[RegisteredProject], + publishing: bool, +) -> Result, String> { + let parser = Parser::new(vault_root.to_owned()); + let mut paths = ROOT_PAGES.iter().map(PathBuf::from).collect::>(); + for directory in TOPIC_DIRS { + collect_managed_topics(content_root, &content_root.join(directory), &mut paths)?; + } + paths.sort(); + paths.dedup(); + let mut pages = Vec::with_capacity(paths.len()); + for path in paths { + let full_path = content_root.join(&path); + let content = fs::read_to_string(&full_path) + .map_err(|error| format!("Could not read candidate {}: {error}", path.display()))?; + let parsed = parser + .parse_file(&path, &content) + .map_err(|error| format!("Could not parse {}: {error}", path.display()))?; + let headings = parsed + .headings + .iter() + .map(|heading| heading.text.clone()) + .collect::>(); + let title = headings + .first() + .cloned() + .or_else(|| { + path.file_stem() + .and_then(|stem| stem.to_str()) + .map(str::to_owned) + }) + .ok_or_else(|| format!("Page has no valid identity: {}", path.display()))?; + let links = parsed + .links + .iter() + .filter(|link| { + matches!( + link.type_, + LinkType::WikiLink + | LinkType::Embed + | LinkType::BlockRef + | LinkType::HeadingRef + | LinkType::Anchor + ) + }) + .map(|link| RawLink { + target: link.target.clone(), + kind: link.type_, + line: link.position.line, + }) + .collect(); + let tags = parsed.tags.iter().map(|tag| tag.name.clone()).collect(); + let relative = path_to_string(&path); + if ROOT_PAGES.contains(&relative.as_str()) { + pages.push(Page { + path: relative, + title, + page_type: path.file_stem().unwrap().to_string_lossy().into_owned(), + status: "verified".into(), + verified_at: String::new(), + headings, + tags, + links, + sources: Vec::new(), + body: to_plain_text(&content), + content, + }); + continue; + } + let frontmatter = parsed + .frontmatter + .ok_or_else(|| format!("{relative} is missing YAML frontmatter."))?; + let frontmatter = serde_json::from_value::(Value::Object( + frontmatter.data.into_iter().collect(), + )) + .map_err(|error| format!("Invalid frontmatter in {relative}: {error}"))?; + if !frontmatter.dev_brain { + return Err(format!( + "{relative} is not declared as a managed Dev Brain page." + )); + } + validate_page_type(&path, &frontmatter.page_type)?; + if !matches!( + frontmatter.status.as_str(), + "verified" | "stale" | "needs-review" + ) { + return Err(format!( + "Invalid status in {relative}: {}", + frontmatter.status + )); + } + OffsetDateTime::parse(&frontmatter.verified_at, &Rfc3339) + .map_err(|_| format!("{relative} verified_at must be an RFC 3339 timestamp."))?; + if frontmatter.sources.is_empty() { + return Err(format!("{relative} must cite at least one project source.")); + } + if !projects + .iter() + .any(|project| project.name == frontmatter.project) + { + return Err(format!( + "{relative} names an unregistered project: {}", + frontmatter.project + )); + } + if !frontmatter + .sources + .iter() + .any(|source| source.project == frontmatter.project) + { + return Err(format!( + "{relative} must cite evidence from its declared project {}.", + frontmatter.project + )); + } + let fresh = frontmatter + .sources + .iter() + .map(|source| validate_source(source, projects)) + .collect::, _>>()? + .into_iter() + .all(|fresh| fresh); + if publishing && frontmatter.status == "verified" && !fresh { + return Err(format!( + "{relative} cannot be published as verified because its evidence has drifted." + )); + } + let status = if frontmatter.status == "verified" && !fresh { + "stale".into() + } else { + frontmatter.status + }; + pages.push(Page { + path: relative, + title, + page_type: frontmatter.page_type, + status, + verified_at: frontmatter.verified_at, + headings, + tags, + links, + sources: frontmatter.sources, + body: to_plain_text(&content), + content, + }); + } + let mut identities = HashMap::new(); + for page in &pages { + for identity in [ + page.title.to_lowercase(), + Path::new(&page.path) + .file_stem() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(), + ] { + if identities + .insert(identity.clone(), page.path.clone()) + .is_some_and(|existing| existing != page.path) + { + return Err(format!("Duplicate Dev Brain page identity: {identity}")); + } + } + } + Ok(pages) +} + +fn validate_graph( + pages: &[Page], + content_root: &Path, + vault_root: &Path, +) -> Result, String> { + let by_path = pages + .iter() + .map(|page| (page.path.clone(), page)) + .collect::>(); + let mut identities: HashMap> = HashMap::new(); + for page in pages { + for identity in [ + page.title.to_lowercase(), + Path::new(&page.path) + .file_stem() + .unwrap_or_default() + .to_string_lossy() + .to_lowercase(), + ] { + identities.entry(identity).or_default().push(page); + } + } + let mut edges = Vec::new(); + for page in pages { + for link in &page.links { + match resolve_link(page, link, &by_path, &identities, content_root, vault_root)? { + Some(target) => edges.push(( + page.path.clone(), + target, + if link.kind == LinkType::Embed { + "embed".into() + } else { + "wikilink".into() + }, + )), + None if link.kind == LinkType::Anchor => {} + None => { + return Err(format!( + "Broken link in {} at line {}: [[{}]]", + page.path, link.line, link.target + )); + } + } + } + } + let indexed = edges + .iter() + .filter(|(source, target, _)| source == "index.md" && is_topic_path(Path::new(target))) + .map(|(_, target, _)| target.clone()) + .collect::>(); + let topics = pages + .iter() + .filter(|page| is_topic_path(Path::new(&page.path))) + .map(|page| page.path.clone()) + .collect::>(); + if indexed != topics { + let missing = topics.difference(&indexed).cloned().collect::>(); + let extra = indexed.difference(&topics).cloned().collect::>(); + return Err(format!( + "index.md does not match managed topic pages. Missing: [{}]. Extra: [{}].", + missing.join(", "), + extra.join(", ") + )); + } + Ok(edges) +} + +fn resolve_link( + source: &Page, + link: &RawLink, + by_path: &HashMap, + identities: &HashMap>, + content_root: &Path, + vault_root: &Path, +) -> Result, String> { + let (target, heading) = link + .target + .split_once('#') + .map_or((link.target.as_str(), None), |(target, heading)| { + (target, Some(heading)) + }); + if target.is_empty() { + if let Some(heading) = heading + && !heading.starts_with('^') + && !source + .headings + .iter() + .any(|candidate| candidate.eq_ignore_ascii_case(heading)) + { + return Ok(None); + } + return Ok(Some(source.path.clone())); + } + if Path::new(target) + .extension() + .is_some_and(|extension| extension != "md") + { + let path = normalize_link_path(Path::new(&source.path), target)?; + for root in [content_root, vault_root] { + let full = root.join(&path); + if full.is_file() + && full + .canonicalize() + .is_ok_and(|resolved| resolved.starts_with(vault_root)) + { + return Ok(Some(path_to_string(&path))); + } + } + return Ok(None); + } + let mut candidates = Vec::new(); + if target.contains('/') || target.ends_with(".md") { + for path in [ + normalize_link_path(Path::new("index.md"), target)?, + normalize_link_path(Path::new(&source.path), target)?, + ] { + let path = path_to_string(&path); + if let Some(page) = by_path.get(&path) + && !candidates + .iter() + .any(|candidate: &&Page| candidate.path == page.path) + { + candidates.push(*page); + } + } + } else if let Some(matches) = identities.get(&target.to_lowercase()) { + candidates.extend(matches.iter().copied()); + } + if candidates.len() != 1 { + return Ok(None); + } + let target = candidates[0]; + if let Some(heading) = heading + && !heading.starts_with('^') + && !target + .headings + .iter() + .any(|candidate| candidate.eq_ignore_ascii_case(heading)) + { + return Ok(None); + } + Ok(Some(target.path.clone())) +} + +fn validate_source(source: &SourceRecord, projects: &[RegisteredProject]) -> Result { + let project = projects + .iter() + .find(|project| project.name == source.project) + .ok_or_else(|| format!("Source names an unregistered project: {}", source.project))?; + let relative = normalize_source_path(&source.path)?; + let full = project.root.join(&relative); + let resolved = full.canonicalize().map_err(|error| { + format!( + "Source does not exist {}:{}: {error}", + source.project, source.path + ) + })?; + if !resolved.starts_with(&project.root) || !resolved.is_file() { + return Err(format!( + "Source escapes its registered project: {}:{}", + source.project, source.path + )); + } + if source.revision.is_some() == source.hash.is_some() { + return Err(format!( + "Source must contain exactly one revision or hash: {}:{}", + source.project, source.path + )); + } + if let Some(revision) = &source.revision { + return Ok( + git_state(&project.root).is_some_and(|(current, clean)| clean && current == *revision) + ); + } + let expected = source.hash.as_deref().unwrap(); + if expected.len() != 64 + || !expected + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(format!( + "Source hash must be lowercase SHA-256: {}:{}", + source.project, source.path + )); + } + Ok(hash_file(&resolved)? == expected + && !git_state(&project.root).is_some_and(|(_, clean)| clean)) +} + +fn git_state(root: &Path) -> Option<(String, bool)> { + let revision = Command::new("git") + .args(["-C"]) + .arg(root) + .args(["rev-parse", "HEAD"]) + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())?; + let status = Command::new("git") + .args(["-C"]) + .arg(root) + .args(["status", "--porcelain"]) + .output() + .ok()?; + Some(( + revision, + status.status.success() && status.stdout.is_empty(), + )) +} + +fn collect_managed_topics( + root: &Path, + directory: &Path, + paths: &mut Vec, +) -> Result<(), String> { + let entries = match fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.to_string()), + }; + for entry in entries { + let entry = entry.map_err(|error| error.to_string())?; + let metadata = fs::symlink_metadata(entry.path()).map_err(|error| error.to_string())?; + if metadata.file_type().is_symlink() || entry.file_name().to_string_lossy().starts_with('.') + { + continue; + } + if metadata.is_dir() { + collect_managed_topics(root, &entry.path(), paths)?; + } else if entry + .path() + .extension() + .is_some_and(|extension| extension == "md") + { + let content = fs::read_to_string(entry.path()).map_err(|error| error.to_string())?; + if is_managed_topic(&content) { + paths.push( + entry + .path() + .strip_prefix(root) + .map_err(|error| error.to_string())? + .to_owned(), + ); + } + } + } + Ok(()) +} + +fn managed_files(vault: &Path) -> Result, String> { + let mut paths = ROOT_PAGES.iter().map(PathBuf::from).collect::>(); + for directory in TOPIC_DIRS { + collect_managed_topics(vault, &vault.join(directory), &mut paths)?; + } + Ok(paths.into_iter().collect()) +} + +fn is_managed_topic(content: &str) -> bool { + Parser::new(PathBuf::new()) + .parse_file(Path::new("topic.md"), content) + .ok() + .and_then(|parsed| parsed.frontmatter) + .and_then(|frontmatter| frontmatter.data.get("dev_brain").and_then(Value::as_bool)) + == Some(true) +} + +fn validate_page_type(path: &Path, page_type: &str) -> Result<(), String> { + let expected = match path.components().next() { + Some(Component::Normal(directory)) => directory.to_string_lossy(), + _ => return Err(format!("Invalid managed page path: {}", path.display())), + }; + let valid = match page_type { + "project" => expected == "projects", + "subsystem" => expected == "subsystems", + "concept" => expected == "concepts", + "decision" => expected == "decisions", + "invariant" => expected == "invariants", + "workflow" => expected == "workflows", + _ => false, + }; + valid.then_some(()).ok_or_else(|| { + format!( + "Page type {page_type} does not match managed path {}.", + path.display() + ) + }) +} + +fn normalize_managed_path(value: &str) -> Result { + let path = normalize_relative_path(value)?; + if path.extension().is_none_or(|extension| extension != "md") { + return Err(format!("Dev Brain manages Markdown files only: {value}")); + } + if path + .components() + .any(|component| component.as_os_str().to_string_lossy().starts_with('.')) + { + return Err(format!( + "Hidden paths are outside Dev Brain ownership: {value}" + )); + } + let display = path_to_string(&path); + if ROOT_PAGES.contains(&display.as_str()) || is_topic_path(&path) { + Ok(path) + } else { + Err(format!( + "Path is outside the managed Dev Brain boundary: {value}" + )) + } +} + +fn normalize_source_path(value: &str) -> Result { + normalize_relative_path(value) + .map_err(|_| format!("Invalid project-relative source path: {value}")) +} + +fn normalize_relative_path(value: &str) -> Result { + let path = Path::new(value); + if path.as_os_str().is_empty() || path.is_absolute() { + return Err(format!("Path must be relative: {value}")); + } + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Normal(part) => normalized.push(part), + Component::CurDir => {} + _ => return Err(format!("Path contains traversal: {value}")), + } + } + if normalized.as_os_str().is_empty() { + Err(format!("Path must not be empty: {value}")) + } else { + Ok(normalized) + } +} + +fn normalize_link_path(source: &Path, target: &str) -> Result { + let mut path = if target.contains('/') { + normalize_relative_path(target)? + } else { + source + .parent() + .unwrap_or_else(|| Path::new("")) + .join(normalize_relative_path(target)?) + }; + if path.extension().is_none() { + path.set_extension("md"); + } + Ok(path) +} + +fn is_topic_path(path: &Path) -> bool { + path.components().next().is_some_and(|component| { + let directory = component.as_os_str().to_string_lossy(); + TOPIC_DIRS.contains(&directory.as_ref()) + }) +} + +fn path_to_string(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +fn temporary_sibling(vault: &Path, label: &str) -> Result { + let parent = vault + .parent() + .ok_or_else(|| "Dev Brain vault has no parent directory.".to_owned())?; + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + Ok(parent.join(format!( + ".ds4-dev-brain-{label}-{}-{stamp}", + std::process::id() + ))) +} + +fn commit_batch(vault: &Path, staging: &Path, changed: &BTreeSet) -> Result<(), String> { + for path in changed { + validate_destination(vault, path)?; + } + let backup = temporary_sibling(vault, "backup")?; + fs::create_dir(&backup).map_err(|error| error.to_string())?; + let mut applied = Vec::new(); + let result = (|| { + for path in changed { + let live = vault.join(path); + let old = backup.join(path); + let candidate = staging.join(path); + if live.exists() { + if let Some(parent) = old.parent() { + fs::create_dir_all(parent).map_err(|error| error.to_string())?; + } + fs::rename(&live, &old).map_err(|error| error.to_string())?; + } + if candidate.exists() { + if let Some(parent) = live.parent() { + fs::create_dir_all(parent).map_err(|error| error.to_string())?; + } + if let Err(error) = fs::rename(&candidate, &live) { + if old.exists() { + let _ = fs::rename(&old, &live); + } + return Err(error.to_string()); + } + } + applied.push(path.clone()); + } + Ok(()) + })(); + if let Err(error) = result { + for path in applied.into_iter().rev() { + let live = vault.join(&path); + let old = backup.join(&path); + if live.exists() { + let _ = fs::remove_file(&live); + } + if old.exists() { + if let Some(parent) = live.parent() { + let _ = fs::create_dir_all(parent); + } + let _ = fs::rename(old, live); + } + } + let _ = fs::remove_dir_all(&backup); + return Err(format!( + "Dev Brain publication failed; the previous managed files were restored: {error}" + )); + } + let _ = fs::remove_dir_all(&backup); + Ok(()) +} + +fn validate_destination(vault: &Path, path: &Path) -> Result<(), String> { + let vault = vault + .canonicalize() + .map_err(|error| format!("Could not resolve Dev Brain vault: {error}"))?; + let mut current = vault.clone(); + for component in path.components() { + let Component::Normal(component) = component else { + return Err(format!("Invalid managed path: {}", path.display())); + }; + current.push(component); + match fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "Managed Dev Brain path crosses a symbolic link: {}", + path.display() + )); + } + Ok(_) => { + let resolved = current.canonicalize().map_err(|error| error.to_string())?; + if !resolved.starts_with(&vault) { + return Err(format!( + "Managed Dev Brain path escapes the vault: {}", + path.display() + )); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.to_string()), + } + } + Ok(()) +} + +fn vault_fingerprint(vault: &Path) -> Result, String> { + let mut fingerprints = managed_files(vault)? + .into_iter() + .map(|path| { + let full = vault.join(&path); + let metadata = full + .metadata() + .map_err(|error| format!("Could not inspect {}: {error}", path.display()))?; + let modified_nanos = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_nanos()); + Ok(Fingerprint { + path: path_to_string(&path), + modified_nanos, + size: metadata.len(), + hash: hash_file(&full)?, + }) + }) + .collect::, String>>()?; + fingerprints.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(fingerprints) +} + +fn hash_file(path: &Path) -> Result { + let data = + fs::read(path).map_err(|error| format!("Could not hash {}: {error}", path.display()))?; + Ok(hash_bytes(&data)) +} + +fn hash_bytes(data: &[u8]) -> String { + let digest = Sha256::digest(data); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn fts_query(query: &str) -> String { + query + .split_whitespace() + .filter(|term| !term.is_empty()) + .map(|term| format!("\"{}\"", term.replace('"', "\"\""))) + .collect::>() + .join(" OR ") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + static FIXTURE_ID: AtomicU64 = AtomicU64::new(1); + + struct Fixture { + root: PathBuf, + vault: PathBuf, + project: PathBuf, + } + + impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "ds4-dev-brain-{}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(), + FIXTURE_ID.fetch_add(1, Ordering::Relaxed) + )); + let vault = root.join("vault"); + let project = root.join("project"); + fs::create_dir_all(vault.join(".obsidian")).unwrap(); + fs::create_dir_all(&project).unwrap(); + fs::write(project.join("source.rs"), "pub fn answer() -> u8 { 42 }\n").unwrap(); + Self { + root, + vault, + project, + } + } + + fn config(&self) -> DevBrainConfig { + DevBrainConfig { + enabled: true, + vault_path: Some(self.vault.to_string_lossy().into_owned()), + } + } + + fn projects(&self) -> Vec { + vec![Project { + id: 1, + name: "Fixture".into(), + path: self.project.to_string_lossy().into_owned(), + collapsed: false, + }] + } + + fn brain(&self) -> DevBrain { + DevBrain::open(&self.config(), &self.projects()).unwrap() + } + + fn topic(&self, status: &str, hash: &str, body: &str) -> String { + format!( + "---\ndev_brain: true\ntype: concept\nproject: Fixture\nstatus: {status}\nverified_at: 2026-07-27T12:00:00Z\nsources:\n - project: Fixture\n path: source.rs\n hash: {hash}\n---\n\n# Answer\n\n{body}\n" + ) + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } + } + + fn source_hash(fixture: &Fixture) -> String { + hash_file(&fixture.project.join("source.rs")).unwrap() + } + + fn publish_topic(brain: &mut DevBrain, topic: &str) -> Result { + let mut files = Map::new(); + files.insert("concepts/answer.md".into(), Value::String(topic.into())); + files.insert( + "index.md".into(), + Value::String( + "# Dev Brain index\n\n- [[concepts/answer|Answer]] — Test answer.\n".into(), + ), + ); + files.insert( + "log.md".into(), + Value::String(format!("{DEFAULT_LOG}\n- Added answer.\n")), + ); + brain.publish(&files, &[]) + } + + #[test] + fn path_confinement_and_unrelated_notes_are_preserved() { + let fixture = Fixture::new(); + let mut brain = fixture.brain(); + fs::create_dir_all(fixture.vault.join("concepts")).unwrap(); + fs::write(fixture.vault.join("concepts/private.md"), "# Private\n").unwrap(); + let mut files = Map::new(); + files.insert("../escape.md".into(), Value::String("bad".into())); + assert!(brain.publish(&files, &[]).is_err()); + assert_eq!( + fs::read_to_string(fixture.vault.join("concepts/private.md")).unwrap(), + "# Private\n" + ); + + let outside = fixture.root.join("outside"); + fs::create_dir(&outside).unwrap(); + std::os::unix::fs::symlink(&outside, fixture.vault.join("concepts/escape")).unwrap(); + let mut files = Map::new(); + files.insert( + "concepts/escape/answer.md".into(), + Value::String(fixture.topic("verified", &source_hash(&fixture), "Confined.")), + ); + files.insert( + "index.md".into(), + Value::String( + "# Dev Brain index\n\n- [[concepts/escape/answer|Answer]] — Test.\n".into(), + ), + ); + files.insert( + "log.md".into(), + Value::String(format!("{DEFAULT_LOG}\n- Added confined answer.\n")), + ); + assert!(brain.publish(&files, &[]).is_err()); + assert!(!outside.join("answer.md").exists()); + } + + #[test] + fn parser_and_index_rebuild_ignore_links_in_code() { + let fixture = Fixture::new(); + let mut brain = fixture.brain(); + let topic = fixture.topic( + "verified", + &source_hash(&fixture), + "The durable answer is searchable.\n\n`[[Missing]]`\n\n```md\n[[Also Missing]]\n```", + ); + publish_topic(&mut brain, &topic).unwrap(); + let result = brain.search("durable searchable", 10, true).unwrap(); + assert!(result.contains("concepts/answer.md")); + assert!(result.contains("Fixture:source.rs")); + } + + #[test] + fn source_hash_drift_excludes_page_from_authoritative_queries() { + let fixture = Fixture::new(); + let mut brain = fixture.brain(); + publish_topic( + &mut brain, + &fixture.topic( + "verified", + &source_hash(&fixture), + "Unique drift knowledge.", + ), + ) + .unwrap(); + assert!( + brain + .search("Unique drift knowledge", 10, true) + .unwrap() + .contains("concepts/answer.md") + ); + fs::write(fixture.project.join("source.rs"), "changed\n").unwrap(); + assert!( + !brain + .search("Unique drift knowledge", 10, true) + .unwrap() + .contains("concepts/answer.md") + ); + assert!( + brain + .search("Unique drift knowledge", 10, false) + .unwrap() + .contains("status: stale") + ); + } + + #[test] + fn clean_git_revisions_become_stale_when_the_worktree_changes() { + let fixture = Fixture::new(); + for arguments in [ + vec!["init"], + vec!["add", "source.rs"], + vec![ + "-c", + "user.name=DS4Server", + "-c", + "user.email=ds4@example.invalid", + "commit", + "-m", + "Initial", + ], + ] { + assert!( + Command::new("git") + .arg("-C") + .arg(&fixture.project) + .args(arguments) + .status() + .unwrap() + .success() + ); + } + let revision = git_state(&fixture.project).unwrap().0; + let topic = fixture + .topic("verified", &"0".repeat(64), "Revision-backed knowledge.") + .replace( + &format!("hash: {}", "0".repeat(64)), + &format!("revision: {revision}"), + ); + let mut brain = fixture.brain(); + publish_topic(&mut brain, &topic).unwrap(); + assert!( + brain + .search("Revision-backed knowledge", 10, true) + .unwrap() + .contains("concepts/answer.md") + ); + fs::write(fixture.project.join("source.rs"), "dirty\n").unwrap(); + assert!( + brain + .search("Revision-backed knowledge", 10, false) + .unwrap() + .contains("status: stale") + ); + } + + #[test] + fn broken_links_and_invalid_provenance_leave_live_vault_untouched() { + let fixture = Fixture::new(); + let mut brain = fixture.brain(); + let index_before = fs::read_to_string(fixture.vault.join("index.md")).unwrap(); + let log_before = fs::read_to_string(fixture.vault.join("log.md")).unwrap(); + let bad_topic = fixture.topic( + "verified", + &source_hash(&fixture), + "This has a [[Missing Page]].", + ); + assert!(publish_topic(&mut brain, &bad_topic).is_err()); + assert_eq!( + fs::read_to_string(fixture.vault.join("index.md")).unwrap(), + index_before + ); + assert_eq!( + fs::read_to_string(fixture.vault.join("log.md")).unwrap(), + log_before + ); + assert!(!fixture.vault.join("concepts/answer.md").exists()); + + let invalid = fixture.topic("verified", &"0".repeat(64), "Wrong evidence."); + assert!(publish_topic(&mut brain, &invalid).is_err()); + assert!(!fixture.vault.join("concepts/answer.md").exists()); + + let escaped = fixture + .topic("verified", &source_hash(&fixture), "Escaped evidence.") + .replace("path: source.rs", "path: ../source.rs"); + assert!(publish_topic(&mut brain, &escaped).is_err()); + assert!(!fixture.vault.join("concepts/answer.md").exists()); + + let mut files = Map::new(); + files.insert( + "log.md".into(), + Value::String("# Dev Brain log\n\nRewritten history.\n".into()), + ); + assert!(brain.publish(&files, &[]).is_err()); + assert_eq!( + fs::read_to_string(fixture.vault.join("log.md")).unwrap(), + log_before + ); + } +} diff --git a/src/main.rs b/src/main.rs index df4d374..09209ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ mod app; mod compaction; mod config; mod database; +mod dev_brain; mod engine; mod metrics; mod model;