diff --git a/Cargo.lock b/Cargo.lock index 021b488..3341d0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,7 +68,7 @@ dependencies = [ "actix-rt", "actix-service", "actix-utils", - "base64", + "base64 0.22.1", "bitflags 2.11.0", "brotli", "bytes", @@ -250,6 +250,7 @@ dependencies = [ "cfg-if", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -777,6 +778,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -801,12 +808,13 @@ name = "bds-core" version = "0.1.0" dependencies = [ "axum", - "base64", + "base64 0.22.1", "chrono", "deunicode", "diesel", "diesel_migrations", "dirs 5.0.1", + "fastembed", "fluent-bundle", "fluent-syntax", "htmd", @@ -816,6 +824,7 @@ dependencies = [ "liquid", "liquid-core", "mlua", + "ort", "pagefind", "pulldown-cmark 0.13.3", "quick-xml 0.41.0", @@ -832,6 +841,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "url", + "usearch", "uuid", "walkdir", ] @@ -1188,6 +1198,15 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.58" @@ -1337,6 +1356,17 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width 0.2.2", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -1390,6 +1420,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "compression-codecs" version = "0.4.37" @@ -1495,10 +1540,29 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" dependencies = [ + "percent-encoding", "time", "version_check", ] +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie 0.18.1", + "document-features", + "idna", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1684,6 +1748,68 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "cxx" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe442a792c7c736eea18b32a7f8a3b63cf8aafabda6760042dc2fdeda456291" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash 0.2.0", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3184a94384c663718698311a78a51ac00c484c10b4eeac06fb0a068c5f64fa2" +dependencies = [ + "cc", + "codespan-reporting 0.13.1", + "indexmap 2.13.0", + "proc-macro2", + "quote", + "scratch", + "syn 3.0.0", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0148d8fd1199329ddf1d157a5e134e51ceff37c6a7ddd38615c399d81cb05d8d" +dependencies = [ + "clap", + "codespan-reporting 0.13.1", + "indexmap 2.13.0", + "proc-macro2", + "quote", + "syn 3.0.0", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52850339faed2eaadd24e286dc1d8268cc6f8a7bd9524d713adc9099566b4c89" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c77c856545d886c9bd5215409ebb63b925e262135248b50c79e5a5f194ee47c" +dependencies = [ + "indexmap 2.13.0", + "proc-macro2", + "quote", + "syn 3.0.0", +] + [[package]] name = "d3d12" version = "0.19.0" @@ -1711,14 +1837,38 @@ dependencies = [ "zbus 4.4.0", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", ] [[package]] @@ -1735,17 +1885,37 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core", + "darling_core 0.21.3", "quote", "syn 2.0.117", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "data-url" version = "0.3.2" @@ -1788,6 +1958,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1985,6 +2186,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dom_query" version = "0.27.0" @@ -2064,7 +2274,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd122633e4bef06db27737f21d3738fb89c8f6d5360d6d9d7635dda142a7757e" dependencies = [ - "darling", + "darling 0.21.3", "either", "heck 0.5.0", "proc-macro2", @@ -2212,6 +2422,12 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etagere" version = "0.2.15" @@ -2273,6 +2489,22 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2e7510819d6fbf51a5545c8f922716ecfb14df168a3242f7d33e0239efe6a1" +[[package]] +name = "fastembed" +version = "5.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c8600c9ec79b51d60c19911fe14eac04fe9c2895e87d2a3e80e2213d645a32" +dependencies = [ + "anyhow", + "hf-hub", + "ndarray", + "ort", + "safetensors", + "serde", + "serde_json", + "tokenizers", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -3114,6 +3346,32 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hf-hub" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" +dependencies = [ + "dirs 6.0.0", + "http 1.4.0", + "indicatif", + "libc", + "log", + "rand 0.9.2", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.18", + "ureq", + "windows-sys 0.61.2", +] + +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "htmd" version = "0.5.4" @@ -3250,7 +3508,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3705,6 +3963,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f007684f2e9727160da8b960ec161264703bfd1af084fd2e34d040c9a0dd4" +dependencies = [ + "console", + "portable-atomic", + "unicode-width 0.2.2", + "unit-prefix", + "web-time", +] + [[package]] name = "instant" version = "0.1.13" @@ -4185,6 +4456,15 @@ dependencies = [ "x11", ] +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -4269,6 +4549,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "local-channel" version = "0.1.5" @@ -4412,6 +4698,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -4450,6 +4758,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4625,6 +4943,28 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -4669,7 +5009,7 @@ checksum = "50e3524642f53d9af419ab5e8dd29d3ba155708267667c2f3f06c88c9e130843" dependencies = [ "bit-set 0.5.3", "bitflags 2.11.0", - "codespan-reporting", + "codespan-reporting 0.11.1", "hexf-parse", "indexmap 2.13.0", "log", @@ -4681,6 +5021,21 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.9.0" @@ -4774,6 +5129,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.1" @@ -4843,6 +5207,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "numkong" +version = "7.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58d4bb97df102ebdde66a352a20c0bc65c7c407ee9bef12ebe317edc2458a55" +dependencies = [ + "cc", +] + [[package]] name = "objc" version = "0.2.7" @@ -5338,6 +5711,30 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "owned_ttf_parser" version = "0.25.1" @@ -5357,7 +5754,7 @@ dependencies = [ "actix-web", "anyhow", "async-compression", - "base64", + "base64 0.22.1", "bit-set 0.10.0", "clap", "console", @@ -5736,7 +6133,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" dependencies = [ - "base64", + "base64 0.22.1", "indexmap 2.13.0", "quick-xml 0.38.4", "serde", @@ -5798,6 +6195,21 @@ dependencies = [ "nom 7.1.3", ] +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "portpicker" version = "0.1.1" @@ -6217,6 +6629,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.11.0" @@ -6227,6 +6645,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -6352,7 +6781,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -6376,12 +6805,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots", ] @@ -6569,6 +7000,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -6627,6 +7059,19 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safetensors" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b079b829cb27a1c3c374341345ed2e8b2c0c839034522cee576c140bd7f846" +dependencies = [ + "hashbrown 0.16.1", + "libc", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6648,6 +7093,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + [[package]] name = "sctk-adwaita" version = "0.10.1" @@ -7104,6 +7555,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "softbuffer" version = "0.4.8" @@ -7171,6 +7633,18 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlite-wasm-rs" version = "0.5.5" @@ -7296,6 +7770,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fac314a64dc9a36e61a9eb4261a5e9bbfbc922b27e518af97bc32b926cf967" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -7589,6 +8074,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash 0.8.12", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.2", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.50.0" @@ -8007,6 +8525,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -8049,6 +8576,18 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -8061,6 +8600,39 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http 1.4.0", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -8080,13 +8652,24 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "usearch" +version = "2.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd7f672d20412962c457b11c858c6c5aecb949808a5345a95e1d671112bcf72" +dependencies = [ + "cxx", + "cxx-build", + "numkong", +] + [[package]] name = "usvg" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b84ea542ae85c715f07b082438a4231c3760539d902e11d093847a0b22963032" dependencies = [ - "base64", + "base64 0.22.1", "data-url", "flate2", "fontdb 0.18.0", @@ -8119,6 +8702,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -8298,6 +8887,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-timer" version = "0.2.5" @@ -8637,7 +9239,7 @@ dependencies = [ "bit-vec 0.6.3", "bitflags 2.11.0", "cfg_aliases 0.1.1", - "codespan-reporting", + "codespan-reporting 0.11.1", "indexmap 2.13.0", "log", "naga", @@ -9440,7 +10042,7 @@ version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3013fd6116aac351dd2e18f349b28b2cfef3a5ff3253a9d0ce2d7193bb1b4429" dependencies = [ - "base64", + "base64 0.22.1", "block2 0.6.2", "cookie 0.18.1", "crossbeam-channel", diff --git a/Cargo.toml b/Cargo.toml index b43ff8d..8b502f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,9 @@ diesel_migrations = "2.3.2" libsqlite3-sys = { version = "0.37.0", features = ["bundled"] } mlua = { version = "0.12", features = ["lua54", "vendored", "serialize"] } url = "2.5" +fastembed = { version = "5.17.3", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"] } +usearch = "2.26" +ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "api-24", "coreml", "directml"] } # UI framework iced = { version = "0.13", features = ["wgpu", "advanced", "canvas", "image", "svg", "tokio", "markdown"] } diff --git a/README.md b/README.md index 548a916..6a30e80 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ The project is under active development. Core blogging workflows are broadly ava - WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping. - Template and Lua script management with explicit syntax-check feedback, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms. - SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search. +- Optional on-device multilingual semantic search and tag suggestions backed by a persistent USearch index, with duplicate-post review and dismissal in the desktop workspace. - Project-scoped typed domain events synchronize desktop views with shared-engine and future CLI mutations; persisted CLI notifications are consumed once, and the selected UI language is shared through settings. - Headless `bds-cli` automation for rebuild/repair/render, publishing and Git sync, post/media/gallery creation, shared settings/projects, utility Lua tasks, JSON I/O, airplane-mode AI routing, and guarded launcher installation from Settings → Data or `bds-cli install`. - Local MCP automation over stdio or a localhost-only stateless HTTP endpoint, with project resources, read/search/count tools, inert write proposals, explicit desktop approval, and opt-in Claude Code/Copilot configuration. diff --git a/RUST_PLAN_CORE.md b/RUST_PLAN_CORE.md index 3a11253..8588441 100644 --- a/RUST_PLAN_CORE.md +++ b/RUST_PLAN_CORE.md @@ -132,7 +132,7 @@ Available: - Fixed `.lua` script file contract. - Generated and bundled API/type references plus executable macro, transform, and utility examples under `docs/scripting/`. - Manifest-driven runtime, documentation, and completion data with drift checks. -- `bds.sync` and `bds.embeddings` remain extension APIs and are not exposed by core. +- The embedding extension adds the bDS2-compatible `bds.embeddings` namespace; `bds.sync` remains outside the core API. ## Remaining Core Blocks diff --git a/RUST_PLAN_EXTENSION.md b/RUST_PLAN_EXTENSION.md index b35b807..c0e40cf 100644 --- a/RUST_PLAN_EXTENSION.md +++ b/RUST_PLAN_EXTENSION.md @@ -42,17 +42,13 @@ Done: - Fixed native A2UI cards, charts, forms, lists, metrics, mind maps, tables, and tabs with safe nested fallbacks, persistent stable-ID interaction state, debounced form values, and strictly allowlisted application actions. - Online/airplane endpoint routing uses the shared secure endpoint and model infrastructure; unavailable modes direct the user to the existing localized AI settings. -### Embeddings, semantic search, and duplicates — Open +### Embeddings, semantic search, and duplicates — Done -Existing source contains schema placeholders only. - -Open: - -- Embedding generation and persistence. -- Vector index and semantic search. -- Expose `bds.embeddings` after those engines exist. -- Duplicate detection, dismissed-pair handling, metadata integrity, and UI. -- Replace the Find Duplicates placeholder. +- Lazy, locally cached `multilingual-e5-small` inference with 384-dimensional mean-pooled, normalized vectors and native Core ML acceleration on macOS or DirectML on Windows. +- Project-isolated USearch HNSW indexes with SQLite BLOB vectors as the recovery source, debounced persistence, lifecycle updates, rebuild/backfill, and CLI repair. +- Semantic sidebar search, link ranking, editor tag suggestions, and the bDS2-compatible `bds.embeddings` Lua API. +- Localized duplicate review with exact-match detection, 500-pair pagination, canonical single/batch dismissal, and post navigation. +- Embedding-aware filesystem rebuild and metadata diff/repair, settings gating, project-switch/shutdown flushes, and native menu commands. ### Translation QA and documentation UX — Partly done diff --git a/crates/bds-cli/src/lib.rs b/crates/bds-cli/src/lib.rs index f30844a..78d94fe 100644 --- a/crates/bds-cli/src/lib.rs +++ b/crates/bds-cli/src/lib.rs @@ -317,12 +317,6 @@ fn active_project(db: &Database) -> Result<(Project, PathBuf)> { fn rebuild(db: &Database, incremental: bool) -> Result { let (project, data_dir) = active_project(db)?; - let metadata = engine::meta::read_project_json(&data_dir)?; - if metadata.semantic_similarity_enabled { - bail!( - "semantic similarity is enabled, but rebuild cannot continue without the real on-device embedding engine" - ); - } if incremental { let (applied, imported, failed) = cli_sync::run_cli_mutation(db.conn(), || { let report = @@ -490,9 +484,13 @@ fn repair(db: &Database, part: RepairPart) -> Result { RepairPart::Embeddings => { let metadata = engine::meta::read_project_json(&data_dir)?; if metadata.semantic_similarity_enabled { - bail!( - "semantic similarity is enabled, but no real on-device embedding engine is registered" - ); + let service = engine::embedding::EmbeddingService::production(db.conn(), &data_dir); + let indexed = service.reindex_all(&project.id)?; + service.flush_project(&project.id)?; + return Ok(output( + "Embedding index rebuilt", + json!({"rebuilt": indexed.len(), "disabled": false}), + )); } Ok(output( "Embedding repair skipped because semantic similarity is disabled", @@ -1431,8 +1429,9 @@ mod tests { let mut metadata = engine::meta::read_project_json(&fixture.project_dir).unwrap(); metadata.semantic_similarity_enabled = true; engine::meta::write_project_json(&fixture.project_dir, &metadata).unwrap(); - assert!(fixture.run(&["repair", "embeddings"], "").is_err()); - assert!(fixture.run(&["rebuild"], "").is_err()); + let repaired = fixture.run(&["repair", "embeddings"], "").unwrap(); + assert_eq!(repaired.data["rebuilt"], 0); + fixture.run(&["rebuild"], "").unwrap(); metadata.semantic_similarity_enabled = false; engine::meta::write_project_json(&fixture.project_dir, &metadata).unwrap(); fixture.run(&["render"], "").unwrap(); diff --git a/crates/bds-core/Cargo.toml b/crates/bds-core/Cargo.toml index 18fb924..2c81053 100644 --- a/crates/bds-core/Cargo.toml +++ b/crates/bds-core/Cargo.toml @@ -38,6 +38,9 @@ mlua = { workspace = true } url = { workspace = true } base64 = { workspace = true } dirs = { workspace = true } +fastembed = { workspace = true } +usearch = { workspace = true } +ort = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/crates/bds-core/migrations/20260718000000_initial_schema/up.sql b/crates/bds-core/migrations/20260718000000_initial_schema/up.sql index 47f5fb0..63de6ad 100644 --- a/crates/bds-core/migrations/20260718000000_initial_schema/up.sql +++ b/crates/bds-core/migrations/20260718000000_initial_schema/up.sql @@ -258,9 +258,12 @@ CREATE TABLE IF NOT EXISTS embedding_keys ( post_id TEXT NOT NULL, project_id TEXT NOT NULL, content_hash TEXT NOT NULL, - vector TEXT NOT NULL + vector BLOB NOT NULL ); +CREATE UNIQUE INDEX IF NOT EXISTS embedding_keys_project_post_idx + ON embedding_keys(project_id, post_id); + CREATE TABLE IF NOT EXISTS dismissed_duplicate_pairs ( id TEXT NOT NULL PRIMARY KEY, project_id TEXT NOT NULL REFERENCES projects(id), diff --git a/crates/bds-core/migrations/20260719090000_embedding_vector_blob/down.sql b/crates/bds-core/migrations/20260719090000_embedding_vector_blob/down.sql new file mode 100644 index 0000000..eba9ecb --- /dev/null +++ b/crates/bds-core/migrations/20260719090000_embedding_vector_blob/down.sql @@ -0,0 +1,15 @@ +-- This file should undo anything in `up.sql` +CREATE TABLE embedding_keys_old ( + label INTEGER NOT NULL PRIMARY KEY, + post_id TEXT NOT NULL, + project_id TEXT NOT NULL, + content_hash TEXT NOT NULL, + vector TEXT NOT NULL +); + +INSERT INTO embedding_keys_old (label, post_id, project_id, content_hash, vector) +SELECT label, post_id, project_id, content_hash, CAST(vector AS TEXT) +FROM embedding_keys; + +DROP TABLE embedding_keys; +ALTER TABLE embedding_keys_old RENAME TO embedding_keys; diff --git a/crates/bds-core/migrations/20260719090000_embedding_vector_blob/up.sql b/crates/bds-core/migrations/20260719090000_embedding_vector_blob/up.sql new file mode 100644 index 0000000..a6f6426 --- /dev/null +++ b/crates/bds-core/migrations/20260719090000_embedding_vector_blob/up.sql @@ -0,0 +1,18 @@ +-- Your SQL goes here +CREATE TABLE embedding_keys_new ( + label INTEGER NOT NULL PRIMARY KEY, + post_id TEXT NOT NULL, + project_id TEXT NOT NULL, + content_hash TEXT NOT NULL, + vector BLOB NOT NULL +); + +INSERT INTO embedding_keys_new (label, post_id, project_id, content_hash, vector) +SELECT label, post_id, project_id, content_hash, CAST(vector AS BLOB) +FROM embedding_keys; + +DROP TABLE embedding_keys; +ALTER TABLE embedding_keys_new RENAME TO embedding_keys; + +CREATE UNIQUE INDEX embedding_keys_project_post_idx + ON embedding_keys(project_id, post_id); diff --git a/crates/bds-core/src/db/migrations.rs b/crates/bds-core/src/db/migrations.rs index 3e14b9c..115522f 100644 --- a/crates/bds-core/src/db/migrations.rs +++ b/crates/bds-core/src/db/migrations.rs @@ -36,7 +36,7 @@ mod tests { let applied = db .conn() .with_migrations(|conn| conn.applied_migrations().unwrap().len()); - assert_eq!(applied, 6); + assert_eq!(applied, 7); } #[test] diff --git a/crates/bds-core/src/db/queries/embedding.rs b/crates/bds-core/src/db/queries/embedding.rs new file mode 100644 index 0000000..7d299a0 --- /dev/null +++ b/crates/bds-core/src/db/queries/embedding.rs @@ -0,0 +1,283 @@ +use diesel::prelude::*; + +use crate::db::DbConnection; +use crate::db::schema::{dismissed_duplicate_pairs, embedding_keys}; +use crate::model::{DismissedDuplicatePair, EmbeddingKey}; + +pub fn get_key_for_post( + conn: &DbConnection, + project_id: &str, + post_id: &str, +) -> QueryResult> { + conn.with(|c| { + embedding_keys::table + .filter(embedding_keys::project_id.eq(project_id)) + .filter(embedding_keys::post_id.eq(post_id)) + .select(EmbeddingKey::as_select()) + .first(c) + .optional() + }) +} + +pub fn list_keys(conn: &DbConnection, project_id: &str) -> QueryResult> { + conn.with(|c| { + embedding_keys::table + .filter(embedding_keys::project_id.eq(project_id)) + .order(embedding_keys::label.asc()) + .select(EmbeddingKey::as_select()) + .load(c) + }) +} + +pub fn max_label(conn: &DbConnection) -> QueryResult { + conn.with(|c| { + embedding_keys::table + .select(embedding_keys::label) + .order(embedding_keys::label.desc()) + .first(c) + .optional() + .map(|label| label.unwrap_or(0)) + }) +} + +pub fn upsert_key(conn: &DbConnection, key: &EmbeddingKey) -> QueryResult<()> { + conn.with(|c| { + diesel::insert_into(embedding_keys::table) + .values(key) + .on_conflict((embedding_keys::project_id, embedding_keys::post_id)) + .do_update() + .set(( + embedding_keys::content_hash.eq(&key.content_hash), + embedding_keys::vector.eq(&key.vector), + )) + .execute(c) + .map(|_| ()) + }) +} + +pub fn delete_key_for_post( + conn: &DbConnection, + project_id: &str, + post_id: &str, +) -> QueryResult<()> { + conn.with(|c| { + diesel::delete( + embedding_keys::table + .filter(embedding_keys::project_id.eq(project_id)) + .filter(embedding_keys::post_id.eq(post_id)), + ) + .execute(c) + .map(|_| ()) + }) +} + +pub fn delete_stale_keys( + conn: &DbConnection, + project_id: &str, + live_post_ids: &[String], +) -> QueryResult { + conn.with(|c| { + let query = embedding_keys::table.filter(embedding_keys::project_id.eq(project_id)); + if live_post_ids.is_empty() { + diesel::delete(query).execute(c) + } else { + diesel::delete(query.filter(embedding_keys::post_id.ne_all(live_post_ids))).execute(c) + } + }) +} + +pub fn insert_dismissed_pair( + conn: &DbConnection, + pair: &DismissedDuplicatePair, +) -> QueryResult<()> { + conn.with(|c| { + diesel::insert_into(dismissed_duplicate_pairs::table) + .values(pair) + .on_conflict(( + dismissed_duplicate_pairs::project_id, + dismissed_duplicate_pairs::post_id_a, + dismissed_duplicate_pairs::post_id_b, + )) + .do_nothing() + .execute(c) + .map(|_| ()) + }) +} + +pub fn insert_dismissed_pairs( + conn: &DbConnection, + pairs: &[DismissedDuplicatePair], +) -> QueryResult { + if pairs.is_empty() { + return Ok(0); + } + conn.with(|c| { + diesel::insert_into(dismissed_duplicate_pairs::table) + .values(pairs) + .on_conflict(( + dismissed_duplicate_pairs::project_id, + dismissed_duplicate_pairs::post_id_a, + dismissed_duplicate_pairs::post_id_b, + )) + .do_nothing() + .execute(c) + }) +} + +pub fn list_dismissed_pairs( + conn: &DbConnection, + project_id: &str, +) -> QueryResult> { + conn.with(|c| { + dismissed_duplicate_pairs::table + .filter(dismissed_duplicate_pairs::project_id.eq(project_id)) + .select(DismissedDuplicatePair::as_select()) + .load(c) + }) +} + +pub fn delete_orphan_dismissals( + conn: &DbConnection, + project_id: &str, + live_post_ids: &[String], +) -> QueryResult { + conn.with(|c| { + let query = dismissed_duplicate_pairs::table + .filter(dismissed_duplicate_pairs::project_id.eq(project_id)); + if live_post_ids.is_empty() { + diesel::delete(query).execute(c) + } else { + diesel::delete( + query.filter( + dismissed_duplicate_pairs::post_id_a + .ne_all(live_post_ids) + .or(dismissed_duplicate_pairs::post_id_b.ne_all(live_post_ids)), + ), + ) + .execute(c) + } + }) +} + +pub fn delete_dismissals_for_post( + conn: &DbConnection, + project_id: &str, + post_id: &str, +) -> QueryResult { + conn.with(|c| { + diesel::delete( + dismissed_duplicate_pairs::table + .filter(dismissed_duplicate_pairs::project_id.eq(project_id)) + .filter( + dismissed_duplicate_pairs::post_id_a + .eq(post_id) + .or(dismissed_duplicate_pairs::post_id_b.eq(post_id)), + ), + ) + .execute(c) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::Database; + use crate::model::{Post, PostStatus, Project}; + + fn seeded() -> (Database, String, String) { + let db = Database::open_in_memory().unwrap(); + db.migrate().unwrap(); + let project_id = "embedding-project".to_string(); + crate::db::queries::project::insert_project( + db.conn(), + &Project { + id: project_id.clone(), + name: "Embedding".into(), + slug: "embedding".into(), + description: None, + data_path: Some("/tmp/embedding".into()), + is_active: true, + created_at: 1, + updated_at: 1, + }, + ) + .unwrap(); + let post_id = "embedding-post".to_string(); + crate::db::queries::post::insert_post( + db.conn(), + &Post { + id: post_id.clone(), + project_id: project_id.clone(), + title: "Post".into(), + slug: "post".into(), + excerpt: None, + content: Some("Body".into()), + status: PostStatus::Draft, + author: None, + language: Some("en".into()), + do_not_translate: false, + template_slug: None, + file_path: "posts/post.md".into(), + checksum: None, + tags: vec![], + categories: vec![], + published_title: None, + published_content: None, + published_tags: None, + published_categories: None, + published_excerpt: None, + created_at: 1, + updated_at: 1, + published_at: None, + }, + ) + .unwrap(); + (db, project_id, post_id) + } + + #[test] + fn embedding_vector_round_trips_as_blob_and_dismissals_are_canonical() { + let (db, project_id, post_id) = seeded(); + let key = EmbeddingKey { + label: 1, + post_id: post_id.clone(), + project_id: project_id.clone(), + content_hash: "hash".into(), + vector: vec![0, 1, 255], + }; + upsert_key(db.conn(), &key).unwrap(); + assert_eq!( + get_key_for_post(db.conn(), &project_id, &post_id).unwrap(), + Some(key) + ); + + let replacement = EmbeddingKey { + label: 2, + post_id: post_id.clone(), + project_id: project_id.clone(), + content_hash: "new-hash".into(), + vector: vec![3, 2, 1], + }; + upsert_key(db.conn(), &replacement).unwrap(); + let updated = get_key_for_post(db.conn(), &project_id, &post_id) + .unwrap() + .unwrap(); + assert_eq!(updated.label, 1, "a post keeps its stable HNSW label"); + assert_eq!(updated.content_hash, "new-hash"); + assert_eq!(updated.vector, vec![3, 2, 1]); + + let pair = DismissedDuplicatePair { + id: "dismissal".into(), + project_id: project_id.clone(), + post_id_a: "a".into(), + post_id_b: "b".into(), + dismissed_at: 1, + }; + insert_dismissed_pair(db.conn(), &pair).unwrap(); + insert_dismissed_pair(db.conn(), &pair).unwrap(); + assert_eq!( + list_dismissed_pairs(db.conn(), &project_id).unwrap(), + vec![pair] + ); + } +} diff --git a/crates/bds-core/src/db/queries/mod.rs b/crates/bds-core/src/db/queries/mod.rs index e2b85b6..974b13c 100644 --- a/crates/bds-core/src/db/queries/mod.rs +++ b/crates/bds-core/src/db/queries/mod.rs @@ -1,5 +1,6 @@ pub mod chat; pub mod db_notification; +pub mod embedding; pub mod generated_file_hash; pub mod import_definition; pub mod mcp_proposal; diff --git a/crates/bds-core/src/db/schema.rs b/crates/bds-core/src/db/schema.rs index 2124a5b..fea662e 100644 --- a/crates/bds-core/src/db/schema.rs +++ b/crates/bds-core/src/db/schema.rs @@ -115,7 +115,7 @@ diesel::table! { post_id -> Text, project_id -> Text, content_hash -> Text, - vector -> Text, + vector -> Binary, } } diff --git a/crates/bds-core/src/engine/embedding.rs b/crates/bds-core/src/engine/embedding.rs new file mode 100644 index 0000000..48e0976 --- /dev/null +++ b/crates/bds-core/src/engine/embedding.rs @@ -0,0 +1,1382 @@ +//! Local multilingual semantic indexing and duplicate detection. +//! +//! SQLite vectors are authoritative. Per-project USearch files are disposable, +//! validated caches that are rebuilt from those vectors when absent or corrupt. + +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use fastembed::{EmbeddingModel, TextEmbedding, TextInitOptions}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use usearch::{Index, IndexOptions, MetricKind, ScalarKind}; + +use crate::db::DbConnection as Connection; +use crate::db::queries::{embedding as qe, post as qp}; +use crate::engine::{EngineError, EngineResult}; +use crate::model::{DismissedDuplicatePair, EmbeddingKey, Post}; +use crate::util::{application_data_dir, now_unix_ms}; + +pub const MODEL_ID: &str = "Xenova/multilingual-e5-small"; +pub const MODEL_REPOSITORY: &str = "intfloat/multilingual-e5-small"; +pub const DIMENSIONS: usize = 384; +pub const BATCH_SIZE: usize = 16; +pub const SEQUENCE_LENGTH: usize = 256; +pub const DUPLICATE_THRESHOLD: f32 = 0.92; +pub const DUPLICATE_PAGE_SIZE: usize = 500; +pub const DUPLICATE_NEIGHBOR_COUNT: usize = 21; +const SAVE_DEBOUNCE: Duration = Duration::from_secs(5); + +pub trait EmbeddingBackend: Send + Sync { + fn embed(&self, prefixed_texts: &[String]) -> Result>, String>; +} + +struct NeuralBackend { + model: Mutex>, + cache_dir: PathBuf, +} + +impl NeuralBackend { + fn global() -> &'static Arc { + static BACKEND: OnceLock> = OnceLock::new(); + BACKEND.get_or_init(|| { + Arc::new(Self { + model: Mutex::new(None), + cache_dir: application_data_dir().join("models"), + }) + }) + } + + fn options(&self) -> TextInitOptions { + let options = TextInitOptions::new(EmbeddingModel::MultilingualE5Small) + .with_cache_dir(self.cache_dir.clone()) + .with_max_length(SEQUENCE_LENGTH) + .with_show_download_progress(false); + #[cfg(target_os = "macos")] + let options = options + .with_execution_providers(vec![ort::ep::CoreML::default().build().fail_silently()]); + #[cfg(target_os = "windows")] + let options = options + .with_execution_providers(vec![ort::ep::DirectML::default().build().fail_silently()]); + options + } +} + +impl EmbeddingBackend for NeuralBackend { + fn embed(&self, prefixed_texts: &[String]) -> Result>, String> { + if prefixed_texts.is_empty() { + return Ok(Vec::new()); + } + let mut guard = self + .model + .lock() + .map_err(|_| "embedding model lock poisoned")?; + if guard.is_none() { + fs::create_dir_all(&self.cache_dir).map_err(|error| error.to_string())?; + *guard = Some( + TextEmbedding::try_new(self.options()) + .map_err(|error| format!("could not load {MODEL_REPOSITORY}: {error}"))?, + ); + } + guard + .as_mut() + .expect("model initialized") + .embed(prefixed_texts, Some(BATCH_SIZE)) + .map_err(|error| error.to_string()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SimilarPost { + pub post_id: String, + pub title: String, + pub similarity: f32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DuplicatePair { + pub post_id_a: String, + pub title_a: String, + pub post_id_b: String, + pub title_b: String, + pub similarity: f32, + pub exact_match: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] +pub struct DuplicateSearchResult { + pub pairs: Vec, + pub has_more: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +struct IndexMetadata { + dimensions: usize, + labels: Vec<(u64, String)>, +} + +struct CachedIndex { + index: Index, + labels: HashMap, + dirty_since: Option, + index_path: PathBuf, +} + +fn registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub struct EmbeddingService<'a> { + conn: &'a Connection, + data_dir: &'a Path, + cache_root: PathBuf, + backend: Arc, +} + +impl<'a> EmbeddingService<'a> { + pub fn production(conn: &'a Connection, data_dir: &'a Path) -> Self { + Self { + conn, + data_dir, + cache_root: application_data_dir(), + backend: NeuralBackend::global().clone(), + } + } + + pub fn with_backend( + conn: &'a Connection, + data_dir: &'a Path, + cache_root: PathBuf, + backend: Arc, + ) -> Self { + Self { + conn, + data_dir, + cache_root, + backend, + } + } + + pub fn enabled(&self) -> bool { + crate::engine::meta::read_project_json(self.data_dir) + .map(|metadata| metadata.semantic_similarity_enabled) + .unwrap_or(false) + } + + pub fn indexing_progress(&self, project_id: &str) -> EngineResult<(usize, usize)> { + let indexed = qe::list_keys(self.conn, project_id)?.len(); + let total = qp::list_posts_by_project(self.conn, project_id)?.len(); + Ok((indexed, total)) + } + + pub fn content_hash_for_post(&self, post: &Post) -> EngineResult { + self.embedding_text(post).map(|text| hash_text(&text)) + } + + pub fn sync_post(&self, post: &Post) -> EngineResult { + if !self.enabled() { + return Ok(false); + } + let text = self.embedding_text(post)?; + let content_hash = hash_text(&text); + if qe::get_key_for_post(self.conn, &post.project_id, &post.id)?.is_some_and(|key| { + key.content_hash == content_hash && decode_vector(&key.vector).is_ok() + }) { + return Ok(false); + } + let vector = self.embed_texts(&[text])?.pop().expect("one embedding"); + let existing = qe::get_key_for_post(self.conn, &post.project_id, &post.id)?; + let key = EmbeddingKey { + label: existing + .map(|key| key.label) + .unwrap_or(qe::max_label(self.conn)? + 1), + post_id: post.id.clone(), + project_id: post.project_id.clone(), + content_hash, + vector: encode_vector(&vector), + }; + qe::upsert_key(self.conn, &key)?; + self.rebuild_cached_index(&post.project_id, false)?; + Ok(true) + } + + pub fn remove_post(&self, project_id: &str, post_id: &str) -> EngineResult<()> { + qe::delete_key_for_post(self.conn, project_id, post_id)?; + qe::delete_dismissals_for_post(self.conn, project_id, post_id)?; + self.rebuild_cached_index(project_id, false) + } + + pub fn index_unindexed(&self, project_id: &str) -> EngineResult> { + self.index_unindexed_with_progress(project_id, |_, _| true) + } + + pub fn index_unindexed_with_progress( + &self, + project_id: &str, + mut on_progress: impl FnMut(usize, usize) -> bool, + ) -> EngineResult> { + if !self.enabled() { + return Ok(Vec::new()); + } + let posts = qp::list_posts_by_project(self.conn, project_id)?; + let live_ids = posts.iter().map(|post| post.id.clone()).collect::>(); + qe::delete_stale_keys(self.conn, project_id, &live_ids)?; + qe::delete_orphan_dismissals(self.conn, project_id, &live_ids)?; + + let existing = qe::list_keys(self.conn, project_id)? + .into_iter() + .map(|key| (key.post_id.clone(), key)) + .collect::>(); + let mut prepared = Vec::new(); + for post in &posts { + let text = self.embedding_text(post)?; + let hash = hash_text(&text); + if existing + .get(&post.id) + .is_none_or(|key| key.content_hash != hash || decode_vector(&key.vector).is_err()) + { + prepared.push((post, text, hash)); + } + } + let total = prepared.len(); + if !on_progress(0, total) { + self.rebuild_cached_index(project_id, false)?; + return Err(EngineError::Validation( + "embedding indexing cancelled".into(), + )); + } + let mut next_label = qe::max_label(self.conn)? + 1; + let mut completed = 0; + for chunk in prepared.chunks(BATCH_SIZE) { + let vectors = self.embed_texts( + &chunk + .iter() + .map(|(_, text, _)| text.clone()) + .collect::>(), + )?; + for ((post, _, hash), vector) in chunk.iter().zip(vectors) { + let label = existing + .get(&post.id) + .map(|key| key.label) + .unwrap_or_else(|| { + let label = next_label; + next_label += 1; + label + }); + qe::upsert_key( + self.conn, + &EmbeddingKey { + label, + post_id: post.id.clone(), + project_id: project_id.to_string(), + content_hash: hash.clone(), + vector: encode_vector(&vector), + }, + )?; + } + completed += chunk.len(); + if !on_progress(completed, total) { + self.rebuild_cached_index(project_id, false)?; + return Err(EngineError::Validation( + "embedding indexing cancelled".into(), + )); + } + } + self.rebuild_cached_index(project_id, false)?; + Ok(qe::list_keys(self.conn, project_id)? + .into_iter() + .map(|key| key.post_id) + .collect()) + } + + pub fn reindex_all(&self, project_id: &str) -> EngineResult> { + self.reindex_all_with_progress(project_id, |_, _| true) + } + + pub fn reindex_all_with_progress( + &self, + project_id: &str, + mut on_progress: impl FnMut(usize, usize) -> bool, + ) -> EngineResult> { + if !self.enabled() { + return Ok(Vec::new()); + } + let posts = qp::list_posts_by_project(self.conn, project_id)?; + let live_ids = posts.iter().map(|post| post.id.clone()).collect::>(); + qe::delete_stale_keys(self.conn, project_id, &live_ids)?; + qe::delete_orphan_dismissals(self.conn, project_id, &live_ids)?; + let existing = qe::list_keys(self.conn, project_id)? + .into_iter() + .map(|key| (key.post_id.clone(), key.label)) + .collect::>(); + let texts = posts + .iter() + .map(|post| self.embedding_text(post)) + .collect::>>()?; + let total = posts.len(); + if !on_progress(0, total) { + self.rebuild_cached_index(project_id, false)?; + return Err(EngineError::Validation( + "embedding indexing cancelled".into(), + )); + } + let mut next_label = qe::max_label(self.conn)? + 1; + let mut completed = 0; + for (post_chunk, text_chunk) in posts.chunks(BATCH_SIZE).zip(texts.chunks(BATCH_SIZE)) { + let vectors = self.embed_texts(text_chunk)?; + for ((post, text), vector) in post_chunk.iter().zip(text_chunk).zip(vectors) { + let label = existing.get(&post.id).copied().unwrap_or_else(|| { + let label = next_label; + next_label += 1; + label + }); + qe::upsert_key( + self.conn, + &EmbeddingKey { + label, + post_id: post.id.clone(), + project_id: project_id.to_string(), + content_hash: hash_text(text), + vector: encode_vector(&vector), + }, + )?; + } + completed += post_chunk.len(); + if !on_progress(completed, total) { + self.rebuild_cached_index(project_id, false)?; + return Err(EngineError::Validation( + "embedding indexing cancelled".into(), + )); + } + } + self.rebuild_cached_index(project_id, false)?; + Ok(live_ids) + } + + pub fn semantic_search( + &self, + project_id: &str, + query: &str, + limit: usize, + ) -> EngineResult> { + if !self.enabled() || query.trim().is_empty() || limit == 0 { + return Ok(Vec::new()); + } + self.index_unindexed(project_id)?; + let vector = self + .embed_texts(&[query.trim().to_string()])? + .pop() + .expect("one embedding"); + self.search_index(project_id, &vector, limit, None) + } + + pub fn find_similar(&self, post_id: &str, limit: usize) -> EngineResult> { + if !self.enabled() || limit == 0 { + return Ok(Vec::new()); + } + let post = qp::get_post_by_id(self.conn, post_id)?; + self.sync_post(&post)?; + let Some(key) = qe::get_key_for_post(self.conn, &post.project_id, post_id)? else { + return Ok(Vec::new()); + }; + self.search_index( + &post.project_id, + &decode_vector(&key.vector)?, + limit, + Some(key.label as u64), + ) + } + + pub fn compute_similarities( + &self, + source_post_id: &str, + target_post_ids: &[String], + ) -> EngineResult> { + if !self.enabled() { + return Ok(HashMap::new()); + } + let source = qp::get_post_by_id(self.conn, source_post_id)?; + self.sync_post(&source)?; + let Some(source_key) = qe::get_key_for_post(self.conn, &source.project_id, source_post_id)? + else { + return Ok(HashMap::new()); + }; + let source_vector = decode_vector(&source_key.vector)?; + let targets = target_post_ids.iter().collect::>(); + Ok(qe::list_keys(self.conn, &source.project_id)? + .into_iter() + .filter(|key| key.post_id != source_post_id && targets.contains(&key.post_id)) + .filter_map(|key| { + decode_vector(&key.vector) + .ok() + .map(|vector| (key.post_id, cosine(&source_vector, &vector))) + }) + .collect()) + } + + pub fn suggest_tags(&self, post_id: &str) -> EngineResult> { + let source = qp::get_post_by_id(self.conn, post_id)?; + let current = source + .tags + .iter() + .map(|tag| tag.to_lowercase()) + .collect::>(); + let similar = self.find_similar(post_id, 10)?; + let mut scores = HashMap::::new(); + for neighbor in similar { + if let Ok(post) = qp::get_post_by_id(self.conn, &neighbor.post_id) { + for tag in post.tags { + let normalized = tag.to_lowercase(); + if !current.contains(&normalized) { + scores + .entry(normalized) + .and_modify(|(_, score)| *score += neighbor.similarity) + .or_insert((tag, neighbor.similarity)); + } + } + } + } + let mut ranked = scores.into_values().collect::>(); + ranked.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + Ok(ranked.into_iter().take(5).map(|(tag, _)| tag).collect()) + } + + pub fn find_duplicates( + &self, + project_id: &str, + page: usize, + ) -> EngineResult { + if !self.enabled() { + return Ok(DuplicateSearchResult::default()); + } + self.index_unindexed(project_id)?; + let keys = qe::list_keys(self.conn, project_id)?; + let dismissed = qe::list_dismissed_pairs(self.conn, project_id)? + .into_iter() + .map(|pair| (pair.post_id_a, pair.post_id_b)) + .collect::>(); + let posts = qp::list_posts_by_project(self.conn, project_id)? + .into_iter() + .map(|post| (post.id.clone(), post)) + .collect::>(); + let mut seen = HashSet::new(); + let mut pairs = Vec::new(); + for key in &keys { + let vector = decode_vector(&key.vector)?; + for neighbor in self.search_raw( + project_id, + &vector, + DUPLICATE_NEIGHBOR_COUNT, + Some(key.label as u64), + )? { + if neighbor.1 < DUPLICATE_THRESHOLD { + continue; + } + let Some(other_id) = neighbor.0 else { continue }; + let (a, b) = canonical_pair(&key.post_id, &other_id); + if !seen.insert((a.clone(), b.clone())) + || dismissed.contains(&(a.clone(), b.clone())) + { + continue; + } + let (Some(post_a), Some(post_b)) = (posts.get(&a), posts.get(&b)) else { + continue; + }; + let exact_match = neighbor.1 >= 0.999_999 + && post_a.title == post_b.title + && self.post_body(post_a)? == self.post_body(post_b)?; + pairs.push(DuplicatePair { + post_id_a: a, + title_a: post_a.title.clone(), + post_id_b: b, + title_b: post_b.title.clone(), + similarity: neighbor.1, + exact_match, + }); + } + } + pairs.sort_by(|a, b| { + b.exact_match + .cmp(&a.exact_match) + .then_with(|| b.similarity.total_cmp(&a.similarity)) + .then_with(|| a.post_id_a.cmp(&b.post_id_a)) + .then_with(|| a.post_id_b.cmp(&b.post_id_b)) + }); + let end = page + .saturating_add(1) + .saturating_mul(DUPLICATE_PAGE_SIZE) + .min(pairs.len()); + let has_more = end < pairs.len(); + pairs.truncate(end); + Ok(DuplicateSearchResult { pairs, has_more }) + } + + pub fn dismiss_duplicate_pair(&self, post_id_a: &str, post_id_b: &str) -> EngineResult<()> { + let post_a = qp::get_post_by_id(self.conn, post_id_a)?; + let post_b = qp::get_post_by_id(self.conn, post_id_b)?; + if post_id_a == post_id_b || post_a.project_id != post_b.project_id { + return Err(EngineError::Validation( + "duplicate pair must contain two posts in one project".into(), + )); + } + let (a, b) = canonical_pair(post_id_a, post_id_b); + qe::insert_dismissed_pair( + self.conn, + &DismissedDuplicatePair { + id: uuid::Uuid::new_v4().to_string(), + project_id: post_a.project_id, + post_id_a: a, + post_id_b: b, + dismissed_at: now_unix_ms(), + }, + )?; + Ok(()) + } + + pub fn dismiss_duplicate_pairs(&self, pair_ids: &[(String, String)]) -> EngineResult<()> { + let post_ids = pair_ids + .iter() + .flat_map(|(a, b)| [a.as_str(), b.as_str()]) + .collect::>(); + let posts = post_ids + .into_iter() + .map(|post_id| { + qp::get_post_by_id(self.conn, post_id).map(|post| (post_id.to_string(), post)) + }) + .collect::, _>>()?; + let mut seen = HashSet::new(); + let mut dismissals = Vec::new(); + for (post_id_a, post_id_b) in pair_ids { + let post_a = &posts[post_id_a]; + let post_b = &posts[post_id_b]; + if post_id_a == post_id_b || post_a.project_id != post_b.project_id { + return Err(EngineError::Validation( + "duplicate pair must contain two posts in one project".into(), + )); + } + let (a, b) = canonical_pair(post_id_a, post_id_b); + if seen.insert((post_a.project_id.clone(), a.clone(), b.clone())) { + dismissals.push(DismissedDuplicatePair { + id: uuid::Uuid::new_v4().to_string(), + project_id: post_a.project_id.clone(), + post_id_a: a, + post_id_b: b, + dismissed_at: now_unix_ms(), + }); + } + } + for chunk in dismissals.chunks(100) { + qe::insert_dismissed_pairs(self.conn, chunk)?; + } + Ok(()) + } + + pub fn flush_project(&self, project_id: &str) -> EngineResult<()> { + let path = self.index_path(project_id); + let mut registry = registry() + .lock() + .map_err(|_| EngineError::Validation("embedding index lock poisoned".into()))?; + if let Some(cached) = registry.get_mut(&path) { + persist_cached(cached)?; + } + Ok(()) + } + + pub fn flush_due() -> EngineResult<()> { + let mut registry = registry() + .lock() + .map_err(|_| EngineError::Validation("embedding index lock poisoned".into()))?; + for cached in registry.values_mut() { + if cached + .dirty_since + .is_some_and(|at| at.elapsed() >= SAVE_DEBOUNCE) + { + persist_cached(cached)?; + } + } + Ok(()) + } + + pub fn flush_all() -> EngineResult<()> { + let mut registry = registry() + .lock() + .map_err(|_| EngineError::Validation("embedding index lock poisoned".into()))?; + for cached in registry.values_mut() { + persist_cached(cached)?; + } + Ok(()) + } + + pub fn forget_project(project_id: &str) { + let index_path = application_data_dir() + .join("projects") + .join(project_id) + .join("embeddings.usearch"); + if let Ok(mut indexes) = registry().lock() { + indexes.remove(&index_path); + } + if let Some(project_cache_dir) = index_path.parent() { + let _ = fs::remove_dir_all(project_cache_dir); + } + } + + fn embedding_text(&self, post: &Post) -> EngineResult { + Ok(format!("{}\n\n{}", post.title, self.post_body(post)?)) + } + + fn post_body(&self, post: &Post) -> EngineResult { + if let Some(content) = post.content.as_ref().filter(|content| !content.is_empty()) { + return Ok(content.clone()); + } + if post.file_path.is_empty() { + return Ok(String::new()); + } + let raw = match fs::read_to_string(self.data_dir.join(&post.file_path)) { + Ok(raw) => raw, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(String::new()), + Err(error) => return Err(error.into()), + }; + crate::util::frontmatter::read_post_file(&raw) + .map(|(_, body)| body) + .map_err(EngineError::Parse) + } + + fn embed_texts(&self, texts: &[String]) -> EngineResult>> { + if texts.is_empty() { + return Ok(Vec::new()); + } + let prefixed = texts + .iter() + .map(|text| format!("query: {text}")) + .collect::>(); + let vectors = self + .backend + .embed(&prefixed) + .map_err(EngineError::Validation)?; + if vectors.len() != texts.len() || vectors.iter().any(|vector| vector.len() != DIMENSIONS) { + return Err(EngineError::Validation(format!( + "{MODEL_ID} returned invalid embedding dimensions" + ))); + } + Ok(vectors.into_iter().map(normalize).collect()) + } + + fn search_index( + &self, + project_id: &str, + vector: &[f32], + limit: usize, + exclude: Option, + ) -> EngineResult> { + let raw = self.search_raw(project_id, vector, limit, exclude)?; + Ok(raw + .into_iter() + .filter_map(|(post_id, similarity)| { + let post_id = post_id?; + qp::get_post_by_id(self.conn, &post_id) + .ok() + .map(|post| SimilarPost { + post_id, + title: post.title, + similarity, + }) + }) + .collect()) + } + + fn search_raw( + &self, + project_id: &str, + vector: &[f32], + limit: usize, + exclude: Option, + ) -> EngineResult, f32)>> { + self.ensure_index(project_id)?; + let path = self.index_path(project_id); + let registry = registry() + .lock() + .map_err(|_| EngineError::Validation("embedding index lock poisoned".into()))?; + let cached = registry.get(&path).expect("index ensured"); + if cached.index.size() == 0 { + return Ok(Vec::new()); + } + let count = (limit + usize::from(exclude.is_some())).min(cached.index.size()); + let matches = cached.index.search(vector, count).map_err(index_error)?; + Ok(matches + .keys + .into_iter() + .zip(matches.distances) + .filter(|(label, _)| Some(*label) != exclude) + .take(limit) + .map(|(label, distance)| { + ( + cached.labels.get(&label).cloned(), + (1.0 - distance).max(0.0), + ) + }) + .collect()) + } + + fn ensure_index(&self, project_id: &str) -> EngineResult<()> { + let path = self.index_path(project_id); + if registry() + .lock() + .map_err(|_| EngineError::Validation("embedding index lock poisoned".into()))? + .contains_key(&path) + { + return Ok(()); + } + let keys = qe::list_keys(self.conn, project_id)?; + let expected = keys + .iter() + .map(|key| (key.label as u64, key.post_id.clone())) + .collect::>(); + if let Ok(cached) = load_cached(&path, &expected) { + registry() + .lock() + .map_err(|_| EngineError::Validation("embedding index lock poisoned".into()))? + .insert(path, cached); + return Ok(()); + } + self.rebuild_cached_index(project_id, true) + } + + fn rebuild_cached_index(&self, project_id: &str, persist_now: bool) -> EngineResult<()> { + let keys = qe::list_keys(self.conn, project_id)?; + let index = new_index(keys.len())?; + let mut labels = HashMap::new(); + for key in keys { + let vector = decode_vector(&key.vector)?; + index.add(key.label as u64, &vector).map_err(index_error)?; + labels.insert(key.label as u64, key.post_id); + } + let path = self.index_path(project_id); + let mut cached = CachedIndex { + index, + labels, + dirty_since: Some(Instant::now()), + index_path: path.clone(), + }; + if persist_now { + persist_cached(&mut cached)?; + } + registry() + .lock() + .map_err(|_| EngineError::Validation("embedding index lock poisoned".into()))? + .insert(path, cached); + Ok(()) + } + + fn index_path(&self, project_id: &str) -> PathBuf { + self.cache_root + .join("projects") + .join(project_id) + .join("embeddings.usearch") + } +} + +pub fn sync_post_best_effort(conn: &Connection, data_dir: &Path, post: &Post) { + if let Err(error) = EmbeddingService::production(conn, data_dir).sync_post(post) { + eprintln!("embedding unavailable for post {}: {error}", post.id); + } +} + +pub fn remove_post_best_effort( + conn: &Connection, + data_dir: &Path, + project_id: &str, + post_id: &str, +) { + if let Err(error) = + EmbeddingService::production(conn, data_dir).remove_post(project_id, post_id) + { + eprintln!("could not remove embedding for post {post_id}: {error}"); + } +} + +fn new_index(capacity: usize) -> EngineResult { + let index = Index::new(&IndexOptions { + dimensions: DIMENSIONS, + metric: MetricKind::Cos, + quantization: ScalarKind::F32, + connectivity: 16, + expansion_add: 128, + expansion_search: 64, + ..IndexOptions::default() + }) + .map_err(index_error)?; + index.reserve(capacity.max(1)).map_err(index_error)?; + Ok(index) +} + +fn load_cached(path: &Path, expected: &HashMap) -> EngineResult { + let metadata: IndexMetadata = serde_json::from_slice(&fs::read(meta_path(path))?)?; + let labels = metadata.labels.into_iter().collect::>(); + if metadata.dimensions != DIMENSIONS || &labels != expected || !path.exists() { + return Err(EngineError::Validation( + "embedding index metadata is stale".into(), + )); + } + let index = new_index(expected.len())?; + index + .load(path.to_string_lossy().as_ref()) + .map_err(index_error)?; + if index.size() != expected.len() { + return Err(EngineError::Validation( + "embedding index size is stale".into(), + )); + } + Ok(CachedIndex { + index, + labels, + dirty_since: None, + index_path: path.to_path_buf(), + }) +} + +fn persist_cached(cached: &mut CachedIndex) -> EngineResult<()> { + if cached.dirty_since.is_none() { + return Ok(()); + } + if let Some(parent) = cached.index_path.parent() { + fs::create_dir_all(parent)?; + } + cached + .index + .save(cached.index_path.to_string_lossy().as_ref()) + .map_err(index_error)?; + let mut labels = cached + .labels + .iter() + .map(|(label, post_id)| (*label, post_id.clone())) + .collect::>(); + labels.sort_by_key(|(label, _)| *label); + let metadata = serde_json::to_vec(&IndexMetadata { + dimensions: DIMENSIONS, + labels, + })?; + fs::write(meta_path(&cached.index_path), metadata)?; + cached.dirty_since = None; + Ok(()) +} + +fn meta_path(path: &Path) -> PathBuf { + PathBuf::from(format!("{}.meta.json", path.display())) +} + +fn index_error(error: impl std::fmt::Display) -> EngineError { + EngineError::Validation(format!("embedding index error: {error}")) +} + +pub fn hash_text(text: &str) -> String { + format!("{:x}", Sha256::digest(text.as_bytes())) +} + +pub fn encode_vector(vector: &[f32]) -> Vec { + vector + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect() +} + +pub fn decode_vector(bytes: &[u8]) -> EngineResult> { + if bytes.len() != DIMENSIONS * size_of::() { + return Err(EngineError::Validation(format!( + "invalid embedding vector length: {}", + bytes.len() + ))); + } + Ok(bytes + .chunks_exact(4) + .map(|chunk| f32::from_le_bytes(chunk.try_into().expect("four-byte chunk"))) + .collect()) +} + +fn normalize(mut vector: Vec) -> Vec { + let norm = vector.iter().map(|value| value * value).sum::().sqrt(); + if norm > 0.0 { + for value in &mut vector { + *value /= norm; + } + } + vector +} + +fn cosine(a: &[f32], b: &[f32]) -> f32 { + a.iter() + .zip(b) + .map(|(left, right)| left * right) + .sum::() + .clamp(0.0, 1.0) +} + +fn canonical_pair(a: &str, b: &str) -> (String, String) { + if a <= b { + (a.to_string(), b.to_string()) + } else { + (b.to_string(), a.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vector_blob_is_exactly_1536_bytes_and_round_trips() { + let vector = (0..DIMENSIONS) + .map(|index| index as f32 / 10.0) + .collect::>(); + let encoded = encode_vector(&vector); + assert_eq!(encoded.len(), 1536); + assert_eq!(decode_vector(&encoded).unwrap(), vector); + assert!(decode_vector(&encoded[..100]).is_err()); + } + + #[test] + fn content_hash_uses_normative_title_blank_line_body_source() { + assert_eq!( + hash_text("Title\n\nBody"), + "45777c14d90fa79dc6ce71ceb6f81cced62c929c472199547cfd939a54e954c6" + ); + } + + #[test] + fn model_selection_is_real_multilingual_e5_small() { + let info = TextEmbedding::get_model_info(&EmbeddingModel::MultilingualE5Small).unwrap(); + assert_eq!(MODEL_ID, "Xenova/multilingual-e5-small"); + assert_eq!(info.model_code, MODEL_REPOSITORY); + assert_eq!(info.dim, DIMENSIONS); + #[cfg(target_os = "macos")] + { + use ort::ep::ExecutionProvider; + assert!(ort::ep::CoreML::default().is_available().unwrap()); + } + } + + #[test] + fn usearch_index_uses_the_normative_hnsw_configuration() { + let index = new_index(1).unwrap(); + assert_eq!(index.dimensions(), DIMENSIONS); + assert_eq!(index.metric_kind(), MetricKind::Cos); + assert_eq!(index.connectivity(), 16); + assert_eq!(index.expansion_add(), 128); + assert_eq!(index.expansion_search(), 64); + } + + #[test] + #[ignore = "downloads the real multilingual model; run for release verification"] + fn real_model_is_multilingual_and_reloads_from_its_local_cache() { + let cache = tempfile::tempdir().unwrap(); + let backend = NeuralBackend { + model: Mutex::new(None), + cache_dir: cache.path().into(), + }; + let texts = vec![ + "query: A rocket launches into space".to_string(), + "query: Eine Rakete startet in den Weltraum".to_string(), + "query: Baking bread in the kitchen".to_string(), + ]; + let first = backend.embed(&texts).unwrap(); + assert_eq!( + first.iter().map(Vec::len).collect::>(), + vec![DIMENSIONS; 3] + ); + assert!(cosine(&first[0], &first[1]) > cosine(&first[0], &first[2])); + + let cached_backend = NeuralBackend { + model: Mutex::new(None), + cache_dir: cache.path().into(), + }; + let cached = cached_backend.embed(&texts[..1]).unwrap(); + assert_eq!(cached[0].len(), DIMENSIONS); + } + + use crate::db::Database; + use crate::model::{PostStatus, Project}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct FixtureBackend(AtomicUsize); + + impl FixtureBackend { + fn new() -> Self { + Self(AtomicUsize::new(0)) + } + fn embedded(&self) -> usize { + self.0.load(Ordering::SeqCst) + } + } + + impl EmbeddingBackend for FixtureBackend { + fn embed(&self, texts: &[String]) -> Result>, String> { + self.0.fetch_add(texts.len(), Ordering::SeqCst); + Ok(texts + .iter() + .map(|text| { + let text = text.to_lowercase(); + let mut vector = vec![0.0; DIMENSIONS]; + if text.contains("rocket") || text.contains("rakete") || text.contains("space") + { + vector[0] = 1.0; + vector[1] = 0.02; + } else if text.contains("bread") || text.contains("brot") { + vector[1] = 1.0; + } else { + vector[2] = 1.0; + } + vector + }) + .collect()) + } + } + + fn setup_service( + enabled: bool, + ) -> ( + Database, + tempfile::TempDir, + tempfile::TempDir, + String, + Arc, + ) { + let db = Database::open_in_memory().unwrap(); + db.migrate().unwrap(); + let data = tempfile::tempdir().unwrap(); + let cache = tempfile::tempdir().unwrap(); + crate::engine::meta::startup_sync(data.path()).unwrap(); + let mut metadata = crate::engine::meta::read_project_json(data.path()).unwrap(); + metadata.semantic_similarity_enabled = enabled; + crate::engine::meta::write_project_json(data.path(), &metadata).unwrap(); + let project_id = uuid::Uuid::new_v4().to_string(); + crate::db::queries::project::insert_project( + db.conn(), + &Project { + id: project_id.clone(), + name: "Semantic".into(), + slug: format!("semantic-{project_id}"), + description: None, + data_path: Some(data.path().to_string_lossy().into()), + is_active: true, + created_at: 1, + updated_at: 1, + }, + ) + .unwrap(); + (db, data, cache, project_id, Arc::new(FixtureBackend::new())) + } + + fn insert_post( + db: &Database, + project_id: &str, + id: &str, + title: &str, + body: &str, + tags: &[&str], + ) -> Post { + let post = Post { + id: id.into(), + project_id: project_id.into(), + title: title.into(), + slug: id.into(), + excerpt: None, + content: Some(body.into()), + status: PostStatus::Draft, + author: None, + language: Some("en".into()), + do_not_translate: false, + template_slug: None, + file_path: String::new(), + checksum: None, + tags: tags.iter().map(|tag| (*tag).into()).collect(), + categories: vec![], + published_title: None, + published_content: None, + published_tags: None, + published_categories: None, + published_excerpt: None, + created_at: 1, + updated_at: 1, + published_at: None, + }; + qp::insert_post(db.conn(), &post).unwrap(); + post + } + + #[test] + fn lifecycle_is_gated_hash_cached_and_recovers_disposable_index_from_db() { + let (db, data, cache, project_id, backend) = setup_service(false); + let post = insert_post( + &db, + &project_id, + "space", + "Space", + "rocket launch", + &["space"], + ); + let service = EmbeddingService::with_backend( + db.conn(), + data.path(), + cache.path().into(), + backend.clone(), + ); + assert!(!service.sync_post(&post).unwrap()); + assert_eq!(backend.embedded(), 0); + + let mut metadata = crate::engine::meta::read_project_json(data.path()).unwrap(); + metadata.semantic_similarity_enabled = true; + crate::engine::meta::write_project_json(data.path(), &metadata).unwrap(); + assert_eq!(service.index_unindexed(&project_id).unwrap(), vec!["space"]); + assert_eq!(backend.embedded(), 1); + assert_eq!( + qe::list_keys(db.conn(), &project_id).unwrap()[0] + .vector + .len(), + 1536 + ); + service.index_unindexed(&project_id).unwrap(); + assert_eq!(backend.embedded(), 1, "unchanged hash must skip inference"); + + let path = service.index_path(&project_id); + assert!(!path.exists(), "index save must be debounced"); + service.flush_project(&project_id).unwrap(); + assert!(path.exists()); + assert!(meta_path(&path).exists()); + + registry().lock().unwrap().remove(&path); + fs::write(&path, b"corrupt index").unwrap(); + let results = service.semantic_search(&project_id, "rocket", 5).unwrap(); + assert_eq!(results[0].post_id, "space"); + service.flush_project(&project_id).unwrap(); + assert!(fs::metadata(&path).unwrap().len() > b"corrupt index".len() as u64); + + let mut changed = qp::get_post_by_id(db.conn(), "space").unwrap(); + changed.content = Some("Bread baking".into()); + qp::update_post(db.conn(), &changed).unwrap(); + assert!(service.sync_post(&changed).unwrap()); + assert_eq!(backend.embedded(), 3); // query + changed post + insert_post(&db, &project_id, "other", "Other", "Other body", &[]); + service.dismiss_duplicate_pair("space", "other").unwrap(); + service.remove_post(&project_id, "space").unwrap(); + assert_eq!(qe::list_keys(db.conn(), &project_id).unwrap().len(), 0); + assert!( + qe::list_dismissed_pairs(db.conn(), &project_id) + .unwrap() + .is_empty() + ); + } + + #[test] + fn background_indexing_reports_progress_and_honors_cancellation_before_inference() { + let (db, data, cache, project_id, backend) = setup_service(true); + insert_post(&db, &project_id, "space", "Space", "rocket launch", &[]); + let service = EmbeddingService::with_backend( + db.conn(), + data.path(), + cache.path().into(), + backend.clone(), + ); + + let cancelled = service.index_unindexed_with_progress(&project_id, |_, _| false); + assert!(cancelled.is_err()); + assert_eq!(backend.embedded(), 0); + assert!(qe::list_keys(db.conn(), &project_id).unwrap().is_empty()); + + let mut progress = Vec::new(); + let indexed = service + .index_unindexed_with_progress(&project_id, |current, total| { + progress.push((current, total)); + true + }) + .unwrap(); + assert_eq!(indexed, vec!["space"]); + assert_eq!(progress, vec![(0, 1), (1, 1)]); + } + + #[test] + fn semantic_queries_tags_duplicates_and_dismissals_match_spec() { + let (db, data, cache, project_id, backend) = setup_service(true); + insert_post( + &db, + &project_id, + "alpha", + "Space", + "rocket launch mission", + &["space"], + ); + insert_post( + &db, + &project_id, + "beta", + "Raumfahrt", + "Rakete im All", + &["space", "science"], + ); + insert_post( + &db, + &project_id, + "exact-a", + "Exact", + "rocket duplicate", + &[], + ); + insert_post( + &db, + &project_id, + "exact-b", + "Exact", + "rocket duplicate", + &[], + ); + insert_post( + &db, + &project_id, + "bread", + "Bread", + "bread baking oven", + &["food"], + ); + let service = + EmbeddingService::with_backend(db.conn(), data.path(), cache.path().into(), backend); + service.index_unindexed(&project_id).unwrap(); + + let similar = service.find_similar("alpha", 4).unwrap(); + assert_eq!(similar.last().unwrap().post_id, "bread"); + assert!(similar[0].similarity > similar.last().unwrap().similarity); + let search = service.semantic_search(&project_id, "Rakete", 3).unwrap(); + assert!(search.iter().all(|result| result.post_id != "bread")); + let scores = service + .compute_similarities("alpha", &["beta".into(), "bread".into()]) + .unwrap(); + assert!(scores["beta"] > scores["bread"]); + assert_eq!(service.suggest_tags("alpha").unwrap()[0], "science"); + + let duplicates = service.find_duplicates(&project_id, 0).unwrap(); + let exact = duplicates + .pairs + .iter() + .find(|pair| { + HashSet::from([pair.post_id_a.as_str(), pair.post_id_b.as_str()]) + == HashSet::from(["exact-a", "exact-b"]) + }) + .unwrap(); + assert!(exact.exact_match); + service + .dismiss_duplicate_pair("exact-b", "exact-a") + .unwrap(); + assert_eq!( + qe::list_dismissed_pairs(db.conn(), &project_id).unwrap()[0].post_id_a, + "exact-a" + ); + let filtered = service.find_duplicates(&project_id, 0).unwrap(); + assert!( + !filtered + .pairs + .iter() + .any(|pair| pair.post_id_a == "exact-a" && pair.post_id_b == "exact-b") + ); + } + + #[test] + fn duplicate_search_paginates_and_batch_dismisses_in_chunks() { + let (db, data, cache, project_id, backend) = setup_service(true); + for index in 0..60 { + insert_post( + &db, + &project_id, + &format!("post-{index:02}"), + "Same", + "rocket duplicate", + &[], + ); + } + let service = + EmbeddingService::with_backend(db.conn(), data.path(), cache.path().into(), backend); + service.index_unindexed(&project_id).unwrap(); + let first = service.find_duplicates(&project_id, 0).unwrap(); + assert_eq!(first.pairs.len(), DUPLICATE_PAGE_SIZE); + assert!(first.has_more); + let mut selected = first + .pairs + .iter() + .take(205) + .map(|pair| (pair.post_id_b.clone(), pair.post_id_a.clone())) + .collect::>(); + selected.push((selected[0].1.clone(), selected[0].0.clone())); + service.dismiss_duplicate_pairs(&selected).unwrap(); + assert_eq!( + qe::list_dismissed_pairs(db.conn(), &project_id) + .unwrap() + .len(), + 205 + ); + let remaining = service.find_duplicates(&project_id, 1).unwrap(); + assert!( + remaining + .pairs + .iter() + .all(|pair| !selected.contains(&(pair.post_id_b.clone(), pair.post_id_a.clone()))) + ); + } + + #[test] + fn project_indexes_and_search_results_are_isolated_while_backend_is_shared() { + let (db, data, cache, first_project, backend) = setup_service(true); + let second_project = uuid::Uuid::new_v4().to_string(); + crate::db::queries::project::insert_project( + db.conn(), + &Project { + id: second_project.clone(), + name: "Second".into(), + slug: format!("second-{second_project}"), + description: None, + data_path: Some(data.path().to_string_lossy().into()), + is_active: false, + created_at: 2, + updated_at: 2, + }, + ) + .unwrap(); + insert_post( + &db, + &first_project, + "first-space", + "Space", + "rocket mission", + &[], + ); + insert_post( + &db, + &second_project, + "second-bread", + "Bread", + "bread baking", + &[], + ); + let service = + EmbeddingService::with_backend(db.conn(), data.path(), cache.path().into(), backend); + service.index_unindexed(&first_project).unwrap(); + service.index_unindexed(&second_project).unwrap(); + assert_ne!( + service.index_path(&first_project), + service.index_path(&second_project) + ); + assert_eq!( + service + .semantic_search(&first_project, "rocket", 5) + .unwrap()[0] + .post_id, + "first-space" + ); + assert_eq!( + service + .semantic_search(&second_project, "bread", 5) + .unwrap()[0] + .post_id, + "second-bread" + ); + } +} diff --git a/crates/bds-core/src/engine/metadata_diff.rs b/crates/bds-core/src/engine/metadata_diff.rs index bb32dde..c34c6c7 100644 --- a/crates/bds-core/src/engine/metadata_diff.rs +++ b/crates/bds-core/src/engine/metadata_diff.rs @@ -5,6 +5,7 @@ use std::path::Path; use crate::db::DbConnection as Connection; use walkdir::WalkDir; +use crate::db::queries::embedding as qe; use crate::db::queries::media as qm; use crate::db::queries::media_translation as qmt; use crate::db::queries::post as qp; @@ -159,6 +160,49 @@ pub fn compute_metadata_diff( } // 6. Detect orphans + if crate::engine::meta::read_project_json(data_dir) + .is_ok_and(|metadata| metadata.semantic_similarity_enabled) + { + let service = crate::engine::embedding::EmbeddingService::production(conn, data_dir); + let keys = qe::list_keys(conn, project_id)? + .into_iter() + .map(|key| (key.post_id.clone(), key)) + .collect::>(); + for post in &posts { + let expected = service.content_hash_for_post(post)?; + let key = keys.get(&post.id); + let current_hash = key.map(|key| key.content_hash.as_str()).unwrap_or(""); + let vector_status = key + .filter(|key| crate::engine::embedding::decode_vector(&key.vector).is_ok()) + .map(|_| "ready") + .unwrap_or("missing"); + if current_hash != expected || vector_status != "ready" { + let mut fields = Vec::new(); + if current_hash != expected { + fields.push(DiffField { + field_name: "content_hash".into(), + db_value: current_hash.into(), + file_value: expected, + }); + } + if vector_status != "ready" { + fields.push(DiffField { + field_name: "embedding".into(), + db_value: vector_status.into(), + file_value: "ready".into(), + }); + } + report.diffs.push(EntityDiff { + entity_type: "embedding".into(), + entity_id: post.id.clone(), + file_path: format!("projects/{project_id}/embeddings.usearch"), + fields, + }); + } + } + } + + // 7. Detect orphans let orphans = detect_orphan_files(conn, data_dir, project_id)?; report.orphans = orphans; @@ -204,6 +248,11 @@ pub fn repair_metadata_diff_item( conn, data_dir, project_id, &path, )?; } + "embedding" => { + let post = qp::get_post_by_id(conn, &item.entity_id)?; + crate::engine::embedding::EmbeddingService::production(conn, data_dir) + .sync_post(&post)?; + } other => return unsupported_repair(other), } } @@ -222,6 +271,8 @@ pub fn repair_metadata_diff_item( )?, "script" => rewrite_script_from_database(conn, data_dir, &item.entity_id)?, "template" => rewrite_template_from_database(conn, data_dir, &item.entity_id)?, + "embedding" => crate::engine::embedding::EmbeddingService::production(conn, data_dir) + .flush_project(project_id)?, other => return unsupported_repair(other), }, } diff --git a/crates/bds-core/src/engine/mod.rs b/crates/bds-core/src/engine/mod.rs index 8586369..286d1f4 100644 --- a/crates/bds-core/src/engine/mod.rs +++ b/crates/bds-core/src/engine/mod.rs @@ -8,6 +8,7 @@ mod chat_tools; pub mod cli_launcher; pub mod cli_sync; pub mod domain_events; +pub mod embedding; pub mod error; pub mod gallery_import; pub mod generation; diff --git a/crates/bds-core/src/engine/post.rs b/crates/bds-core/src/engine/post.rs index b2991ef..317ed32 100644 --- a/crates/bds-core/src/engine/post.rs +++ b/crates/bds-core/src/engine/post.rs @@ -36,7 +36,7 @@ pub struct RebuildReport { )] pub fn create_post( conn: &Connection, - _data_dir: &Path, + data_dir: &Path, project_id: &str, title: &str, content: Option<&str>, @@ -91,6 +91,7 @@ pub fn create_post( fts_index_post(conn, &post)?; emit_post(&post, NotificationAction::Created); + crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post); Ok(post) } @@ -102,7 +103,7 @@ pub fn create_post( )] pub fn update_post( conn: &Connection, - _data_dir: &Path, + data_dir: &Path, post_id: &str, title: Option<&str>, slug: Option<&str>, @@ -169,7 +170,7 @@ pub fn update_post( if post.status == PostStatus::Published || post.status == PostStatus::Archived { // Reload content from filesystem if content field is NULL (published state) if post.content.is_none() && !post.file_path.is_empty() { - let abs_path = _data_dir.join(&post.file_path); + let abs_path = data_dir.join(&post.file_path); if abs_path.exists() && let Ok(file_content) = fs::read_to_string(&abs_path) && let Ok((_fm, body)) = read_post_file(&file_content) @@ -187,6 +188,7 @@ pub fn update_post( fts_index_post(conn, &post)?; emit_post(&post, NotificationAction::Updated); + crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post); Ok(post) } @@ -210,6 +212,7 @@ pub fn publish_post(conn: &Connection, data_dir: &Path, post_id: &str) -> Engine Ok(post) => { conn.release_savepoint()?; emit_post(&post, NotificationAction::Updated); + crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post); Ok(post) } Err(error) => { @@ -343,7 +346,9 @@ pub fn archive_post(conn: &Connection, data_dir: &Path, post_id: &str) -> Engine } let now = now_unix_ms(); qp::update_post_status(conn, post_id, &PostStatus::Archived, now)?; + post.status = PostStatus::Archived; emit_post(&post, NotificationAction::Updated); + crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post); Ok(()) } @@ -455,6 +460,7 @@ pub fn discard_post_draft(conn: &Connection, data_dir: &Path, post_id: &str) -> Ok(post) => { conn.release_savepoint()?; emit_post(&post, NotificationAction::Updated); + crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post); Ok(post) } Err(error) => { @@ -503,6 +509,8 @@ pub fn delete_post(conn: &Connection, data_dir: &Path, post_id: &str) -> EngineR // Delete post from DB qp::delete_post(conn, post_id)?; + crate::engine::embedding::remove_post_best_effort(conn, data_dir, &post.project_id, post_id); + emit_post(&post, NotificationAction::Deleted); Ok(()) diff --git a/crates/bds-core/src/engine/project.rs b/crates/bds-core/src/engine/project.rs index ba62723..b18c03c 100644 --- a/crates/bds-core/src/engine/project.rs +++ b/crates/bds-core/src/engine/project.rs @@ -238,6 +238,7 @@ pub fn delete_project( let is_custom_path = project.data_path.is_some(); q::delete_project(conn, project_id)?; + crate::engine::embedding::EmbeddingService::forget_project(project_id); // Clean up internal filesystem only (not custom external paths per spec) if !is_custom_path diff --git a/crates/bds-core/src/engine/rebuild.rs b/crates/bds-core/src/engine/rebuild.rs index acdbf5c..b62bb97 100644 --- a/crates/bds-core/src/engine/rebuild.rs +++ b/crates/bds-core/src/engine/rebuild.rs @@ -170,15 +170,18 @@ fn rebuild_from_filesystem_inner( ))); } + progress(0.98, "Refreshing semantic index..."); + crate::engine::embedding::EmbeddingService::production(conn, data_dir) + .index_unindexed(project_id)?; + progress(1.0, "Rebuild complete"); Ok(report) } fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> { use crate::db::schema::{ - dismissed_duplicate_pairs, embedding_keys, generated_file_hashes, import_definitions, - media, media_translations, post_links, post_media, post_translations, posts, scripts, tags, - templates, + generated_file_hashes, import_definitions, media, media_translations, post_links, + post_media, post_translations, posts, scripts, tags, templates, }; let post_ids = crate::db::queries::post::list_posts_by_project(conn, project_id)? @@ -219,13 +222,6 @@ fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> { generated_file_hashes::table.filter(generated_file_hashes::project_id.eq(project_id)), ) .execute(connection)?; - diesel::delete(embedding_keys::table.filter(embedding_keys::project_id.eq(project_id))) - .execute(connection)?; - diesel::delete( - dismissed_duplicate_pairs::table - .filter(dismissed_duplicate_pairs::project_id.eq(project_id)), - ) - .execute(connection)?; diesel::delete( import_definitions::table.filter(import_definitions::project_id.eq(project_id)), ) diff --git a/crates/bds-core/src/model/embedding.rs b/crates/bds-core/src/model/embedding.rs new file mode 100644 index 0000000..7ff60db --- /dev/null +++ b/crates/bds-core/src/model/embedding.rs @@ -0,0 +1,50 @@ +use serde::{Deserialize, Serialize}; + +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + diesel::Queryable, + diesel::Selectable, + diesel::Insertable, + diesel::AsChangeset, +)] +#[diesel( + table_name = crate::db::schema::embedding_keys, + check_for_backend(diesel::sqlite::Sqlite), + treat_none_as_default_value = false +)] +pub struct EmbeddingKey { + pub label: i64, + pub post_id: String, + pub project_id: String, + pub content_hash: String, + pub vector: Vec, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + diesel::Queryable, + diesel::Selectable, + diesel::Insertable, + diesel::AsChangeset, +)] +#[diesel( + table_name = crate::db::schema::dismissed_duplicate_pairs, + check_for_backend(diesel::sqlite::Sqlite), + treat_none_as_default_value = false +)] +pub struct DismissedDuplicatePair { + pub id: String, + pub project_id: String, + pub post_id_a: String, + pub post_id_b: String, + pub dismissed_at: i64, +} diff --git a/crates/bds-core/src/model/mod.rs b/crates/bds-core/src/model/mod.rs index d8bb512..9727bb1 100644 --- a/crates/bds-core/src/model/mod.rs +++ b/crates/bds-core/src/model/mod.rs @@ -1,4 +1,5 @@ mod chat; +mod embedding; mod event; mod generation; mod import; @@ -12,6 +13,7 @@ mod tag; mod template; pub use chat::{ChatConversation, ChatMessage, ChatRole, NewChatConversation, NewChatMessage}; +pub use embedding::{DismissedDuplicatePair, EmbeddingKey}; pub use event::DomainEvent; pub use generation::{ DbNotification, DomainEntity, GeneratedFileHash, NotificationAction, NotificationEntity, diff --git a/crates/bds-core/src/scripting/core_host.rs b/crates/bds-core/src/scripting/core_host.rs index f2ebcf7..69fbd39 100644 --- a/crates/bds-core/src/scripting/core_host.rs +++ b/crates/bds-core/src/scripting/core_host.rs @@ -1382,6 +1382,95 @@ impl CoreHost { (_, response) => one_shot_json(response), } } + + fn embeddings(&self, method: &str, args: &[Value]) -> HostResult { + let db = self.database()?; + let service = engine::embedding::EmbeddingService::production(db.conn(), &self.data_dir); + match method { + "get_progress" => { + let (indexed, total) = service.indexing_progress(&self.project_id)?; + Ok(json!({"indexed": indexed, "total": total})) + } + "find_similar" => { + let post_id = string_arg(args, 0)?; + self.scoped( + |conn| crate::db::queries::post::get_post_by_id(conn, post_id), + |post| post.project_id.as_str(), + )?; + let limit = args.get(1).and_then(Value::as_u64).unwrap_or(5) as usize; + Ok(Value::Array( + service + .find_similar(post_id, limit)? + .into_iter() + .map(|post| { + json!({ + "post_id": post.post_id, + "title": post.title, + "score": post.similarity, + }) + }) + .collect(), + )) + } + "compute_similarities" => { + let post_id = string_arg(args, 0)?; + self.scoped( + |conn| crate::db::queries::post::get_post_by_id(conn, post_id), + |post| post.project_id.as_str(), + )?; + let target_ids = string_array_arg(args, 1)?; + json_value(service.compute_similarities(post_id, &target_ids)) + } + "suggest_tags" => { + let post_id = string_arg(args, 0)?; + self.scoped( + |conn| crate::db::queries::post::get_post_by_id(conn, post_id), + |post| post.project_id.as_str(), + )?; + json_value(service.suggest_tags(post_id)) + } + "find_duplicates" => { + let mut page = 0; + let pairs = loop { + let result = service.find_duplicates(&self.project_id, page)?; + if !result.has_more { + break result.pairs; + } + page += 1; + }; + Ok(Value::Array( + pairs + .into_iter() + .map(|pair| { + json!({ + "post_id_a": pair.post_id_a, + "title_a": pair.title_a, + "post_id_b": pair.post_id_b, + "title_b": pair.title_b, + "score": pair.similarity, + "similarity": pair.similarity, + "exact_match": pair.exact_match, + }) + }) + .collect(), + )) + } + "dismiss_pair" => { + let post_id_a = string_arg(args, 0)?; + let post_id_b = string_arg(args, 1)?; + for post_id in [post_id_a, post_id_b] { + self.scoped( + |conn| crate::db::queries::post::get_post_by_id(conn, post_id), + |post| post.project_id.as_str(), + )?; + } + service.dismiss_duplicate_pair(post_id_a, post_id_b)?; + Ok(Value::Bool(true)) + } + "index_unindexed_posts" => json_value(service.index_unindexed(&self.project_id)), + _ => Err(format!("unknown embeddings capability: {method}").into()), + } + } } impl HostApi for CoreHost { @@ -1398,6 +1487,7 @@ impl HostApi for CoreHost { "tasks" => self.tasks(method, &arguments), "publish" => self.publish(method, &arguments), "chat" => self.chat(method, &arguments), + "embeddings" => self.embeddings(method, &arguments), "bds" if method == "report_progress" => self.report_progress(&arguments), _ => Err(format!("unknown host capability: {namespace}.{method}").into()), }; @@ -1537,6 +1627,19 @@ fn string_arg(args: &[Value], index: usize) -> HostResult<&str> { .ok_or_else(|| format!("argument {} must be a string", index + 1).into()) } +fn string_array_arg(args: &[Value], index: usize) -> HostResult> { + args.get(index) + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .ok_or_else(|| format!("argument {} must be a table", index + 1).into()) +} + fn string_field<'a>(value: &'a Map, field: &str) -> HostResult<&'a str> { value .get(field) @@ -1821,6 +1924,9 @@ mod tests { upload = upload, timestamp = post.created_at, project_path = bds.app.get_default_project_path(), + embedding_progress = bds.embeddings.get_progress(), + embedding_backfill = bds.embeddings.index_unindexed_posts(), + foreign_embedding = bds.embeddings.find_similar(input.foreign_post, 5), } end "#, @@ -1842,5 +1948,11 @@ mod tests { assert!(result.value["upload"].is_null()); assert!(result.value["timestamp"].as_str().unwrap().contains('T')); assert_eq!(manager.progress(task_id), Some(0.5)); + assert_eq!( + result.value["embedding_progress"], + json!({"indexed": 0, "total": 1}) + ); + assert_eq!(result.value["embedding_backfill"], json!([])); + assert!(result.value["foreign_embedding"].is_null()); } } diff --git a/crates/bds-core/src/scripting/mod.rs b/crates/bds-core/src/scripting/mod.rs index 7329166..7fffbac 100644 --- a/crates/bds-core/src/scripting/mod.rs +++ b/crates/bds-core/src/scripting/mod.rs @@ -456,6 +456,7 @@ mod tests { [ "app", "chat", + "embeddings", "media", "meta", "posts", @@ -473,7 +474,7 @@ mod tests { function main() return { sync = bds.sync, - embeddings = bds.embeddings, + embeddings = type(bds.embeddings), report_progress = type(bds.report_progress), post_search = type(bds.posts.search), app_toast = type(bds.app.toast), @@ -492,6 +493,7 @@ mod tests { "report_progress": "function", "post_search": "function", "app_toast": "function", + "embeddings": "table", }) ); } @@ -514,6 +516,7 @@ mod tests { bds.tasks.status_snapshot(), bds.publish.upload_site({}), bds.chat.detect_post_language("title", "body"), + bds.embeddings.get_progress(), } end "#, @@ -525,8 +528,8 @@ mod tests { ) .unwrap(); - assert_eq!(execution.value.as_array().unwrap().len(), 11); - assert_eq!(host.0.lock().unwrap().len(), 11); + assert_eq!(execution.value.as_array().unwrap().len(), 12); + assert_eq!(host.0.lock().unwrap().len(), 12); } #[test] diff --git a/crates/bds-ui/src/app.rs b/crates/bds-ui/src/app.rs index 2b138b3..04ededc 100644 --- a/crates/bds-ui/src/app.rs +++ b/crates/bds-ui/src/app.rs @@ -33,6 +33,7 @@ use crate::views::{ DashboardCategory, DashboardRecentPost, DashboardState, DashboardStats, DashboardTag, DashboardTimelineMonth, }, + duplicates::DuplicatesState, git::{GitDiffLoad, GitDiffState, GitNetworkCompletion, GitSnapshot, GitUiState}, import_editor::{ ImportAnalysisEvent, ImportEditorMsg, ImportEditorState, ImportExecutionEvent, @@ -241,6 +242,23 @@ pub enum Message { result: Result, }, SiteValidationLoaded(Result), + DuplicatesRefresh, + DuplicatesLoaded(Result), + DuplicatesToggle(String, String), + DuplicatesCheckAll, + DuplicatesUncheckAll, + DuplicatesDismiss(String, String), + DuplicatesDismissSelected, + DuplicatesDismissed(Result<(), String>), + DuplicatesShowMore, + DuplicatesOpenPost(String), + EmbeddingReindex, + EmbeddingBackfill, + LoadSemanticTagSuggestions(String), + SemanticTagSuggestionsLoaded { + post_id: String, + result: Result, String>, + }, // Git GitRefresh, @@ -927,6 +945,7 @@ pub struct BdsApp { settings_state: Option, dashboard_state: Option, site_validation_state: SiteValidationState, + duplicates_state: DuplicatesState, metadata_diff_state: MetadataDiffState, translation_validation_state: crate::views::translation_validation::TranslationValidationState, git_state: GitUiState, @@ -1091,6 +1110,7 @@ impl BdsApp { settings_state: None, dashboard_state: None, site_validation_state: SiteValidationState::default(), + duplicates_state: DuplicatesState::default(), metadata_diff_state: MetadataDiffState::default(), translation_validation_state: Default::default(), git_state: GitUiState::default(), @@ -1169,6 +1189,7 @@ impl BdsApp { settings_state: None, dashboard_state: None, site_validation_state: SiteValidationState::default(), + duplicates_state: DuplicatesState::default(), metadata_diff_state: MetadataDiffState::default(), translation_validation_state: Default::default(), git_state: GitUiState::default(), @@ -1495,14 +1516,22 @@ impl BdsApp { Message::OpenTab(tab) => { self.flush_active_post_editor(); let idx = tabs::open_tab(&mut self.tabs, tab); + let mut semantic_post_id = None; if let Some(t) = self.tabs.get(idx) { self.active_tab = Some(t.id.clone()); let tab_clone = t.clone(); + if tab_clone.tab_type == TabType::Post { + semantic_post_id = Some(tab_clone.id.clone()); + } self.load_editor_for_tab(&tab_clone); } self.enforce_panel_tab_fallback(); self.sync_menu_state(); - self.sync_embedded_preview_for_active_post() + let mut tasks = vec![self.sync_embedded_preview_for_active_post()]; + if let Some(post_id) = semantic_post_id { + tasks.push(Task::done(Message::LoadSemanticTagSuggestions(post_id))); + } + Task::batch(tasks) } Message::CloseTab(id) => { if self.active_tab.as_deref() == Some(id.as_str()) { @@ -1573,17 +1602,24 @@ impl BdsApp { } let sidebar_task = self.refresh_counts(); self.sync_menu_state(); - sidebar_task + Task::batch([sidebar_task, Task::done(Message::EmbeddingBackfill)]) } Message::SwitchProject(project_id) => { self.project_dropdown_open = false; if let Some(ref db) = self.db { + if let (Some(outgoing), Some(data_dir)) = (&self.active_project, &self.data_dir) + { + let _ = + engine::embedding::EmbeddingService::production(db.conn(), data_dir) + .flush_project(&outgoing.id); + } match engine::project::set_active_project(db.conn(), &project_id) { Ok(()) => { self.reset_git_for_project_change(); self.active_project = self.projects.iter().find(|p| p.id == project_id).cloned(); self.preview_session = None; + self.duplicates_state = DuplicatesState::default(); self.hide_embedded_preview(); self.data_dir = self .active_project @@ -1626,7 +1662,11 @@ impl BdsApp { } } self.sync_menu_state(); - Task::batch([self.refresh_counts(), self.refresh_git_if_visible()]) + Task::batch([ + self.refresh_counts(), + self.refresh_git_if_visible(), + Task::done(Message::EmbeddingBackfill), + ]) } Message::ProjectSwitched(result) => { match result { @@ -2119,6 +2159,118 @@ impl BdsApp { } Task::none() } + Message::DuplicatesRefresh => self.start_duplicate_search(0), + Message::DuplicatesShowMore => { + let next = self.duplicates_state.page.saturating_add(1); + self.start_duplicate_search(next) + } + Message::DuplicatesLoaded(result) => { + self.duplicates_state.is_loading = false; + self.duplicates_state.has_run = true; + match result { + Ok(result) => { + self.duplicates_state.result = result; + self.duplicates_state.error = None; + self.duplicates_state.selected.retain(|pair| { + self.duplicates_state.result.pairs.iter().any(|candidate| { + candidate.post_id_a == pair.0 && candidate.post_id_b == pair.1 + }) + }); + } + Err(error) => self.duplicates_state.error = Some(error), + } + Task::none() + } + Message::DuplicatesToggle(a, b) => { + if !self + .duplicates_state + .selected + .remove(&(a.clone(), b.clone())) + { + self.duplicates_state.selected.insert((a, b)); + } + Task::none() + } + Message::DuplicatesCheckAll => { + self.duplicates_state.selected = self + .duplicates_state + .result + .pairs + .iter() + .map(|pair| (pair.post_id_a.clone(), pair.post_id_b.clone())) + .collect(); + Task::none() + } + Message::DuplicatesUncheckAll => { + self.duplicates_state.selected.clear(); + Task::none() + } + Message::DuplicatesDismiss(a, b) => self.dismiss_duplicate_pairs(vec![(a, b)]), + Message::DuplicatesDismissSelected => self + .dismiss_duplicate_pairs(self.duplicates_state.selected.iter().cloned().collect()), + Message::DuplicatesDismissed(result) => { + match result { + Ok(()) => { + self.duplicates_state.selected.clear(); + self.notify( + ToastLevel::Success, + &t(self.ui_locale, "duplicates.dismissed"), + ); + return self.start_duplicate_search(self.duplicates_state.page); + } + Err(error) => self.notify(ToastLevel::Error, &error), + } + Task::none() + } + Message::DuplicatesOpenPost(post_id) => { + let Some(db) = &self.db else { + return Task::none(); + }; + let Ok(post) = bds_core::db::queries::post::get_post_by_id(db.conn(), &post_id) + else { + return Task::none(); + }; + Task::done(Message::OpenTab(Tab { + id: post.id, + tab_type: TabType::Post, + title: post.title, + is_transient: false, + is_dirty: false, + })) + } + Message::EmbeddingReindex => self.start_embedding_reindex(), + Message::EmbeddingBackfill => self.start_embedding_backfill(), + Message::LoadSemanticTagSuggestions(post_id) => { + let Some(data_dir) = self.data_dir.clone() else { + return Task::none(); + }; + let db_path = self.db_path.clone(); + let returned_post_id = post_id.clone(); + Task::perform( + async move { + tokio::task::spawn_blocking(move || { + let db = Database::open(&db_path).map_err(|error| error.to_string())?; + engine::embedding::EmbeddingService::production(db.conn(), &data_dir) + .suggest_tags(&post_id) + .map_err(|error| error.to_string()) + }) + .await + .unwrap_or_else(|error| Err(format!("task panicked: {error}"))) + }, + move |result| Message::SemanticTagSuggestionsLoaded { + post_id: returned_post_id.clone(), + result, + }, + ) + } + Message::SemanticTagSuggestionsLoaded { post_id, result } => { + if let (Some(state), Ok(suggestions)) = + (self.post_editors.get_mut(&post_id), result) + { + state.semantic_tag_suggestions = suggestions; + } + Task::none() + } message @ (Message::MainWindowLoaded(_) | Message::EmbeddedPreviewReady(_)) => { self.handle_preview_message(message) } @@ -2129,6 +2281,7 @@ impl BdsApp { self.refresh_task_snapshots(); self.process_chat_events(); self.persist_due_chat_surface_state(); + let _ = engine::embedding::EmbeddingService::flush_due(); if !self.search_index_rebuild_running { self.auto_save_due_post_editors(); } @@ -2646,6 +2799,151 @@ impl BdsApp { } } + fn start_duplicate_search(&mut self, page: usize) -> Task { + let (Some(project), Some(data_dir)) = (&self.active_project, &self.data_dir) else { + return Task::none(); + }; + self.duplicates_state.enabled = engine::meta::read_project_json(data_dir) + .is_ok_and(|metadata| metadata.semantic_similarity_enabled); + self.duplicates_state.page = page; + self.duplicates_state.error = None; + if !self.duplicates_state.enabled { + self.duplicates_state.is_loading = false; + self.duplicates_state.has_run = false; + self.duplicates_state.result = Default::default(); + return Task::none(); + } + self.duplicates_state.is_loading = true; + let db_path = self.db_path.clone(); + let data_dir = data_dir.clone(); + let project_id = project.id.clone(); + Task::perform( + async move { + tokio::task::spawn_blocking(move || { + let db = Database::open(&db_path).map_err(|error| error.to_string())?; + engine::embedding::EmbeddingService::production(db.conn(), &data_dir) + .find_duplicates(&project_id, page) + .map_err(|error| error.to_string()) + }) + .await + .unwrap_or_else(|error| Err(format!("task panicked: {error}"))) + }, + Message::DuplicatesLoaded, + ) + } + + fn dismiss_duplicate_pairs(&mut self, pairs: Vec<(String, String)>) -> Task { + if pairs.is_empty() { + return Task::none(); + } + let (Some(_), Some(data_dir)) = (&self.db, &self.data_dir) else { + return Task::none(); + }; + let db_path = self.db_path.clone(); + let data_dir = data_dir.clone(); + Task::perform( + async move { + tokio::task::spawn_blocking(move || { + let db = Database::open(&db_path).map_err(|error| error.to_string())?; + engine::embedding::EmbeddingService::production(db.conn(), &data_dir) + .dismiss_duplicate_pairs(&pairs) + .map_err(|error| error.to_string()) + }) + .await + .unwrap_or_else(|error| Err(format!("task panicked: {error}"))) + }, + Message::DuplicatesDismissed, + ) + } + + fn start_embedding_reindex(&mut self) -> Task { + let Some(data_dir) = &self.data_dir else { + return Task::none(); + }; + if !engine::meta::read_project_json(data_dir) + .is_ok_and(|metadata| metadata.semantic_similarity_enabled) + { + self.notify( + ToastLevel::Warning, + &t(self.ui_locale, "duplicates.disabled"), + ); + return Task::none(); + } + let locale = self.ui_locale; + self.spawn_engine_task( + "menu.item.rebuildEmbeddingIndex", + move |db_path, project_id, data_dir, tm, tid| { + let db = Database::open(&db_path).map_err(|error| error.to_string())?; + let service = engine::embedding::EmbeddingService::production(db.conn(), &data_dir); + let indexed = service + .reindex_all_with_progress(&project_id, |current, total| { + tm.report_progress( + tid, + Some(current as f32 / total.max(1) as f32), + Some(tw( + locale, + "embeddings.indexingProgress", + &[ + ("current", ¤t.to_string()), + ("total", &total.to_string()), + ], + )), + ); + !tm.is_cancelled(tid) + }) + .map_err(|error| error.to_string())?; + service + .flush_project(&project_id) + .map_err(|error| error.to_string())?; + Ok(tw( + locale, + "embeddings.reindexed", + &[("count", &indexed.len().to_string())], + )) + }, + ) + } + + fn start_embedding_backfill(&mut self) -> Task { + let Some(data_dir) = &self.data_dir else { + return Task::none(); + }; + if !engine::meta::read_project_json(data_dir) + .is_ok_and(|metadata| metadata.semantic_similarity_enabled) + { + return Task::none(); + } + let locale = self.ui_locale; + self.spawn_engine_task( + "embeddings.indexing", + move |db_path, project_id, data_dir, tm, tid| { + let db = Database::open(&db_path).map_err(|error| error.to_string())?; + let indexed = engine::embedding::EmbeddingService::production(db.conn(), &data_dir) + .index_unindexed_with_progress(&project_id, |current, total| { + tm.report_progress( + tid, + Some(current as f32 / total.max(1) as f32), + Some(tw( + locale, + "embeddings.indexingProgress", + &[ + ("current", ¤t.to_string()), + ("total", &total.to_string()), + ], + )), + ); + !tm.is_cancelled(tid) + }) + .map_err(|error| error.to_string())?; + Ok(tw( + locale, + "embeddings.indexed", + &[("count", &indexed.len().to_string())], + )) + }, + ) + } + pub fn view(&self) -> Element<'_, Message> { let active_name = self.active_project.as_ref().map(|p| p.name.as_str()); let active_post_filter = match self.sidebar_view { @@ -2706,6 +3004,7 @@ impl BdsApp { self.settings_state.as_ref(), self.dashboard_state.as_ref(), &self.site_validation_state, + &self.duplicates_state, &self.metadata_diff_state, &self.translation_validation_state, &self.git_state, @@ -3397,6 +3696,11 @@ impl BdsApp { } MenuAction::RebuildDatabase => Task::done(Message::RebuildDatabase), MenuAction::ReindexText => Task::done(Message::ReindexText), + MenuAction::RebuildEmbeddingIndex => Task::done(Message::EmbeddingReindex), + MenuAction::FindDuplicates => { + self.open_singleton_tab(TabType::FindDuplicates, "tabBar.findDuplicates"); + Task::done(Message::DuplicatesRefresh) + } MenuAction::MetadataDiff => Task::done(Message::RunMetadataDiff), MenuAction::RegenerateCalendar => Task::done(Message::RegenerateCalendar), MenuAction::ValidateTranslations => Task::done(Message::ValidateTranslations), @@ -4777,7 +5081,7 @@ impl BdsApp { ..Default::default() }; - let ids = bds_core::db::fts::search_posts_filtered( + let mut ids = bds_core::db::fts::search_posts_filtered( db.conn(), query, &self.content_language, @@ -4786,6 +5090,19 @@ impl BdsApp { .map(|results| results.post_ids) .unwrap_or_default(); + if let Some(data_dir) = &self.data_dir + && let Ok(scores) = engine::embedding::EmbeddingService::production(db.conn(), data_dir) + .compute_similarities(current_post_id, &ids) + { + ids.sort_by(|a, b| { + scores + .get(b) + .copied() + .unwrap_or_default() + .total_cmp(&scores.get(a).copied().unwrap_or_default()) + }); + } + ids.into_iter() .filter_map(|post_id| { bds_core::db::queries::post::get_post_by_id(db.conn(), &post_id).ok() @@ -6199,6 +6516,7 @@ impl BdsApp { blog_languages: Vec::new(), }, ); + let semantic_was_enabled = meta.semantic_similarity_enabled; meta.name = state.project_name.clone(); meta.description = { let value = state.project_description.text(); @@ -6249,6 +6567,8 @@ impl BdsApp { let file_result = engine::meta::write_project_json(data_dir, &meta); match (db_result, file_result) { (Ok(()), Ok(())) => { + let semantic_should_backfill = + state.semantic_similarity_enabled && !semantic_was_enabled; if let Some(listing) = self.projects.iter_mut().find(|p| p.id == project.id) { @@ -6258,6 +6578,9 @@ impl BdsApp { self.blog_languages = state.blog_languages.clone(); self.dashboard_state = Some(self.hydrate_dashboard_state()); self.notify(ToastLevel::Success, &t(self.ui_locale, "editor.saved")); + if semantic_should_backfill { + return Task::done(Message::EmbeddingBackfill); + } } (Err(e), _) => self.notify_operation_failed("common.save", e), (_, Err(e)) => self.notify_operation_failed("common.save", e), @@ -7927,6 +8250,12 @@ impl BdsApp { } } +impl Drop for BdsApp { + fn drop(&mut self) { + let _ = engine::embedding::EmbeddingService::flush_all(); + } +} + fn content_sample(content: &str, max_len: usize) -> String { content.chars().take(max_len).collect() } @@ -9184,7 +9513,10 @@ mod tests { let mut app = make_app(db, project, &tmp); let _ = app.refresh_counts(); - let dash = app.dashboard_state.expect("dashboard state should be set"); + let dash = app + .dashboard_state + .clone() + .expect("dashboard state should be set"); let now = chrono::Utc::now(); assert_eq!(dash.stats.total_posts, 2); assert_eq!(dash.stats.published_count, 1); @@ -9552,10 +9884,10 @@ mod tests { db_path.as_path(), &project.id, "en", + Some(tmp.path()), &filter, false, - 50, - 0, + (50, 0), ); assert_eq!(posts.len(), 1); diff --git a/crates/bds-ui/src/app/editor_handlers.rs b/crates/bds-ui/src/app/editor_handlers.rs index 3764adf..0d8b6fd 100644 --- a/crates/bds-ui/src/app/editor_handlers.rs +++ b/crates/bds-ui/src/app/editor_handlers.rs @@ -148,6 +148,19 @@ impl BdsApp { } state.tags_input.clear(); } + PostEditorMsg::AddSuggestedTag(tag) => { + if !state + .tags + .iter() + .any(|current| current.eq_ignore_ascii_case(&tag)) + { + state.tags.push(tag.clone()); + state.mark_dirty(); + } + state + .semantic_tag_suggestions + .retain(|candidate| !candidate.eq_ignore_ascii_case(&tag)); + } PostEditorMsg::RemoveTag(tag) => { state.tags.retain(|t| t != &tag); state.mark_dirty(); diff --git a/crates/bds-ui/src/app/search.rs b/crates/bds-ui/src/app/search.rs index 651fbc2..d8b3267 100644 --- a/crates/bds-ui/src/app/search.rs +++ b/crates/bds-ui/src/app/search.rs @@ -1,4 +1,5 @@ use super::*; +use chrono::Datelike; impl BdsApp { /// Number of items to load per sidebar page. @@ -14,6 +15,7 @@ impl BdsApp { let db_path = self.db_path.clone(); let project_id = project.id.clone(); let content_language = self.content_language.clone(); + let data_dir = self.data_dir.clone(); let filter = match self.sidebar_view { SidebarView::Pages => self.page_filter.clone(), _ => self.post_filter.clone(), @@ -27,10 +29,10 @@ impl BdsApp { &db_path, &project_id, &content_language, + data_dir.as_deref(), &filter, is_pages, - Self::SIDEBAR_PAGE_SIZE + 1, - 0, + (Self::SIDEBAR_PAGE_SIZE + 1, 0), ) }) .await @@ -99,6 +101,7 @@ impl BdsApp { let db_path = self.db_path.clone(); let project_id = project.id.clone(); let content_language = self.content_language.clone(); + let data_dir = self.data_dir.clone(); let offset = self.sidebar_posts.len() as i64; let filter = match self.sidebar_view { SidebarView::Pages => self.page_filter.clone(), @@ -113,10 +116,10 @@ impl BdsApp { &db_path, &project_id, &content_language, + data_dir.as_deref(), &filter, is_pages, - Self::SIDEBAR_PAGE_SIZE + 1, - offset, + (Self::SIDEBAR_PAGE_SIZE + 1, offset), ) }) .await @@ -163,15 +166,16 @@ impl BdsApp { db_path: &Path, project_id: &str, content_language: &str, + data_dir: Option<&Path>, filter: &PostFilter, is_pages: bool, - limit: i64, - offset: i64, + pagination: (i64, i64), ) -> Vec { let Ok(db) = Database::open(db_path) else { return Vec::new(); }; + let (limit, offset) = pagination; let params = Self::build_post_filter_params(filter, is_pages); if filter.search_query.trim().is_empty() { return bds_core::db::queries::post::list_posts_filtered( @@ -184,6 +188,11 @@ impl BdsApp { .unwrap_or_default(); } + let semantic_enabled = data_dir.is_some_and(|dir| { + engine::meta::read_project_json(dir) + .is_ok_and(|metadata| metadata.semantic_similarity_enabled) + }); + let requested = limit.saturating_add(offset).max(0) as usize; let fts_filters = bds_core::db::fts::PostSearchFilters { status: params.status.as_deref(), tags: (!params.tags.is_empty()).then_some(params.tags.as_slice()), @@ -193,12 +202,16 @@ impl BdsApp { month: params.month, from: params.from, to: params.to, - limit: Some(limit as usize), - offset: Some(offset as usize), + limit: Some(if semantic_enabled { + requested + } else { + limit as usize + }), + offset: Some(if semantic_enabled { 0 } else { offset as usize }), ..Default::default() }; - let ids = bds_core::db::fts::search_posts_filtered( + let fts_ids = bds_core::db::fts::search_posts_filtered( db.conn(), ¶ms.search_query, content_language, @@ -207,11 +220,60 @@ impl BdsApp { .map(|results| results.post_ids) .unwrap_or_default(); - ids.into_iter() + let mut ids = Vec::new(); + if semantic_enabled + && let Some(data_dir) = data_dir + && let Ok(similar) = + engine::embedding::EmbeddingService::production(db.conn(), data_dir) + .semantic_search(project_id, ¶ms.search_query, requested) + { + ids.extend(similar.into_iter().map(|item| item.post_id)); + } + ids.extend(fts_ids); + let mut seen = HashSet::new(); + let posts = ids + .into_iter() + .filter(|post_id| seen.insert(post_id.clone())) .filter_map(|post_id| { bds_core::db::queries::post::get_post_by_id(db.conn(), &post_id).ok() }) .filter(|post| post.project_id == project_id) + .filter(|post| { + params + .status + .as_ref() + .is_none_or(|status| post.status.as_str() == status) + }) + .filter(|post| { + params + .language + .as_ref() + .is_none_or(|language| post.language.as_deref() == Some(language)) + }) + .filter(|post| { + params + .tags + .iter() + .all(|wanted| post.tags.iter().any(|tag| tag.eq_ignore_ascii_case(wanted))) + }) + .filter(|post| { + params.categories.iter().all(|wanted| { + post.categories + .iter() + .any(|category| category.eq_ignore_ascii_case(wanted)) + }) + }) + .filter(|post| params.from.is_none_or(|from| post.created_at >= from)) + .filter(|post| params.to.is_none_or(|to| post.created_at <= to)) + .filter(|post| { + if params.year.is_none() && params.month.is_none() { + return true; + } + chrono::DateTime::from_timestamp_millis(post.created_at).is_some_and(|date| { + params.year.is_none_or(|year| date.year() == year) + && params.month.is_none_or(|month| date.month() == month) + }) + }) .filter(|post| { let is_page_post = post .categories @@ -222,8 +284,12 @@ impl BdsApp { } else { !is_page_post } - }) - .collect() + }); + if semantic_enabled { + posts.skip(offset as usize).take(limit as usize).collect() + } else { + posts.collect() + } } pub(super) fn query_sidebar_media_blocking( diff --git a/crates/bds-ui/src/platform/menu.rs b/crates/bds-ui/src/platform/menu.rs index 8b89cef..49f260b 100644 --- a/crates/bds-ui/src/platform/menu.rs +++ b/crates/bds-ui/src/platform/menu.rs @@ -35,6 +35,8 @@ pub enum MenuAction { EditMenu, RebuildDatabase, ReindexText, + RebuildEmbeddingIndex, + FindDuplicates, MetadataDiff, RegenerateCalendar, ValidateTranslations, @@ -69,6 +71,8 @@ impl MenuAction { MenuAction::EditMenu, MenuAction::RebuildDatabase, MenuAction::ReindexText, + MenuAction::RebuildEmbeddingIndex, + MenuAction::FindDuplicates, MenuAction::MetadataDiff, MenuAction::RegenerateCalendar, MenuAction::ValidateTranslations, @@ -101,6 +105,8 @@ impl MenuAction { "edit_menu" => Self::EditMenu, "rebuild_database" => Self::RebuildDatabase, "reindex_text" => Self::ReindexText, + "rebuild_embedding_index" => Self::RebuildEmbeddingIndex, + "find_duplicates" => Self::FindDuplicates, "metadata_diff" => Self::MetadataDiff, "regenerate_calendar" => Self::RegenerateCalendar, "validate_translations" => Self::ValidateTranslations, @@ -136,6 +142,8 @@ impl MenuAction { Self::EditMenu => "menu.item.editMenu", Self::RebuildDatabase => "menu.item.rebuildDatabase", Self::ReindexText => "menu.item.reindexText", + Self::RebuildEmbeddingIndex => "menu.item.rebuildEmbeddingIndex", + Self::FindDuplicates => "menu.item.findDuplicates", Self::MetadataDiff => "menu.item.metadataDiff", Self::RegenerateCalendar => "menu.item.regenerateCalendar", Self::ValidateTranslations => "menu.item.validateTranslations", @@ -173,6 +181,8 @@ pub(crate) fn action_enabled( | MenuAction::EditMenu | MenuAction::RebuildDatabase | MenuAction::ReindexText + | MenuAction::RebuildEmbeddingIndex + | MenuAction::FindDuplicates | MenuAction::MetadataDiff | MenuAction::RegenerateCalendar | MenuAction::ValidateTranslations @@ -205,6 +215,8 @@ pub(crate) fn action_enabled( | MenuAction::EditMenu | MenuAction::RebuildDatabase | MenuAction::ReindexText + | MenuAction::RebuildEmbeddingIndex + | MenuAction::FindDuplicates | MenuAction::MetadataDiff | MenuAction::RegenerateCalendar | MenuAction::ValidateTranslations @@ -413,6 +425,13 @@ pub fn build_menu_bar(locale: UiLocale) -> (Menu, MenuRegistry) { let _ = blog_menu.append(&PredefinedMenuItem::separator()); let _ = blog_menu.append(&item(&mut reg, MenuAction::RebuildDatabase, locale, None)); let _ = blog_menu.append(&item(&mut reg, MenuAction::ReindexText, locale, None)); + let _ = blog_menu.append(&item( + &mut reg, + MenuAction::RebuildEmbeddingIndex, + locale, + None, + )); + let _ = blog_menu.append(&item(&mut reg, MenuAction::FindDuplicates, locale, None)); let _ = blog_menu.append(&item(&mut reg, MenuAction::MetadataDiff, locale, None)); let _ = blog_menu.append(&PredefinedMenuItem::separator()); let _ = blog_menu.append(&item( diff --git a/crates/bds-ui/src/views/duplicates.rs b/crates/bds-ui/src/views/duplicates.rs new file mode 100644 index 0000000..1c33390 --- /dev/null +++ b/crates/bds-ui/src/views/duplicates.rs @@ -0,0 +1,182 @@ +use std::collections::HashSet; + +use bds_core::engine::embedding::DuplicateSearchResult; +use bds_core::i18n::UiLocale; +use iced::widget::text::Shaping; +use iced::widget::{Space, button, checkbox, column, container, row, scrollable, text}; +use iced::{Color, Element, Length}; + +use crate::app::Message; +use crate::components::inputs; +use crate::i18n::{t, tw}; + +#[derive(Debug, Clone, Default)] +pub struct DuplicatesState { + pub enabled: bool, + pub is_loading: bool, + pub has_run: bool, + pub page: usize, + pub result: DuplicateSearchResult, + pub selected: HashSet<(String, String)>, + pub error: Option, +} + +pub fn view(state: &DuplicatesState, locale: UiLocale) -> Element<'_, Message> { + let refresh = if state.is_loading { + button(text(t(locale, "duplicates.searching")).size(13)).style(inputs::secondary_button) + } else { + button(text(t(locale, "common.refresh")).size(13)) + .on_press(Message::DuplicatesRefresh) + .style(inputs::secondary_button) + } + .padding([6, 16]); + + let dismiss_checked = if state.selected.is_empty() || state.is_loading { + button(text(t(locale, "duplicates.dismissChecked")).size(13)) + .style(inputs::secondary_button) + } else { + button( + text(tw( + locale, + "duplicates.dismissCheckedCount", + &[("count", &state.selected.len().to_string())], + )) + .size(13), + ) + .on_press(Message::DuplicatesDismissSelected) + .style(inputs::primary_button) + } + .padding([6, 16]); + + let toolbar = inputs::toolbar( + vec![ + text(t(locale, "duplicates.title")) + .size(20) + .shaping(Shaping::Advanced) + .into(), + text(tw( + locale, + "duplicates.count", + &[("count", &state.result.pairs.len().to_string())], + )) + .size(12) + .color(inputs::LABEL_COLOR) + .into(), + ], + vec![ + button(text(t(locale, "duplicates.checkAll")).size(13)) + .on_press(Message::DuplicatesCheckAll) + .padding([6, 12]) + .style(inputs::secondary_button) + .into(), + button(text(t(locale, "duplicates.uncheckAll")).size(13)) + .on_press(Message::DuplicatesUncheckAll) + .padding([6, 12]) + .style(inputs::secondary_button) + .into(), + dismiss_checked.into(), + refresh.into(), + ], + ); + + let body: Element<'_, Message> = if !state.enabled { + inputs::card( + text(t(locale, "duplicates.disabled")) + .size(14) + .color(inputs::LABEL_COLOR), + ) + .into() + } else if state.is_loading && !state.has_run { + inputs::card( + text(t(locale, "duplicates.searching")) + .size(14) + .color(inputs::LABEL_COLOR), + ) + .into() + } else if let Some(error) = &state.error { + inputs::card( + text(error.clone()) + .size(14) + .color(Color::from_rgb(0.90, 0.38, 0.38)), + ) + .into() + } else if state.has_run && state.result.pairs.is_empty() { + inputs::card( + text(t(locale, "duplicates.empty")) + .size(14) + .color(inputs::LABEL_COLOR), + ) + .into() + } else { + let mut pairs = column!().spacing(8); + for pair in &state.result.pairs { + let key = (pair.post_id_a.clone(), pair.post_id_b.clone()); + let checked = state.selected.contains(&key); + let badge = if pair.exact_match { + t(locale, "duplicates.exactMatch") + } else { + format!("{:.1}%", pair.similarity * 100.0) + }; + pairs = pairs.push(inputs::card( + row![ + checkbox("", checked) + .on_toggle({ + let a = pair.post_id_a.clone(); + let b = pair.post_id_b.clone(); + move |_| Message::DuplicatesToggle(a.clone(), b.clone()) + }) + .size(16), + button( + text(pair.title_a.clone()) + .size(13) + .shaping(Shaping::Advanced) + ) + .on_press(Message::DuplicatesOpenPost(pair.post_id_a.clone())) + .padding([5, 8]) + .style(inputs::disclosure_button), + text("→").size(14).color(inputs::LABEL_COLOR), + button( + text(pair.title_b.clone()) + .size(13) + .shaping(Shaping::Advanced) + ) + .on_press(Message::DuplicatesOpenPost(pair.post_id_b.clone())) + .padding([5, 8]) + .style(inputs::disclosure_button), + Space::with_width(Length::Fill), + text(badge).size(12).color(if pair.exact_match { + Color::from_rgb(0.96, 0.68, 0.28) + } else { + Color::from_rgb(0.55, 0.76, 0.92) + }), + button(text(t(locale, "duplicates.dismiss")).size(12)) + .on_press(Message::DuplicatesDismiss( + pair.post_id_a.clone(), + pair.post_id_b.clone() + )) + .padding([5, 12]) + .style(inputs::secondary_button), + ] + .spacing(8) + .align_y(iced::Alignment::Center), + )); + } + if state.result.has_more { + pairs = pairs.push( + button(text(t(locale, "duplicates.showMore")).size(13)) + .on_press(Message::DuplicatesShowMore) + .padding([7, 16]) + .style(inputs::secondary_button), + ); + } + scrollable(container(pairs).padding(2)) + .height(Length::Fill) + .into() + }; + + container(column![toolbar, body].spacing(12)) + .padding(16) + .width(Length::Fill) + .height(Length::Fill) + .into() +} diff --git a/crates/bds-ui/src/views/mod.rs b/crates/bds-ui/src/views/mod.rs index 069b15c..161ed9c 100644 --- a/crates/bds-ui/src/views/mod.rs +++ b/crates/bds-ui/src/views/mod.rs @@ -2,6 +2,7 @@ pub mod activity_bar; pub mod chat_surfaces; pub mod chat_view; pub mod dashboard; +pub mod duplicates; pub mod git; pub mod import_editor; pub mod media_editor; diff --git a/crates/bds-ui/src/views/post_editor.rs b/crates/bds-ui/src/views/post_editor.rs index 03c22b0..cb82cfb 100644 --- a/crates/bds-ui/src/views/post_editor.rs +++ b/crates/bds-ui/src/views/post_editor.rs @@ -76,6 +76,7 @@ pub struct PostEditorState { pub quick_actions_open: bool, pub tags_input: String, pub categories_input: String, + pub semantic_tag_suggestions: Vec, pub active_language: String, pub canonical_language: String, pub blog_languages: Vec, @@ -123,6 +124,7 @@ impl Clone for PostEditorState { quick_actions_open: self.quick_actions_open, tags_input: self.tags_input.clone(), categories_input: self.categories_input.clone(), + semantic_tag_suggestions: self.semantic_tag_suggestions.clone(), active_language: self.active_language.clone(), canonical_language: self.canonical_language.clone(), blog_languages: self.blog_languages.clone(), @@ -188,6 +190,7 @@ impl PostEditorState { quick_actions_open: false, tags_input: String::new(), categories_input: String::new(), + semantic_tag_suggestions: Vec::new(), active_language: canonical_lang.clone(), canonical_language: canonical_lang, blog_languages: blog_languages.to_vec(), @@ -342,6 +345,7 @@ pub enum PostEditorMsg { SwitchLanguage(String), TagsInputChanged(String), TagsInputSubmit, + AddSuggestedTag(String), RemoveTag(String), CategoriesInputChanged(String), CategoriesInputSubmit, @@ -619,6 +623,28 @@ pub fn view<'a>( Message::PostEditor(PostEditorMsg::TagsInputSubmit), |tag| Message::PostEditor(PostEditorMsg::RemoveTag(tag)), ); + let semantic_tags: Element<'a, Message> = if state.semantic_tag_suggestions.is_empty() { + Space::new(0, 0).into() + } else { + let mut chips = row![ + text(t(locale, "editor.semanticTagSuggestions")) + .size(11) + .color(inputs::LABEL_COLOR) + ] + .spacing(6) + .align_y(iced::Alignment::Center); + for tag in &state.semantic_tag_suggestions { + chips = chips.push( + button(text(format!("+ {tag}")).size(11)) + .on_press(Message::PostEditor(PostEditorMsg::AddSuggestedTag( + tag.clone(), + ))) + .padding([4, 8]) + .style(inputs::secondary_button), + ); + } + chips.into() + }; // Categories chip input let categories_section = chip_input_field( @@ -776,6 +802,7 @@ pub fn view<'a>( meta_row1, meta_row2, tags_section, + semantic_tags, categories_section, outlinks_section, backlinks_section, diff --git a/crates/bds-ui/src/views/workspace.rs b/crates/bds-ui/src/views/workspace.rs index c813781..3bcac80 100644 --- a/crates/bds-ui/src/views/workspace.rs +++ b/crates/bds-ui/src/views/workspace.rs @@ -18,6 +18,7 @@ use crate::views::{ activity_bar, chat_view::{self, ChatEditorState}, dashboard::DashboardState, + duplicates::{self, DuplicatesState}, git::{self, GitDiffState, GitUiState}, media_editor::{self, MediaEditorState}, metadata_diff::{self, MetadataDiffState}, @@ -128,6 +129,7 @@ pub fn view<'a>( settings_state: Option<&'a SettingsViewState>, dashboard_state: Option<&'a DashboardState>, site_validation_state: &'a SiteValidationState, + duplicates_state: &'a DuplicatesState, metadata_diff_state: &'a MetadataDiffState, translation_validation_state: &'a TranslationValidationState, git_state: &'a GitUiState, @@ -158,6 +160,7 @@ pub fn view<'a>( settings_state, dashboard_state, site_validation_state, + duplicates_state, metadata_diff_state, translation_validation_state, git_diffs, @@ -400,6 +403,7 @@ fn route_content_area<'a>( settings_state: Option<&'a SettingsViewState>, dashboard_state: Option<&'a DashboardState>, site_validation_state: &'a SiteValidationState, + duplicates_state: &'a DuplicatesState, metadata_diff_state: &'a MetadataDiffState, translation_validation_state: &'a TranslationValidationState, git_diffs: &'a HashMap, @@ -489,6 +493,7 @@ fn route_content_area<'a>( } } ContentRoute::SiteValidation => site_validation::view(site_validation_state, locale), + ContentRoute::FindDuplicates => duplicates::view(duplicates_state, locale), ContentRoute::MetadataDiff => metadata_diff::view(metadata_diff_state, locale), ContentRoute::TranslationValidation => { translation_validation::view(translation_validation_state, locale) @@ -524,6 +529,7 @@ enum ContentRoute<'a> { Tags, Settings, SiteValidation, + FindDuplicates, MetadataDiff, TranslationValidation, GitDiff(&'a str), @@ -612,11 +618,11 @@ fn route_kind<'a>( TabType::SiteValidation => ContentRoute::SiteValidation, TabType::MetadataDiff => ContentRoute::MetadataDiff, TabType::GitDiff => ContentRoute::GitDiff(tab_id), + TabType::FindDuplicates => ContentRoute::FindDuplicates, TabType::Style | TabType::MenuEditor | TabType::Documentation - | TabType::ApiDocumentation - | TabType::FindDuplicates => ContentRoute::Placeholder(&tab.title), + | TabType::ApiDocumentation => ContentRoute::Placeholder(&tab.title), TabType::TranslationValidation => ContentRoute::TranslationValidation, } } @@ -678,7 +684,6 @@ mod tests { TabType::MenuEditor, TabType::Documentation, TabType::ApiDocumentation, - TabType::FindDuplicates, ]; for tab_type in unsupported { @@ -703,6 +708,35 @@ mod tests { } } + #[test] + fn duplicate_tab_routes_to_real_review_surface() { + let tabs = vec![tab( + "find_duplicates", + TabType::FindDuplicates, + "Duplicates", + )]; + let empty_posts = HashMap::new(); + let empty_media = HashMap::new(); + let empty_templates = HashMap::new(); + let empty_scripts = HashMap::new(); + let empty_imports = HashMap::new(); + let site_validation = SiteValidationState::default(); + let route = route_kind( + &tabs, + Some("find_duplicates"), + &empty_posts, + &empty_media, + &empty_templates, + &empty_scripts, + &empty_imports, + None, + None, + None, + &site_validation, + ); + assert!(matches!(route, ContentRoute::FindDuplicates)); + } + #[test] fn git_diff_tab_routes_to_real_view() { let empty_posts = HashMap::new(); diff --git a/docs/scripting/API_REFERENCE.md b/docs/scripting/API_REFERENCE.md index a53cb46..79a49c2 100644 --- a/docs/scripting/API_REFERENCE.md +++ b/docs/scripting/API_REFERENCE.md @@ -32,6 +32,7 @@ Macro scripts expose `render(input, context)` and transform scripts expose `main - [Root helpers](#root-helpers) - [`bds.app`](#bdsapp) - [`bds.chat`](#bdschat) +- [`bds.embeddings`](#bdsembeddings) - [`bds.media`](#bdsmedia) - [`bds.meta`](#bdsmeta) - [`bds.posts`](#bdsposts) @@ -795,6 +796,230 @@ local result = bds.chat.translate_media_metadata("example-id", "en") { key = "value" } ``` +## `bds.embeddings` + +### `bds.embeddings.compute_similarities` + +Compute similarity scores from one source post to target posts. + +**Signature** + +```text +bds.embeddings.compute_similarities(post_id: string, target_ids: table) -> table | nil +``` + +**Parameters** + +| Name | Type | Required | Example | +| --- | --- | --- | --- | +| `post_id` | `string` | Yes | `"example-id"` | +| `target_ids` | `table` | Yes | `{ title = "Example" }` | + +**Returns** + +`table | nil`. `nil` means no value was available or the host operation failed. + +**Example call** + +```lua +local result = bds.embeddings.compute_similarities("example-id", { title = "Example" }) +``` + +**Example response** + +```lua +{ key = "value" } +``` + +### `bds.embeddings.dismiss_pair` + +Dismiss a duplicate candidate pair. + +**Signature** + +```text +bds.embeddings.dismiss_pair(post_id_a: string, post_id_b: string) -> boolean +``` + +**Parameters** + +| Name | Type | Required | Example | +| --- | --- | --- | --- | +| `post_id_a` | `string` | Yes | `"example"` | +| `post_id_b` | `string` | Yes | `"example"` | + +**Returns** + +`boolean`. `false` means the operation was rejected or failed. + +**Example call** + +```lua +local result = bds.embeddings.dismiss_pair("example", "example") +``` + +**Example response** + +```lua +true +``` + +### `bds.embeddings.find_duplicates` + +Find duplicate post candidates for the current project. + +**Signature** + +```text +bds.embeddings.find_duplicates() -> table | nil +``` + +**Parameters** + +None. + +**Returns** + +`table | nil`. `nil` means no value was available or the host operation failed. + +**Example call** + +```lua +local result = bds.embeddings.find_duplicates() +``` + +**Example response** + +```lua +{ key = "value" } +``` + +### `bds.embeddings.find_similar` + +Find posts similar to the given post id. + +**Signature** + +```text +bds.embeddings.find_similar(post_id: string, limit?: integer) -> table | nil +``` + +**Parameters** + +| Name | Type | Required | Example | +| --- | --- | --- | --- | +| `post_id` | `string` | Yes | `"example-id"` | +| `limit` | `integer` | No | `nil` | + +**Returns** + +`table | nil`. `nil` means no value was available or the host operation failed. + +**Example call** + +```lua +local result = bds.embeddings.find_similar("example-id", nil) +``` + +**Example response** + +```lua +{ key = "value" } +``` + +### `bds.embeddings.get_progress` + +Get embedding index progress for the current project. + +**Signature** + +```text +bds.embeddings.get_progress() -> table | nil +``` + +**Parameters** + +None. + +**Returns** + +`table | nil`. `nil` means no value was available or the host operation failed. + +**Example call** + +```lua +local result = bds.embeddings.get_progress() +``` + +**Example response** + +```lua +{ key = "value" } +``` + +### `bds.embeddings.index_unindexed_posts` + +Index posts missing embeddings for the current project. + +**Signature** + +```text +bds.embeddings.index_unindexed_posts() -> table | nil +``` + +**Parameters** + +None. + +**Returns** + +`table | nil`. `nil` means no value was available or the host operation failed. + +**Example call** + +```lua +local result = bds.embeddings.index_unindexed_posts() +``` + +**Example response** + +```lua +{ key = "value" } +``` + +### `bds.embeddings.suggest_tags` + +Suggest tags for a post from semantic similarity. + +**Signature** + +```text +bds.embeddings.suggest_tags(post_id: string, exclude_tags?: table) -> table | nil +``` + +**Parameters** + +| Name | Type | Required | Example | +| --- | --- | --- | --- | +| `post_id` | `string` | Yes | `"example-id"` | +| `exclude_tags` | `table` | No | `{ title = "Example" }` | + +**Returns** + +`table | nil`. `nil` means no value was available or the host operation failed. + +**Example call** + +```lua +local result = bds.embeddings.suggest_tags("example-id", { title = "Example" }) +``` + +**Example response** + +```lua +{ key = "value" } +``` + ## `bds.media` ### `bds.media.delete_translation` diff --git a/docs/scripting/api.json b/docs/scripting/api.json index 964468d..e2ce122 100644 --- a/docs/scripting/api.json +++ b/docs/scripting/api.json @@ -1543,6 +1543,99 @@ } ], "returns": "table | nil" + }, + { + "description": "Compute similarity scores from one source post to target posts.", + "name": "compute_similarities", + "namespace": "embeddings", + "params": [ + { + "name": "post_id", + "required": true, + "type": "string" + }, + { + "name": "target_ids", + "required": true, + "type": "table" + } + ], + "returns": "table | nil" + }, + { + "description": "Dismiss a duplicate candidate pair.", + "name": "dismiss_pair", + "namespace": "embeddings", + "params": [ + { + "name": "post_id_a", + "required": true, + "type": "string" + }, + { + "name": "post_id_b", + "required": true, + "type": "string" + } + ], + "returns": "boolean" + }, + { + "description": "Find duplicate post candidates for the current project.", + "name": "find_duplicates", + "namespace": "embeddings", + "params": [], + "returns": "table | nil" + }, + { + "description": "Find posts similar to the given post id.", + "name": "find_similar", + "namespace": "embeddings", + "params": [ + { + "name": "post_id", + "required": true, + "type": "string" + }, + { + "name": "limit", + "required": false, + "type": "integer" + } + ], + "returns": "table | nil" + }, + { + "description": "Get embedding index progress for the current project.", + "name": "get_progress", + "namespace": "embeddings", + "params": [], + "returns": "table | nil" + }, + { + "description": "Index posts missing embeddings for the current project.", + "name": "index_unindexed_posts", + "namespace": "embeddings", + "params": [], + "returns": "table | nil" + }, + { + "description": "Suggest tags for a post from semantic similarity.", + "name": "suggest_tags", + "namespace": "embeddings", + "params": [ + { + "name": "post_id", + "required": true, + "type": "string" + }, + { + "name": "exclude_tags", + "required": false, + "type": "table" + } + ], + "returns": "table | nil" } ], "root_methods": [ diff --git a/docs/scripting/bds2-core-signatures.json b/docs/scripting/bds2-core-signatures.json index 7273e7f..4c3d3b3 100644 --- a/docs/scripting/bds2-core-signatures.json +++ b/docs/scripting/bds2-core-signatures.json @@ -1 +1 @@ -{"version":"0.4.0","methods":[{"namespace":"app","name":"copy_to_clipboard","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"get_blogmark_bookmarklet","params":[],"returns":"string"},{"namespace":"app","name":"get_data_paths","params":[],"returns":"table"},{"namespace":"app","name":"get_default_project_path","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_system_language","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_title_bar_metrics","params":[],"returns":"table | nil"},{"namespace":"app","name":"log","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"notify_renderer_ready","params":[],"returns":"boolean"},{"namespace":"app","name":"open_folder","params":[{"name":"folder_path","type":"string","required":true}],"returns":"string"},{"namespace":"app","name":"read_project_metadata","params":[{"name":"folder_path","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"app","name":"select_folder","params":[{"name":"title","type":"string","required":false}],"returns":"string | nil"},{"namespace":"app","name":"set_preview_post_target","params":[{"name":"post_id","type":"string","required":false}],"returns":"boolean"},{"namespace":"app","name":"show_item_in_folder","params":[{"name":"item_path","type":"string","required":true}],"returns":"nil"},{"namespace":"app","name":"trigger_menu_action","params":[{"name":"action","type":"string","required":true}],"returns":"nil"},{"namespace":"projects","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"delete_with_data","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"get_all","params":[],"returns":"ProjectData[]"},{"namespace":"projects","name":"get_active","params":[],"returns":"ProjectData | nil"},{"namespace":"projects","name":"set_active","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"posts","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"posts","name":"discard","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"filter","params":[{"name":"filters","type":"table","required":true}],"returns":"PostData[] | nil"},{"namespace":"posts","name":"generate_unique_slug","params":[{"name":"title","type":"string","required":true},{"name":"exclude_post_id","type":"string","required":false}],"returns":"string"},{"namespace":"posts","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_all","params":[],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_slug","params":[{"name":"slug","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_by_status","params":[{"name":"status","type":"string","required":true}],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_year_month","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_categories","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_categories_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_dashboard_stats","params":[],"returns":"table"},{"namespace":"posts","name":"get_linked_by","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_links_to","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_preview_url","params":[{"name":"post_id","type":"string","required":true},{"name":"options","type":"table","required":false}],"returns":"string | nil"},{"namespace":"posts","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_tags_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_translation","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"posts","name":"get_translations","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"has_published_version","params":[{"name":"post_id","type":"string","required":true}],"returns":"boolean"},{"namespace":"posts","name":"is_slug_available","params":[{"name":"slug","type":"string","required":true},{"name":"exclude_post_id","type":"string","required":false}],"returns":"boolean"},{"namespace":"posts","name":"publish","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"publish_translation","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"posts","name":"rebuild_from_files","params":[],"returns":"PostData[] | nil"},{"namespace":"posts","name":"rebuild_links","params":[],"returns":"boolean"},{"namespace":"posts","name":"reindex_text","params":[],"returns":"boolean"},{"namespace":"posts","name":"search","params":[{"name":"query","type":"string","required":true}],"returns":"PostData[] | nil"},{"namespace":"media","name":"delete_translation","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"boolean"},{"namespace":"media","name":"filter","params":[{"name":"filters","type":"table","required":true}],"returns":"MediaData[]"},{"namespace":"media","name":"import","params":[{"name":"data","type":"table","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"get_by_year_month","params":[],"returns":"table[]"},{"namespace":"media","name":"get_file_path","params":[{"name":"media_id","type":"string","required":true}],"returns":"string | nil"},{"namespace":"media","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"media","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"get_all","params":[],"returns":"MediaData[]"},{"namespace":"media","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"media","name":"get_tags_with_counts","params":[],"returns":"table[]"},{"namespace":"media","name":"get_thumbnail","params":[{"name":"media_id","type":"string","required":true},{"name":"size","type":"string","required":false}],"returns":"string | nil"},{"namespace":"media","name":"get_translation","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"media","name":"get_translations","params":[{"name":"media_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"media","name":"get_url","params":[{"name":"media_id","type":"string","required":true}],"returns":"string | nil"},{"namespace":"media","name":"rebuild_from_files","params":[],"returns":"MediaData[] | nil"},{"namespace":"media","name":"regenerate_missing_thumbnails","params":[],"returns":"table"},{"namespace":"media","name":"regenerate_thumbnails","params":[{"name":"media_id","type":"string","required":true}],"returns":"table | nil"},{"namespace":"media","name":"reindex_text","params":[],"returns":"boolean"},{"namespace":"media","name":"replace_file","params":[{"name":"media_id","type":"string","required":true},{"name":"source_path","type":"string","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"search","params":[{"name":"query","type":"string","required":true}],"returns":"MediaData[] | nil"},{"namespace":"media","name":"upsert_translation","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"table | nil"},{"namespace":"scripts","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"scripts","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"get_all","params":[],"returns":"ScriptData[]"},{"namespace":"scripts","name":"publish","params":[{"name":"id","type":"string","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"rebuild_from_files","params":[],"returns":"ScriptData[] | nil"},{"namespace":"templates","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"templates","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"get_all","params":[],"returns":"TemplateData[]"},{"namespace":"templates","name":"get_enabled_by_kind","params":[{"name":"kind","type":"string","required":true}],"returns":"TemplateData[]"},{"namespace":"templates","name":"publish","params":[{"name":"id","type":"string","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"rebuild_from_files","params":[],"returns":"TemplateData[] | nil"},{"namespace":"templates","name":"validate","params":[{"name":"content","type":"string","required":true}],"returns":"ValidationResult | nil"},{"namespace":"meta","name":"add_category","params":[{"name":"name","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"meta","name":"add_tag","params":[{"name":"name","type":"string","required":true}],"returns":"string[]"},{"namespace":"meta","name":"clear_publishing_preferences","params":[],"returns":"table | nil"},{"namespace":"meta","name":"get_categories","params":[],"returns":"string[]"},{"namespace":"meta","name":"get_project_metadata","params":[],"returns":"ProjectMetadata"},{"namespace":"meta","name":"get_publishing_preferences","params":[],"returns":"table | nil"},{"namespace":"meta","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"meta","name":"remove_category","params":[{"name":"name","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"meta","name":"remove_tag","params":[{"name":"name","type":"string","required":true}],"returns":"string[]"},{"namespace":"meta","name":"set_project_metadata","params":[{"name":"updates","type":"table","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"meta","name":"set_publishing_preferences","params":[{"name":"prefs","type":"table","required":true}],"returns":"table | nil"},{"namespace":"meta","name":"sync_on_startup","params":[],"returns":"table"},{"namespace":"meta","name":"update_project_metadata","params":[{"name":"updates","type":"table","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"tags","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"tags","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"get_all","params":[],"returns":"TagData[]"},{"namespace":"tags","name":"get_by_name","params":[{"name":"name","type":"string","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"get_posts_with_tag","params":[{"name":"tag_id","type":"string","required":true}],"returns":"string[]"},{"namespace":"tags","name":"get_with_counts","params":[],"returns":"table[]"},{"namespace":"tags","name":"merge","params":[{"name":"source_tag_ids","type":"table","required":true},{"name":"target_tag_id","type":"string","required":true}],"returns":"boolean"},{"namespace":"tags","name":"rename","params":[{"name":"id","type":"string","required":true},{"name":"new_name","type":"string","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"sync_from_posts","params":[],"returns":"TagData[] | nil"},{"namespace":"tasks","name":"cancel","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"tasks","name":"clear_completed","params":[],"returns":"boolean"},{"namespace":"tasks","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"TaskData | nil"},{"namespace":"tasks","name":"get_all","params":[],"returns":"TaskData[]"},{"namespace":"tasks","name":"get_running","params":[],"returns":"TaskData[]"},{"namespace":"tasks","name":"status_snapshot","params":[],"returns":"TaskStatus"},{"namespace":"publish","name":"upload_site","params":[{"name":"credentials","type":"table","required":true}],"returns":"TaskData | nil"},{"namespace":"chat","name":"analyze_media_image","params":[{"name":"media_id","type":"string","required":true}],"returns":"table | nil"},{"namespace":"chat","name":"analyze_post","params":[{"name":"post_id","type":"string","required":true}],"returns":"table | nil"},{"namespace":"chat","name":"detect_media_language","params":[{"name":"title","type":"string","required":true},{"name":"alt","type":"string","required":false},{"name":"caption","type":"string","required":false}],"returns":"table"},{"namespace":"chat","name":"detect_post_language","params":[{"name":"title","type":"string","required":true},{"name":"content","type":"string","required":true}],"returns":"table"},{"namespace":"chat","name":"translate_media_metadata","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"chat","name":"translate_post","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"}]} +{"version":"0.4.0","methods":[{"namespace":"app","name":"copy_to_clipboard","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"get_blogmark_bookmarklet","params":[],"returns":"string"},{"namespace":"app","name":"get_data_paths","params":[],"returns":"table"},{"namespace":"app","name":"get_default_project_path","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_system_language","params":[],"returns":"string | nil"},{"namespace":"app","name":"get_title_bar_metrics","params":[],"returns":"table | nil"},{"namespace":"app","name":"log","params":[{"name":"text","type":"string","required":true}],"returns":"boolean"},{"namespace":"app","name":"notify_renderer_ready","params":[],"returns":"boolean"},{"namespace":"app","name":"open_folder","params":[{"name":"folder_path","type":"string","required":true}],"returns":"string"},{"namespace":"app","name":"read_project_metadata","params":[{"name":"folder_path","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"app","name":"select_folder","params":[{"name":"title","type":"string","required":false}],"returns":"string | nil"},{"namespace":"app","name":"set_preview_post_target","params":[{"name":"post_id","type":"string","required":false}],"returns":"boolean"},{"namespace":"app","name":"show_item_in_folder","params":[{"name":"item_path","type":"string","required":true}],"returns":"nil"},{"namespace":"app","name":"trigger_menu_action","params":[{"name":"action","type":"string","required":true}],"returns":"nil"},{"namespace":"projects","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"delete_with_data","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"projects","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"get_all","params":[],"returns":"ProjectData[]"},{"namespace":"projects","name":"get_active","params":[],"returns":"ProjectData | nil"},{"namespace":"projects","name":"set_active","params":[{"name":"id","type":"string","required":true}],"returns":"ProjectData | nil"},{"namespace":"projects","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"ProjectData | nil"},{"namespace":"posts","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"posts","name":"discard","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"filter","params":[{"name":"filters","type":"table","required":true}],"returns":"PostData[] | nil"},{"namespace":"posts","name":"generate_unique_slug","params":[{"name":"title","type":"string","required":true},{"name":"exclude_post_id","type":"string","required":false}],"returns":"string"},{"namespace":"posts","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_all","params":[],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_slug","params":[{"name":"slug","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"get_by_status","params":[{"name":"status","type":"string","required":true}],"returns":"PostData[]"},{"namespace":"posts","name":"get_by_year_month","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_categories","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_categories_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_dashboard_stats","params":[],"returns":"table"},{"namespace":"posts","name":"get_linked_by","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_links_to","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"get_preview_url","params":[{"name":"post_id","type":"string","required":true},{"name":"options","type":"table","required":false}],"returns":"string | nil"},{"namespace":"posts","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"posts","name":"get_tags_with_counts","params":[],"returns":"table[]"},{"namespace":"posts","name":"get_translation","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"posts","name":"get_translations","params":[{"name":"post_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"posts","name":"has_published_version","params":[{"name":"post_id","type":"string","required":true}],"returns":"boolean"},{"namespace":"posts","name":"is_slug_available","params":[{"name":"slug","type":"string","required":true},{"name":"exclude_post_id","type":"string","required":false}],"returns":"boolean"},{"namespace":"posts","name":"publish","params":[{"name":"id","type":"string","required":true}],"returns":"PostData | nil"},{"namespace":"posts","name":"publish_translation","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"posts","name":"rebuild_from_files","params":[],"returns":"PostData[] | nil"},{"namespace":"posts","name":"rebuild_links","params":[],"returns":"boolean"},{"namespace":"posts","name":"reindex_text","params":[],"returns":"boolean"},{"namespace":"posts","name":"search","params":[{"name":"query","type":"string","required":true}],"returns":"PostData[] | nil"},{"namespace":"media","name":"delete_translation","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"boolean"},{"namespace":"media","name":"filter","params":[{"name":"filters","type":"table","required":true}],"returns":"MediaData[]"},{"namespace":"media","name":"import","params":[{"name":"data","type":"table","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"get_by_year_month","params":[],"returns":"table[]"},{"namespace":"media","name":"get_file_path","params":[{"name":"media_id","type":"string","required":true}],"returns":"string | nil"},{"namespace":"media","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"media","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"get_all","params":[],"returns":"MediaData[]"},{"namespace":"media","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"media","name":"get_tags_with_counts","params":[],"returns":"table[]"},{"namespace":"media","name":"get_thumbnail","params":[{"name":"media_id","type":"string","required":true},{"name":"size","type":"string","required":false}],"returns":"string | nil"},{"namespace":"media","name":"get_translation","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"media","name":"get_translations","params":[{"name":"media_id","type":"string","required":true}],"returns":"table[]"},{"namespace":"media","name":"get_url","params":[{"name":"media_id","type":"string","required":true}],"returns":"string | nil"},{"namespace":"media","name":"rebuild_from_files","params":[],"returns":"MediaData[] | nil"},{"namespace":"media","name":"regenerate_missing_thumbnails","params":[],"returns":"table"},{"namespace":"media","name":"regenerate_thumbnails","params":[{"name":"media_id","type":"string","required":true}],"returns":"table | nil"},{"namespace":"media","name":"reindex_text","params":[],"returns":"boolean"},{"namespace":"media","name":"replace_file","params":[{"name":"media_id","type":"string","required":true},{"name":"source_path","type":"string","required":true}],"returns":"MediaData | nil"},{"namespace":"media","name":"search","params":[{"name":"query","type":"string","required":true}],"returns":"MediaData[] | nil"},{"namespace":"media","name":"upsert_translation","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"table | nil"},{"namespace":"scripts","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"scripts","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"get_all","params":[],"returns":"ScriptData[]"},{"namespace":"scripts","name":"publish","params":[{"name":"id","type":"string","required":true}],"returns":"ScriptData | nil"},{"namespace":"scripts","name":"rebuild_from_files","params":[],"returns":"ScriptData[] | nil"},{"namespace":"templates","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"templates","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"get_all","params":[],"returns":"TemplateData[]"},{"namespace":"templates","name":"get_enabled_by_kind","params":[{"name":"kind","type":"string","required":true}],"returns":"TemplateData[]"},{"namespace":"templates","name":"publish","params":[{"name":"id","type":"string","required":true}],"returns":"TemplateData | nil"},{"namespace":"templates","name":"rebuild_from_files","params":[],"returns":"TemplateData[] | nil"},{"namespace":"templates","name":"validate","params":[{"name":"content","type":"string","required":true}],"returns":"ValidationResult | nil"},{"namespace":"meta","name":"add_category","params":[{"name":"name","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"meta","name":"add_tag","params":[{"name":"name","type":"string","required":true}],"returns":"string[]"},{"namespace":"meta","name":"clear_publishing_preferences","params":[],"returns":"table | nil"},{"namespace":"meta","name":"get_categories","params":[],"returns":"string[]"},{"namespace":"meta","name":"get_project_metadata","params":[],"returns":"ProjectMetadata"},{"namespace":"meta","name":"get_publishing_preferences","params":[],"returns":"table | nil"},{"namespace":"meta","name":"get_tags","params":[],"returns":"string[]"},{"namespace":"meta","name":"remove_category","params":[{"name":"name","type":"string","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"meta","name":"remove_tag","params":[{"name":"name","type":"string","required":true}],"returns":"string[]"},{"namespace":"meta","name":"set_project_metadata","params":[{"name":"updates","type":"table","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"meta","name":"set_publishing_preferences","params":[{"name":"prefs","type":"table","required":true}],"returns":"table | nil"},{"namespace":"meta","name":"sync_on_startup","params":[],"returns":"table"},{"namespace":"meta","name":"update_project_metadata","params":[{"name":"updates","type":"table","required":true}],"returns":"ProjectMetadata | nil"},{"namespace":"tags","name":"create","params":[{"name":"data","type":"table","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"update","params":[{"name":"id","type":"string","required":true},{"name":"data","type":"table","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"delete","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"tags","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"get_all","params":[],"returns":"TagData[]"},{"namespace":"tags","name":"get_by_name","params":[{"name":"name","type":"string","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"get_posts_with_tag","params":[{"name":"tag_id","type":"string","required":true}],"returns":"string[]"},{"namespace":"tags","name":"get_with_counts","params":[],"returns":"table[]"},{"namespace":"tags","name":"merge","params":[{"name":"source_tag_ids","type":"table","required":true},{"name":"target_tag_id","type":"string","required":true}],"returns":"boolean"},{"namespace":"tags","name":"rename","params":[{"name":"id","type":"string","required":true},{"name":"new_name","type":"string","required":true}],"returns":"TagData | nil"},{"namespace":"tags","name":"sync_from_posts","params":[],"returns":"TagData[] | nil"},{"namespace":"tasks","name":"cancel","params":[{"name":"id","type":"string","required":true}],"returns":"boolean"},{"namespace":"tasks","name":"clear_completed","params":[],"returns":"boolean"},{"namespace":"tasks","name":"get","params":[{"name":"id","type":"string","required":true}],"returns":"TaskData | nil"},{"namespace":"tasks","name":"get_all","params":[],"returns":"TaskData[]"},{"namespace":"tasks","name":"get_running","params":[],"returns":"TaskData[]"},{"namespace":"tasks","name":"status_snapshot","params":[],"returns":"TaskStatus"},{"namespace":"publish","name":"upload_site","params":[{"name":"credentials","type":"table","required":true}],"returns":"TaskData | nil"},{"namespace":"chat","name":"analyze_media_image","params":[{"name":"media_id","type":"string","required":true}],"returns":"table | nil"},{"namespace":"chat","name":"analyze_post","params":[{"name":"post_id","type":"string","required":true}],"returns":"table | nil"},{"namespace":"chat","name":"detect_media_language","params":[{"name":"title","type":"string","required":true},{"name":"alt","type":"string","required":false},{"name":"caption","type":"string","required":false}],"returns":"table"},{"namespace":"chat","name":"detect_post_language","params":[{"name":"title","type":"string","required":true},{"name":"content","type":"string","required":true}],"returns":"table"},{"namespace":"chat","name":"translate_media_metadata","params":[{"name":"media_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"chat","name":"translate_post","params":[{"name":"post_id","type":"string","required":true},{"name":"language","type":"string","required":true}],"returns":"table | nil"},{"namespace":"embeddings","name":"compute_similarities","params":[{"name":"post_id","type":"string","required":true},{"name":"target_ids","type":"table","required":true}],"returns":"table | nil"},{"namespace":"embeddings","name":"dismiss_pair","params":[{"name":"post_id_a","type":"string","required":true},{"name":"post_id_b","type":"string","required":true}],"returns":"boolean"},{"namespace":"embeddings","name":"find_duplicates","params":[],"returns":"table | nil"},{"namespace":"embeddings","name":"find_similar","params":[{"name":"post_id","type":"string","required":true},{"name":"limit","type":"integer","required":false}],"returns":"table | nil"},{"namespace":"embeddings","name":"get_progress","params":[],"returns":"table | nil"},{"namespace":"embeddings","name":"index_unindexed_posts","params":[],"returns":"table | nil"},{"namespace":"embeddings","name":"suggest_tags","params":[{"name":"post_id","type":"string","required":true},{"name":"exclude_tags","type":"table","required":false}],"returns":"table | nil"}]} diff --git a/docs/scripting/completions.json b/docs/scripting/completions.json index 94d631b..050e512 100644 --- a/docs/scripting/completions.json +++ b/docs/scripting/completions.json @@ -1669,6 +1669,106 @@ ], "returns": "table | nil" }, + { + "description": "Compute similarity scores from one source post to target posts.", + "detail": "bds.embeddings.compute_similarities(post_id: string, target_ids: table) -> table | nil", + "insert_text": "bds.embeddings.compute_similarities(\"example-id\", { title = \"Example\" })", + "label": "bds.embeddings.compute_similarities", + "parameters": [ + { + "name": "post_id", + "required": true, + "type": "string" + }, + { + "name": "target_ids", + "required": true, + "type": "table" + } + ], + "returns": "table | nil" + }, + { + "description": "Dismiss a duplicate candidate pair.", + "detail": "bds.embeddings.dismiss_pair(post_id_a: string, post_id_b: string) -> boolean", + "insert_text": "bds.embeddings.dismiss_pair(\"example\", \"example\")", + "label": "bds.embeddings.dismiss_pair", + "parameters": [ + { + "name": "post_id_a", + "required": true, + "type": "string" + }, + { + "name": "post_id_b", + "required": true, + "type": "string" + } + ], + "returns": "boolean" + }, + { + "description": "Find duplicate post candidates for the current project.", + "detail": "bds.embeddings.find_duplicates() -> table | nil", + "insert_text": "bds.embeddings.find_duplicates()", + "label": "bds.embeddings.find_duplicates", + "parameters": [], + "returns": "table | nil" + }, + { + "description": "Find posts similar to the given post id.", + "detail": "bds.embeddings.find_similar(post_id: string, limit?: integer) -> table | nil", + "insert_text": "bds.embeddings.find_similar(\"example-id\", nil)", + "label": "bds.embeddings.find_similar", + "parameters": [ + { + "name": "post_id", + "required": true, + "type": "string" + }, + { + "name": "limit", + "required": false, + "type": "integer" + } + ], + "returns": "table | nil" + }, + { + "description": "Get embedding index progress for the current project.", + "detail": "bds.embeddings.get_progress() -> table | nil", + "insert_text": "bds.embeddings.get_progress()", + "label": "bds.embeddings.get_progress", + "parameters": [], + "returns": "table | nil" + }, + { + "description": "Index posts missing embeddings for the current project.", + "detail": "bds.embeddings.index_unindexed_posts() -> table | nil", + "insert_text": "bds.embeddings.index_unindexed_posts()", + "label": "bds.embeddings.index_unindexed_posts", + "parameters": [], + "returns": "table | nil" + }, + { + "description": "Suggest tags for a post from semantic similarity.", + "detail": "bds.embeddings.suggest_tags(post_id: string, exclude_tags?: table) -> table | nil", + "insert_text": "bds.embeddings.suggest_tags(\"example-id\", { title = \"Example\" })", + "label": "bds.embeddings.suggest_tags", + "parameters": [ + { + "name": "post_id", + "required": true, + "type": "string" + }, + { + "name": "exclude_tags", + "required": false, + "type": "table" + } + ], + "returns": "table | nil" + }, { "description": "Report progress for the current managed job.", "detail": "bds.report_progress(payload: table) -> boolean", diff --git a/locales/ui/de.ftl b/locales/ui/de.ftl index 5df4a2f..8b5d988 100644 --- a/locales/ui/de.ftl +++ b/locales/ui/de.ftl @@ -19,6 +19,27 @@ menu-item-publishSelected = Ausgewählte veröffentlichen menu-item-previewPost = Beitragsvorschau menu-item-rebuildDatabase = Datenbank aus Dateien neu aufbauen menu-item-reindexText = Suchtext neu indizieren +menu-item-rebuildEmbeddingIndex = Embedding-Index neu erstellen +menu-item-findDuplicates = Doppelte Beiträge finden +common-refresh = Aktualisieren +editor-semanticTagSuggestions = Vorschläge aus ähnlichen Beiträgen +duplicates-title = Doppelte Beiträge finden +duplicates-searching = Duplikate werden gesucht… +duplicates-count = { $count } mögliche Paare +duplicates-disabled = Semantische Ähnlichkeit ist für dieses Projekt nicht aktiviert. +duplicates-empty = Keine möglichen doppelten Beiträge gefunden. +duplicates-checkAll = Alle auswählen +duplicates-uncheckAll = Auswahl aufheben +duplicates-dismissChecked = Ausgewählte verwerfen +duplicates-dismissCheckedCount = Ausgewählte verwerfen ({ $count }) +duplicates-exactMatch = Exakte Übereinstimmung +duplicates-dismiss = Verwerfen +duplicates-dismissed = Duplikatpaar wurde verworfen. +duplicates-showMore = Mehr anzeigen +embeddings-reindexed = Embeddings für { $count } Beiträge neu erstellt. +embeddings-indexing = Semantische Inhalte werden indexiert +embeddings-indexingProgress = Beitrag { $current } von { $total } wird indexiert +embeddings-indexed = Der semantische Index ist für { $count } Beiträge bereit. menu-item-metadataDiff = Metadaten-Diff-Werkzeug menu-item-editMenu = Blog-Menü bearbeiten menu-item-generateSitemap = Site rendern diff --git a/locales/ui/en.ftl b/locales/ui/en.ftl index 745e7a6..56b55ce 100644 --- a/locales/ui/en.ftl +++ b/locales/ui/en.ftl @@ -19,6 +19,27 @@ menu-item-publishSelected = Publish Selected menu-item-previewPost = Preview Post menu-item-rebuildDatabase = Rebuild Database from Files menu-item-reindexText = Reindex Search Text +menu-item-rebuildEmbeddingIndex = Rebuild Embedding Index +menu-item-findDuplicates = Find Duplicate Posts +common-refresh = Refresh +editor-semanticTagSuggestions = Suggested from similar posts +duplicates-title = Find Duplicate Posts +duplicates-searching = Searching for duplicates… +duplicates-count = { $count } potential pairs +duplicates-disabled = Semantic similarity is not enabled for this project. +duplicates-empty = No potential duplicate posts found. +duplicates-checkAll = Check All +duplicates-uncheckAll = Uncheck All +duplicates-dismissChecked = Dismiss Checked +duplicates-dismissCheckedCount = Dismiss Checked ({ $count }) +duplicates-exactMatch = Exact Match +duplicates-dismiss = Dismiss +duplicates-dismissed = Duplicate pair dismissal saved. +duplicates-showMore = Show More +embeddings-reindexed = Rebuilt embeddings for { $count } posts. +embeddings-indexing = Indexing semantic content +embeddings-indexingProgress = Indexing post { $current } of { $total } +embeddings-indexed = Semantic index is ready for { $count } posts. menu-item-metadataDiff = Metadata Diff Tool menu-item-editMenu = Edit Blog Menu menu-item-generateSitemap = Render Site diff --git a/locales/ui/es.ftl b/locales/ui/es.ftl index df415d5..246d0ad 100644 --- a/locales/ui/es.ftl +++ b/locales/ui/es.ftl @@ -19,6 +19,27 @@ menu-item-publishSelected = Publicar selección menu-item-previewPost = Vista previa de entrada menu-item-rebuildDatabase = Reconstruir base de datos desde archivos menu-item-reindexText = Reindexar texto de búsqueda +menu-item-rebuildEmbeddingIndex = Reconstruir índice semántico +menu-item-findDuplicates = Buscar publicaciones duplicadas +common-refresh = Actualizar +editor-semanticTagSuggestions = Sugerencias de publicaciones similares +duplicates-title = Buscar publicaciones duplicadas +duplicates-searching = Buscando duplicados… +duplicates-count = { $count } pares potenciales +duplicates-disabled = La similitud semántica no está activada para este proyecto. +duplicates-empty = No se encontraron publicaciones posiblemente duplicadas. +duplicates-checkAll = Marcar todo +duplicates-uncheckAll = Desmarcar todo +duplicates-dismissChecked = Descartar marcados +duplicates-dismissCheckedCount = Descartar marcados ({ $count }) +duplicates-exactMatch = Coincidencia exacta +duplicates-dismiss = Descartar +duplicates-dismissed = Se descartó el par duplicado. +duplicates-showMore = Mostrar más +embeddings-reindexed = Se reconstruyeron los embeddings de { $count } publicaciones. +embeddings-indexing = Indexando contenido semántico +embeddings-indexingProgress = Indexando publicación { $current } de { $total } +embeddings-indexed = El índice semántico está listo para { $count } publicaciones. menu-item-metadataDiff = Herramienta diff de metadatos menu-item-editMenu = Editar menú del blog menu-item-generateSitemap = Renderizar sitio diff --git a/locales/ui/fr.ftl b/locales/ui/fr.ftl index 99bfb82..cddc526 100644 --- a/locales/ui/fr.ftl +++ b/locales/ui/fr.ftl @@ -19,6 +19,27 @@ menu-item-publishSelected = Publier la sélection menu-item-previewPost = Aperçu de l'article menu-item-rebuildDatabase = Reconstruire la base de données menu-item-reindexText = Réindexer le texte de recherche +menu-item-rebuildEmbeddingIndex = Reconstruire l’index sémantique +menu-item-findDuplicates = Rechercher les articles en double +common-refresh = Actualiser +editor-semanticTagSuggestions = Suggestions d’articles similaires +duplicates-title = Rechercher les articles en double +duplicates-searching = Recherche des doublons… +duplicates-count = { $count } paires potentielles +duplicates-disabled = La similarité sémantique n’est pas activée pour ce projet. +duplicates-empty = Aucun doublon potentiel trouvé. +duplicates-checkAll = Tout cocher +duplicates-uncheckAll = Tout décocher +duplicates-dismissChecked = Ignorer la sélection +duplicates-dismissCheckedCount = Ignorer la sélection ({ $count }) +duplicates-exactMatch = Correspondance exacte +duplicates-dismiss = Ignorer +duplicates-dismissed = La paire de doublons a été ignorée. +duplicates-showMore = Afficher plus +embeddings-reindexed = Embeddings reconstruits pour { $count } articles. +embeddings-indexing = Indexation du contenu sémantique +embeddings-indexingProgress = Indexation de l’article { $current } sur { $total } +embeddings-indexed = L’index sémantique est prêt pour { $count } articles. menu-item-metadataDiff = Outil de diff des métadonnées menu-item-editMenu = Modifier le menu du blog menu-item-generateSitemap = Rendre le site diff --git a/locales/ui/it.ftl b/locales/ui/it.ftl index 137abb3..131e9c2 100644 --- a/locales/ui/it.ftl +++ b/locales/ui/it.ftl @@ -19,6 +19,27 @@ menu-item-publishSelected = Pubblica selezionato menu-item-previewPost = Anteprima post menu-item-rebuildDatabase = Ricostruisci database dai file menu-item-reindexText = Reindicizza testo di ricerca +menu-item-rebuildEmbeddingIndex = Ricostruisci indice semantico +menu-item-findDuplicates = Trova post duplicati +common-refresh = Aggiorna +editor-semanticTagSuggestions = Suggerimenti da post simili +duplicates-title = Trova post duplicati +duplicates-searching = Ricerca duplicati… +duplicates-count = { $count } coppie potenziali +duplicates-disabled = La somiglianza semantica non è abilitata per questo progetto. +duplicates-empty = Nessun possibile post duplicato trovato. +duplicates-checkAll = Seleziona tutto +duplicates-uncheckAll = Deseleziona tutto +duplicates-dismissChecked = Ignora selezionati +duplicates-dismissCheckedCount = Ignora selezionati ({ $count }) +duplicates-exactMatch = Corrispondenza esatta +duplicates-dismiss = Ignora +duplicates-dismissed = Coppia duplicata ignorata. +duplicates-showMore = Mostra altro +embeddings-reindexed = Embedding ricostruiti per { $count } post. +embeddings-indexing = Indicizzazione dei contenuti semantici +embeddings-indexingProgress = Indicizzazione del post { $current } di { $total } +embeddings-indexed = L’indice semantico è pronto per { $count } post. menu-item-metadataDiff = Strumento diff metadati menu-item-editMenu = Modifica menu blog menu-item-generateSitemap = Renderizza sito diff --git a/specs/script.allium b/specs/script.allium index 9af3dcb..31aefb7 100644 --- a/specs/script.allium +++ b/specs/script.allium @@ -142,7 +142,9 @@ surface ScriptRuntimeSurface { -- the core one-shot AI bridge; conversational chat remains extension. -- bds.report_progress(payload) reports managed-job progress. The Rust -- host may retain bds.app.progress and bds.app.toast as additive APIs. - -- bds.sync and bds.embeddings belong to the extension contract. + -- bds.embeddings: compute_similarities, dismiss_pair, find_duplicates, + -- find_similar, get_progress, index_unindexed_posts, suggest_tags. + -- bds.sync remains a future extension contract. @guarantee HostApiFailureValues -- Host failures do not expose implementation exceptions across the Lua