hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Implement semantic embeddings and duplicate review

Browse Source
This commit is contained in:
Chili Palmer
2026-07-19 18:06:47 +02:00
parent fd4fd744e0
commit e8764262a3
42 changed files with 3808 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
crates/bds-cli/src/lib.rs
View File

@@ -317,12 +317,6 @@ fn active_project(db: &Database) -> Result<(Project, PathBuf)> {
fn rebuild(db: &Database, incremental: bool) -> Result<CommandOutput> {
let (project, data_dir) = active_project(db)?;
let metadata = engine::meta::read_project_json(&data_dir)?;
if metadata.semantic_similarity_enabled {
bail!(
"semantic similarity is enabled, but rebuild cannot continue without the real on-device embedding engine"
);
}
if incremental {
let (applied, imported, failed) = cli_sync::run_cli_mutation(db.conn(), || {
let report =
@@ -490,9 +484,13 @@ fn repair(db: &Database, part: RepairPart) -> Result<CommandOutput> {
RepairPart::Embeddings => {
let metadata = engine::meta::read_project_json(&data_dir)?;
if metadata.semantic_similarity_enabled {
bail!(
"semantic similarity is enabled, but no real on-device embedding engine is registered"
);
let service = engine::embedding::EmbeddingService::production(db.conn(), &data_dir);
let indexed = service.reindex_all(&project.id)?;
service.flush_project(&project.id)?;
return Ok(output(
"Embedding index rebuilt",
json!({"rebuilt": indexed.len(), "disabled": false}),
));
}
Ok(output(
"Embedding repair skipped because semantic similarity is disabled",
@@ -1431,8 +1429,9 @@ mod tests {
let mut metadata = engine::meta::read_project_json(&fixture.project_dir).unwrap();
metadata.semantic_similarity_enabled = true;
engine::meta::write_project_json(&fixture.project_dir, &metadata).unwrap();
assert!(fixture.run(&["repair", "embeddings"], "").is_err());
assert!(fixture.run(&["rebuild"], "").is_err());
let repaired = fixture.run(&["repair", "embeddings"], "").unwrap();
assert_eq!(repaired.data["rebuilt"], 0);
fixture.run(&["rebuild"], "").unwrap();
metadata.semantic_similarity_enabled = false;
engine::meta::write_project_json(&fixture.project_dir, &metadata).unwrap();
fixture.run(&["render"], "").unwrap();

3
crates/bds-core/Cargo.toml
View File

@@ -38,6 +38,9 @@ mlua = { workspace = true }
url = { workspace = true }
base64 = { workspace = true }
dirs = { workspace = true }
fastembed = { workspace = true }
usearch = { workspace = true }
ort = { workspace = true }
[dev-dependencies]
tempfile = "3"

5
crates/bds-core/migrations/20260718000000_initial_schema/up.sql
View File

@@ -258,9 +258,12 @@ CREATE TABLE IF NOT EXISTS embedding_keys (
post_id TEXT NOT NULL,
project_id TEXT NOT NULL,
content_hash TEXT NOT NULL,
vector TEXT NOT NULL
vector BLOB NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS embedding_keys_project_post_idx
ON embedding_keys(project_id, post_id);
CREATE TABLE IF NOT EXISTS dismissed_duplicate_pairs (
id TEXT NOT NULL PRIMARY KEY,
project_id TEXT NOT NULL REFERENCES projects(id),

15
crates/bds-core/migrations/20260719090000_embedding_vector_blob/down.sql Normal file
View File

@@ -0,0 +1,15 @@
-- This file should undo anything in `up.sql`
CREATE TABLE embedding_keys_old (
label INTEGER NOT NULL PRIMARY KEY,
post_id TEXT NOT NULL,
project_id TEXT NOT NULL,
content_hash TEXT NOT NULL,
vector TEXT NOT NULL
);
INSERT INTO embedding_keys_old (label, post_id, project_id, content_hash, vector)
SELECT label, post_id, project_id, content_hash, CAST(vector AS TEXT)
FROM embedding_keys;
DROP TABLE embedding_keys;
ALTER TABLE embedding_keys_old RENAME TO embedding_keys;

18
crates/bds-core/migrations/20260719090000_embedding_vector_blob/up.sql Normal file
View File

@@ -0,0 +1,18 @@
-- Your SQL goes here
CREATE TABLE embedding_keys_new (
label INTEGER NOT NULL PRIMARY KEY,
post_id TEXT NOT NULL,
project_id TEXT NOT NULL,
content_hash TEXT NOT NULL,
vector BLOB NOT NULL
);
INSERT INTO embedding_keys_new (label, post_id, project_id, content_hash, vector)
SELECT label, post_id, project_id, content_hash, CAST(vector AS BLOB)
FROM embedding_keys;
DROP TABLE embedding_keys;
ALTER TABLE embedding_keys_new RENAME TO embedding_keys;
CREATE UNIQUE INDEX embedding_keys_project_post_idx
ON embedding_keys(project_id, post_id);

2
crates/bds-core/src/db/migrations.rs
View File

@@ -36,7 +36,7 @@ mod tests {
let applied = db
.conn()
.with_migrations(|conn| conn.applied_migrations().unwrap().len());
assert_eq!(applied, 6);
assert_eq!(applied, 7);
}
#[test]

283
crates/bds-core/src/db/queries/embedding.rs Normal file
View File

@@ -0,0 +1,283 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::{dismissed_duplicate_pairs, embedding_keys};
use crate::model::{DismissedDuplicatePair, EmbeddingKey};
pub fn get_key_for_post(
conn: &DbConnection,
project_id: &str,
post_id: &str,
) -> QueryResult<Option<EmbeddingKey>> {
conn.with(|c| {
embedding_keys::table
.filter(embedding_keys::project_id.eq(project_id))
.filter(embedding_keys::post_id.eq(post_id))
.select(EmbeddingKey::as_select())
.first(c)
.optional()
})
}
pub fn list_keys(conn: &DbConnection, project_id: &str) -> QueryResult<Vec<EmbeddingKey>> {
conn.with(|c| {
embedding_keys::table
.filter(embedding_keys::project_id.eq(project_id))
.order(embedding_keys::label.asc())
.select(EmbeddingKey::as_select())
.load(c)
})
}
pub fn max_label(conn: &DbConnection) -> QueryResult<i64> {
conn.with(|c| {
embedding_keys::table
.select(embedding_keys::label)
.order(embedding_keys::label.desc())
.first(c)
.optional()
.map(|label| label.unwrap_or(0))
})
}
pub fn upsert_key(conn: &DbConnection, key: &EmbeddingKey) -> QueryResult<()> {
conn.with(|c| {
diesel::insert_into(embedding_keys::table)
.values(key)
.on_conflict((embedding_keys::project_id, embedding_keys::post_id))
.do_update()
.set((
embedding_keys::content_hash.eq(&key.content_hash),
embedding_keys::vector.eq(&key.vector),
))
.execute(c)
.map(|_| ())
})
}
pub fn delete_key_for_post(
conn: &DbConnection,
project_id: &str,
post_id: &str,
) -> QueryResult<()> {
conn.with(|c| {
diesel::delete(
embedding_keys::table
.filter(embedding_keys::project_id.eq(project_id))
.filter(embedding_keys::post_id.eq(post_id)),
)
.execute(c)
.map(|_| ())
})
}
pub fn delete_stale_keys(
conn: &DbConnection,
project_id: &str,
live_post_ids: &[String],
) -> QueryResult<usize> {
conn.with(|c| {
let query = embedding_keys::table.filter(embedding_keys::project_id.eq(project_id));
if live_post_ids.is_empty() {
diesel::delete(query).execute(c)
} else {
diesel::delete(query.filter(embedding_keys::post_id.ne_all(live_post_ids))).execute(c)
}
})
}
pub fn insert_dismissed_pair(
conn: &DbConnection,
pair: &DismissedDuplicatePair,
) -> QueryResult<()> {
conn.with(|c| {
diesel::insert_into(dismissed_duplicate_pairs::table)
.values(pair)
.on_conflict((
dismissed_duplicate_pairs::project_id,
dismissed_duplicate_pairs::post_id_a,
dismissed_duplicate_pairs::post_id_b,
))
.do_nothing()
.execute(c)
.map(|_| ())
})
}
pub fn insert_dismissed_pairs(
conn: &DbConnection,
pairs: &[DismissedDuplicatePair],
) -> QueryResult<usize> {
if pairs.is_empty() {
return Ok(0);
}
conn.with(|c| {
diesel::insert_into(dismissed_duplicate_pairs::table)
.values(pairs)
.on_conflict((
dismissed_duplicate_pairs::project_id,
dismissed_duplicate_pairs::post_id_a,
dismissed_duplicate_pairs::post_id_b,
))
.do_nothing()
.execute(c)
})
}
pub fn list_dismissed_pairs(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<DismissedDuplicatePair>> {
conn.with(|c| {
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id))
.select(DismissedDuplicatePair::as_select())
.load(c)
})
}
pub fn delete_orphan_dismissals(
conn: &DbConnection,
project_id: &str,
live_post_ids: &[String],
) -> QueryResult<usize> {
conn.with(|c| {
let query = dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id));
if live_post_ids.is_empty() {
diesel::delete(query).execute(c)
} else {
diesel::delete(
query.filter(
dismissed_duplicate_pairs::post_id_a
.ne_all(live_post_ids)
.or(dismissed_duplicate_pairs::post_id_b.ne_all(live_post_ids)),
),
)
.execute(c)
}
})
}
pub fn delete_dismissals_for_post(
conn: &DbConnection,
project_id: &str,
post_id: &str,
) -> QueryResult<usize> {
conn.with(|c| {
diesel::delete(
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id))
.filter(
dismissed_duplicate_pairs::post_id_a
.eq(post_id)
.or(dismissed_duplicate_pairs::post_id_b.eq(post_id)),
),
)
.execute(c)
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::model::{Post, PostStatus, Project};
fn seeded() -> (Database, String, String) {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
let project_id = "embedding-project".to_string();
crate::db::queries::project::insert_project(
db.conn(),
&Project {
id: project_id.clone(),
name: "Embedding".into(),
slug: "embedding".into(),
description: None,
data_path: Some("/tmp/embedding".into()),
is_active: true,
created_at: 1,
updated_at: 1,
},
)
.unwrap();
let post_id = "embedding-post".to_string();
crate::db::queries::post::insert_post(
db.conn(),
&Post {
id: post_id.clone(),
project_id: project_id.clone(),
title: "Post".into(),
slug: "post".into(),
excerpt: None,
content: Some("Body".into()),
status: PostStatus::Draft,
author: None,
language: Some("en".into()),
do_not_translate: false,
template_slug: None,
file_path: "posts/post.md".into(),
checksum: None,
tags: vec![],
categories: vec![],
published_title: None,
published_content: None,
published_tags: None,
published_categories: None,
published_excerpt: None,
created_at: 1,
updated_at: 1,
published_at: None,
},
)
.unwrap();
(db, project_id, post_id)
}
#[test]
fn embedding_vector_round_trips_as_blob_and_dismissals_are_canonical() {
let (db, project_id, post_id) = seeded();
let key = EmbeddingKey {
label: 1,
post_id: post_id.clone(),
project_id: project_id.clone(),
content_hash: "hash".into(),
vector: vec![0, 1, 255],
};
upsert_key(db.conn(), &key).unwrap();
assert_eq!(
get_key_for_post(db.conn(), &project_id, &post_id).unwrap(),
Some(key)
);
let replacement = EmbeddingKey {
label: 2,
post_id: post_id.clone(),
project_id: project_id.clone(),
content_hash: "new-hash".into(),
vector: vec![3, 2, 1],
};
upsert_key(db.conn(), &replacement).unwrap();
let updated = get_key_for_post(db.conn(), &project_id, &post_id)
.unwrap()
.unwrap();
assert_eq!(updated.label, 1, "a post keeps its stable HNSW label");
assert_eq!(updated.content_hash, "new-hash");
assert_eq!(updated.vector, vec![3, 2, 1]);
let pair = DismissedDuplicatePair {
id: "dismissal".into(),
project_id: project_id.clone(),
post_id_a: "a".into(),
post_id_b: "b".into(),
dismissed_at: 1,
};
insert_dismissed_pair(db.conn(), &pair).unwrap();
insert_dismissed_pair(db.conn(), &pair).unwrap();
assert_eq!(
list_dismissed_pairs(db.conn(), &project_id).unwrap(),
vec![pair]
);
}
}

1
crates/bds-core/src/db/queries/mod.rs
View File

@@ -1,5 +1,6 @@
pub mod chat;
pub mod db_notification;
pub mod embedding;
pub mod generated_file_hash;
pub mod import_definition;
pub mod mcp_proposal;

2
crates/bds-core/src/db/schema.rs
View File

@@ -115,7 +115,7 @@ diesel::table! {
post_id -> Text,
project_id -> Text,
content_hash -> Text,
vector -> Text,
vector -> Binary,
}
}

1382
crates/bds-core/src/engine/embedding.rs Normal file
View File

File diff suppressed because it is too large Load Diff

51
crates/bds-core/src/engine/metadata_diff.rs
View File

@@ -5,6 +5,7 @@ use std::path::Path;
use crate::db::DbConnection as Connection;
use walkdir::WalkDir;
use crate::db::queries::embedding as qe;
use crate::db::queries::media as qm;
use crate::db::queries::media_translation as qmt;
use crate::db::queries::post as qp;
@@ -159,6 +160,49 @@ pub fn compute_metadata_diff(
}
// 6. Detect orphans
if crate::engine::meta::read_project_json(data_dir)
.is_ok_and(|metadata| metadata.semantic_similarity_enabled)
{
let service = crate::engine::embedding::EmbeddingService::production(conn, data_dir);
let keys = qe::list_keys(conn, project_id)?
.into_iter()
.map(|key| (key.post_id.clone(), key))
.collect::<std::collections::HashMap<_, _>>();
for post in &posts {
let expected = service.content_hash_for_post(post)?;
let key = keys.get(&post.id);
let current_hash = key.map(|key| key.content_hash.as_str()).unwrap_or("");
let vector_status = key
.filter(|key| crate::engine::embedding::decode_vector(&key.vector).is_ok())
.map(|_| "ready")
.unwrap_or("missing");
if current_hash != expected || vector_status != "ready" {
let mut fields = Vec::new();
if current_hash != expected {
fields.push(DiffField {
field_name: "content_hash".into(),
db_value: current_hash.into(),
file_value: expected,
});
}
if vector_status != "ready" {
fields.push(DiffField {
field_name: "embedding".into(),
db_value: vector_status.into(),
file_value: "ready".into(),
});
}
report.diffs.push(EntityDiff {
entity_type: "embedding".into(),
entity_id: post.id.clone(),
file_path: format!("projects/{project_id}/embeddings.usearch"),
fields,
});
}
}
}
// 7. Detect orphans
let orphans = detect_orphan_files(conn, data_dir, project_id)?;
report.orphans = orphans;
@@ -204,6 +248,11 @@ pub fn repair_metadata_diff_item(
conn, data_dir, project_id, &path,
)?;
}
"embedding" => {
let post = qp::get_post_by_id(conn, &item.entity_id)?;
crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.sync_post(&post)?;
}
other => return unsupported_repair(other),
}
}
@@ -222,6 +271,8 @@ pub fn repair_metadata_diff_item(
)?,
"script" => rewrite_script_from_database(conn, data_dir, &item.entity_id)?,
"template" => rewrite_template_from_database(conn, data_dir, &item.entity_id)?,
"embedding" => crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.flush_project(project_id)?,
other => return unsupported_repair(other),
},
}

1
crates/bds-core/src/engine/mod.rs
View File

@@ -8,6 +8,7 @@ mod chat_tools;
pub mod cli_launcher;
pub mod cli_sync;
pub mod domain_events;
pub mod embedding;
pub mod error;
pub mod gallery_import;
pub mod generation;

14
crates/bds-core/src/engine/post.rs
View File

@@ -36,7 +36,7 @@ pub struct RebuildReport {
)]
pub fn create_post(
conn: &Connection,
_data_dir: &Path,
data_dir: &Path,
project_id: &str,
title: &str,
content: Option<&str>,
@@ -91,6 +91,7 @@ pub fn create_post(
fts_index_post(conn, &post)?;
emit_post(&post, NotificationAction::Created);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
@@ -102,7 +103,7 @@ pub fn create_post(
)]
pub fn update_post(
conn: &Connection,
_data_dir: &Path,
data_dir: &Path,
post_id: &str,
title: Option<&str>,
slug: Option<&str>,
@@ -169,7 +170,7 @@ pub fn update_post(
if post.status == PostStatus::Published || post.status == PostStatus::Archived {
// Reload content from filesystem if content field is NULL (published state)
if post.content.is_none() && !post.file_path.is_empty() {
let abs_path = _data_dir.join(&post.file_path);
let abs_path = data_dir.join(&post.file_path);
if abs_path.exists()
&& let Ok(file_content) = fs::read_to_string(&abs_path)
&& let Ok((_fm, body)) = read_post_file(&file_content)
@@ -187,6 +188,7 @@ pub fn update_post(
fts_index_post(conn, &post)?;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
@@ -210,6 +212,7 @@ pub fn publish_post(conn: &Connection, data_dir: &Path, post_id: &str) -> Engine
Ok(post) => {
conn.release_savepoint()?;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
Err(error) => {
@@ -343,7 +346,9 @@ pub fn archive_post(conn: &Connection, data_dir: &Path, post_id: &str) -> Engine
}
let now = now_unix_ms();
qp::update_post_status(conn, post_id, &PostStatus::Archived, now)?;
post.status = PostStatus::Archived;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(())
}
@@ -455,6 +460,7 @@ pub fn discard_post_draft(conn: &Connection, data_dir: &Path, post_id: &str) ->
Ok(post) => {
conn.release_savepoint()?;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
Err(error) => {
@@ -503,6 +509,8 @@ pub fn delete_post(conn: &Connection, data_dir: &Path, post_id: &str) -> EngineR
// Delete post from DB
qp::delete_post(conn, post_id)?;
crate::engine::embedding::remove_post_best_effort(conn, data_dir, &post.project_id, post_id);
emit_post(&post, NotificationAction::Deleted);
Ok(())

1
crates/bds-core/src/engine/project.rs
View File

@@ -238,6 +238,7 @@ pub fn delete_project(
let is_custom_path = project.data_path.is_some();
q::delete_project(conn, project_id)?;
crate::engine::embedding::EmbeddingService::forget_project(project_id);
// Clean up internal filesystem only (not custom external paths per spec)
if !is_custom_path

16
crates/bds-core/src/engine/rebuild.rs
View File

@@ -170,15 +170,18 @@ fn rebuild_from_filesystem_inner(
)));
}
progress(0.98, "Refreshing semantic index...");
crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.index_unindexed(project_id)?;
progress(1.0, "Rebuild complete");
Ok(report)
}
fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
use crate::db::schema::{
dismissed_duplicate_pairs, embedding_keys, generated_file_hashes, import_definitions,
media, media_translations, post_links, post_media, post_translations, posts, scripts, tags,
templates,
generated_file_hashes, import_definitions, media, media_translations, post_links,
post_media, post_translations, posts, scripts, tags, templates,
};
let post_ids = crate::db::queries::post::list_posts_by_project(conn, project_id)?
@@ -219,13 +222,6 @@ fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
generated_file_hashes::table.filter(generated_file_hashes::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(embedding_keys::table.filter(embedding_keys::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
import_definitions::table.filter(import_definitions::project_id.eq(project_id)),
)

50
crates/bds-core/src/model/embedding.rs Normal file
View File

@@ -0,0 +1,50 @@
use serde::{Deserialize, Serialize};
#[derive(
Debug,
Clone,
PartialEq,
Serialize,
Deserialize,
diesel::Queryable,
diesel::Selectable,
diesel::Insertable,
diesel::AsChangeset,
)]
#[diesel(
table_name = crate::db::schema::embedding_keys,
check_for_backend(diesel::sqlite::Sqlite),
treat_none_as_default_value = false
)]
pub struct EmbeddingKey {
pub label: i64,
pub post_id: String,
pub project_id: String,
pub content_hash: String,
pub vector: Vec<u8>,
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
diesel::Queryable,
diesel::Selectable,
diesel::Insertable,
diesel::AsChangeset,
)]
#[diesel(
table_name = crate::db::schema::dismissed_duplicate_pairs,
check_for_backend(diesel::sqlite::Sqlite),
treat_none_as_default_value = false
)]
pub struct DismissedDuplicatePair {
pub id: String,
pub project_id: String,
pub post_id_a: String,
pub post_id_b: String,
pub dismissed_at: i64,
}

2
crates/bds-core/src/model/mod.rs
View File

@@ -1,4 +1,5 @@
mod chat;
mod embedding;
mod event;
mod generation;
mod import;
@@ -12,6 +13,7 @@ mod tag;
mod template;
pub use chat::{ChatConversation, ChatMessage, ChatRole, NewChatConversation, NewChatMessage};
pub use embedding::{DismissedDuplicatePair, EmbeddingKey};
pub use event::DomainEvent;
pub use generation::{
DbNotification, DomainEntity, GeneratedFileHash, NotificationAction, NotificationEntity,

112
crates/bds-core/src/scripting/core_host.rs
View File

@@ -1382,6 +1382,95 @@ impl CoreHost {
(_, response) => one_shot_json(response),
}
}
fn embeddings(&self, method: &str, args: &[Value]) -> HostResult<Value> {
let db = self.database()?;
let service = engine::embedding::EmbeddingService::production(db.conn(), &self.data_dir);
match method {
"get_progress" => {
let (indexed, total) = service.indexing_progress(&self.project_id)?;
Ok(json!({"indexed": indexed, "total": total}))
}
"find_similar" => {
let post_id = string_arg(args, 0)?;
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
let limit = args.get(1).and_then(Value::as_u64).unwrap_or(5) as usize;
Ok(Value::Array(
service
.find_similar(post_id, limit)?
.into_iter()
.map(|post| {
json!({
"post_id": post.post_id,
"title": post.title,
"score": post.similarity,
})
})
.collect(),
))
}
"compute_similarities" => {
let post_id = string_arg(args, 0)?;
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
let target_ids = string_array_arg(args, 1)?;
json_value(service.compute_similarities(post_id, &target_ids))
}
"suggest_tags" => {
let post_id = string_arg(args, 0)?;
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
json_value(service.suggest_tags(post_id))
}
"find_duplicates" => {
let mut page = 0;
let pairs = loop {
let result = service.find_duplicates(&self.project_id, page)?;
if !result.has_more {
break result.pairs;
}
page += 1;
};
Ok(Value::Array(
pairs
.into_iter()
.map(|pair| {
json!({
"post_id_a": pair.post_id_a,
"title_a": pair.title_a,
"post_id_b": pair.post_id_b,
"title_b": pair.title_b,
"score": pair.similarity,
"similarity": pair.similarity,
"exact_match": pair.exact_match,
})
})
.collect(),
))
}
"dismiss_pair" => {
let post_id_a = string_arg(args, 0)?;
let post_id_b = string_arg(args, 1)?;
for post_id in [post_id_a, post_id_b] {
self.scoped(
|conn| crate::db::queries::post::get_post_by_id(conn, post_id),
|post| post.project_id.as_str(),
)?;
}
service.dismiss_duplicate_pair(post_id_a, post_id_b)?;
Ok(Value::Bool(true))
}
"index_unindexed_posts" => json_value(service.index_unindexed(&self.project_id)),
_ => Err(format!("unknown embeddings capability: {method}").into()),
}
}
}
impl HostApi for CoreHost {
@@ -1398,6 +1487,7 @@ impl HostApi for CoreHost {
"tasks" => self.tasks(method, &arguments),
"publish" => self.publish(method, &arguments),
"chat" => self.chat(method, &arguments),
"embeddings" => self.embeddings(method, &arguments),
"bds" if method == "report_progress" => self.report_progress(&arguments),
_ => Err(format!("unknown host capability: {namespace}.{method}").into()),
};
@@ -1537,6 +1627,19 @@ fn string_arg(args: &[Value], index: usize) -> HostResult<&str> {
.ok_or_else(|| format!("argument {} must be a string", index + 1).into())
}
fn string_array_arg(args: &[Value], index: usize) -> HostResult<Vec<String>> {
args.get(index)
.and_then(Value::as_array)
.map(|items| {
items
.iter()
.filter_map(Value::as_str)
.map(str::to_owned)
.collect()
})
.ok_or_else(|| format!("argument {} must be a table", index + 1).into())
}
fn string_field<'a>(value: &'a Map<String, Value>, field: &str) -> HostResult<&'a str> {
value
.get(field)
@@ -1821,6 +1924,9 @@ mod tests {
upload = upload,
timestamp = post.created_at,
project_path = bds.app.get_default_project_path(),
embedding_progress = bds.embeddings.get_progress(),
embedding_backfill = bds.embeddings.index_unindexed_posts(),
foreign_embedding = bds.embeddings.find_similar(input.foreign_post, 5),
}
end
"#,
@@ -1842,5 +1948,11 @@ mod tests {
assert!(result.value["upload"].is_null());
assert!(result.value["timestamp"].as_str().unwrap().contains('T'));
assert_eq!(manager.progress(task_id), Some(0.5));
assert_eq!(
result.value["embedding_progress"],
json!({"indexed": 0, "total": 1})
);
assert_eq!(result.value["embedding_backfill"], json!([]));
assert!(result.value["foreign_embedding"].is_null());
}
}

9
crates/bds-core/src/scripting/mod.rs
View File

@@ -456,6 +456,7 @@ mod tests {
[
"app",
"chat",
"embeddings",
"media",
"meta",
"posts",
@@ -473,7 +474,7 @@ mod tests {
function main()
return {
sync = bds.sync,
embeddings = bds.embeddings,
embeddings = type(bds.embeddings),
report_progress = type(bds.report_progress),
post_search = type(bds.posts.search),
app_toast = type(bds.app.toast),
@@ -492,6 +493,7 @@ mod tests {
"report_progress": "function",
"post_search": "function",
"app_toast": "function",
"embeddings": "table",
})
);
}
@@ -514,6 +516,7 @@ mod tests {
bds.tasks.status_snapshot(),
bds.publish.upload_site({}),
bds.chat.detect_post_language("title", "body"),
bds.embeddings.get_progress(),
}
end
"#,
@@ -525,8 +528,8 @@ mod tests {
)
.unwrap();
assert_eq!(execution.value.as_array().unwrap().len(), 11);
assert_eq!(host.0.lock().unwrap().len(), 11);
assert_eq!(execution.value.as_array().unwrap().len(), 12);
assert_eq!(host.0.lock().unwrap().len(), 12);
}
#[test]

346
crates/bds-ui/src/app.rs
View File

@@ -33,6 +33,7 @@ use crate::views::{
DashboardCategory, DashboardRecentPost, DashboardState, DashboardStats, DashboardTag,
DashboardTimelineMonth,
},
duplicates::DuplicatesState,
git::{GitDiffLoad, GitDiffState, GitNetworkCompletion, GitSnapshot, GitUiState},
import_editor::{
ImportAnalysisEvent, ImportEditorMsg, ImportEditorState, ImportExecutionEvent,
@@ -241,6 +242,23 @@ pub enum Message {
result: Result<engine::generation::GenerationReport, String>,
},
SiteValidationLoaded(Result<engine::validate_site::SiteValidationReport, String>),
DuplicatesRefresh,
DuplicatesLoaded(Result<engine::embedding::DuplicateSearchResult, String>),
DuplicatesToggle(String, String),
DuplicatesCheckAll,
DuplicatesUncheckAll,
DuplicatesDismiss(String, String),
DuplicatesDismissSelected,
DuplicatesDismissed(Result<(), String>),
DuplicatesShowMore,
DuplicatesOpenPost(String),
EmbeddingReindex,
EmbeddingBackfill,
LoadSemanticTagSuggestions(String),
SemanticTagSuggestionsLoaded {
post_id: String,
result: Result<Vec<String>, String>,
},
// Git
GitRefresh,
@@ -927,6 +945,7 @@ pub struct BdsApp {
settings_state: Option<SettingsViewState>,
dashboard_state: Option<DashboardState>,
site_validation_state: SiteValidationState,
duplicates_state: DuplicatesState,
metadata_diff_state: MetadataDiffState,
translation_validation_state: crate::views::translation_validation::TranslationValidationState,
git_state: GitUiState,
@@ -1091,6 +1110,7 @@ impl BdsApp {
settings_state: None,
dashboard_state: None,
site_validation_state: SiteValidationState::default(),
duplicates_state: DuplicatesState::default(),
metadata_diff_state: MetadataDiffState::default(),
translation_validation_state: Default::default(),
git_state: GitUiState::default(),
@@ -1169,6 +1189,7 @@ impl BdsApp {
settings_state: None,
dashboard_state: None,
site_validation_state: SiteValidationState::default(),
duplicates_state: DuplicatesState::default(),
metadata_diff_state: MetadataDiffState::default(),
translation_validation_state: Default::default(),
git_state: GitUiState::default(),
@@ -1495,14 +1516,22 @@ impl BdsApp {
Message::OpenTab(tab) => {
self.flush_active_post_editor();
let idx = tabs::open_tab(&mut self.tabs, tab);
let mut semantic_post_id = None;
if let Some(t) = self.tabs.get(idx) {
self.active_tab = Some(t.id.clone());
let tab_clone = t.clone();
if tab_clone.tab_type == TabType::Post {
semantic_post_id = Some(tab_clone.id.clone());
}
self.load_editor_for_tab(&tab_clone);
}
self.enforce_panel_tab_fallback();
self.sync_menu_state();
self.sync_embedded_preview_for_active_post()
let mut tasks = vec![self.sync_embedded_preview_for_active_post()];
if let Some(post_id) = semantic_post_id {
tasks.push(Task::done(Message::LoadSemanticTagSuggestions(post_id)));
}
Task::batch(tasks)
}
Message::CloseTab(id) => {
if self.active_tab.as_deref() == Some(id.as_str()) {
@@ -1573,17 +1602,24 @@ impl BdsApp {
}
let sidebar_task = self.refresh_counts();
self.sync_menu_state();
sidebar_task
Task::batch([sidebar_task, Task::done(Message::EmbeddingBackfill)])
}
Message::SwitchProject(project_id) => {
self.project_dropdown_open = false;
if let Some(ref db) = self.db {
if let (Some(outgoing), Some(data_dir)) = (&self.active_project, &self.data_dir)
{
let _ =
engine::embedding::EmbeddingService::production(db.conn(), data_dir)
.flush_project(&outgoing.id);
}
match engine::project::set_active_project(db.conn(), &project_id) {
Ok(()) => {
self.reset_git_for_project_change();
self.active_project =
self.projects.iter().find(|p| p.id == project_id).cloned();
self.preview_session = None;
self.duplicates_state = DuplicatesState::default();
self.hide_embedded_preview();
self.data_dir = self
.active_project
@@ -1626,7 +1662,11 @@ impl BdsApp {
}
}
self.sync_menu_state();
Task::batch([self.refresh_counts(), self.refresh_git_if_visible()])
Task::batch([
self.refresh_counts(),
self.refresh_git_if_visible(),
Task::done(Message::EmbeddingBackfill),
])
}
Message::ProjectSwitched(result) => {
match result {
@@ -2119,6 +2159,118 @@ impl BdsApp {
}
Task::none()
}
Message::DuplicatesRefresh => self.start_duplicate_search(0),
Message::DuplicatesShowMore => {
let next = self.duplicates_state.page.saturating_add(1);
self.start_duplicate_search(next)
}
Message::DuplicatesLoaded(result) => {
self.duplicates_state.is_loading = false;
self.duplicates_state.has_run = true;
match result {
Ok(result) => {
self.duplicates_state.result = result;
self.duplicates_state.error = None;
self.duplicates_state.selected.retain(|pair| {
self.duplicates_state.result.pairs.iter().any(|candidate| {
candidate.post_id_a == pair.0 && candidate.post_id_b == pair.1
})
});
}
Err(error) => self.duplicates_state.error = Some(error),
}
Task::none()
}
Message::DuplicatesToggle(a, b) => {
if !self
.duplicates_state
.selected
.remove(&(a.clone(), b.clone()))
{
self.duplicates_state.selected.insert((a, b));
}
Task::none()
}
Message::DuplicatesCheckAll => {
self.duplicates_state.selected = self
.duplicates_state
.result
.pairs
.iter()
.map(|pair| (pair.post_id_a.clone(), pair.post_id_b.clone()))
.collect();
Task::none()
}
Message::DuplicatesUncheckAll => {
self.duplicates_state.selected.clear();
Task::none()
}
Message::DuplicatesDismiss(a, b) => self.dismiss_duplicate_pairs(vec![(a, b)]),
Message::DuplicatesDismissSelected => self
.dismiss_duplicate_pairs(self.duplicates_state.selected.iter().cloned().collect()),
Message::DuplicatesDismissed(result) => {
match result {
Ok(()) => {
self.duplicates_state.selected.clear();
self.notify(
ToastLevel::Success,
&t(self.ui_locale, "duplicates.dismissed"),
);
return self.start_duplicate_search(self.duplicates_state.page);
}
Err(error) => self.notify(ToastLevel::Error, &error),
}
Task::none()
}
Message::DuplicatesOpenPost(post_id) => {
let Some(db) = &self.db else {
return Task::none();
};
let Ok(post) = bds_core::db::queries::post::get_post_by_id(db.conn(), &post_id)
else {
return Task::none();
};
Task::done(Message::OpenTab(Tab {
id: post.id,
tab_type: TabType::Post,
title: post.title,
is_transient: false,
is_dirty: false,
}))
}
Message::EmbeddingReindex => self.start_embedding_reindex(),
Message::EmbeddingBackfill => self.start_embedding_backfill(),
Message::LoadSemanticTagSuggestions(post_id) => {
let Some(data_dir) = self.data_dir.clone() else {
return Task::none();
};
let db_path = self.db_path.clone();
let returned_post_id = post_id.clone();
Task::perform(
async move {
tokio::task::spawn_blocking(move || {
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
engine::embedding::EmbeddingService::production(db.conn(), &data_dir)
.suggest_tags(&post_id)
.map_err(|error| error.to_string())
})
.await
.unwrap_or_else(|error| Err(format!("task panicked: {error}")))
},
move |result| Message::SemanticTagSuggestionsLoaded {
post_id: returned_post_id.clone(),
result,
},
)
}
Message::SemanticTagSuggestionsLoaded { post_id, result } => {
if let (Some(state), Ok(suggestions)) =
(self.post_editors.get_mut(&post_id), result)
{
state.semantic_tag_suggestions = suggestions;
}
Task::none()
}
message @ (Message::MainWindowLoaded(_) | Message::EmbeddedPreviewReady(_)) => {
self.handle_preview_message(message)
}
@@ -2129,6 +2281,7 @@ impl BdsApp {
self.refresh_task_snapshots();
self.process_chat_events();
self.persist_due_chat_surface_state();
let _ = engine::embedding::EmbeddingService::flush_due();
if !self.search_index_rebuild_running {
self.auto_save_due_post_editors();
}
@@ -2646,6 +2799,151 @@ impl BdsApp {
}
}
fn start_duplicate_search(&mut self, page: usize) -> Task<Message> {
let (Some(project), Some(data_dir)) = (&self.active_project, &self.data_dir) else {
return Task::none();
};
self.duplicates_state.enabled = engine::meta::read_project_json(data_dir)
.is_ok_and(|metadata| metadata.semantic_similarity_enabled);
self.duplicates_state.page = page;
self.duplicates_state.error = None;
if !self.duplicates_state.enabled {
self.duplicates_state.is_loading = false;
self.duplicates_state.has_run = false;
self.duplicates_state.result = Default::default();
return Task::none();
}
self.duplicates_state.is_loading = true;
let db_path = self.db_path.clone();
let data_dir = data_dir.clone();
let project_id = project.id.clone();
Task::perform(
async move {
tokio::task::spawn_blocking(move || {
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
engine::embedding::EmbeddingService::production(db.conn(), &data_dir)
.find_duplicates(&project_id, page)
.map_err(|error| error.to_string())
})
.await
.unwrap_or_else(|error| Err(format!("task panicked: {error}")))
},
Message::DuplicatesLoaded,
)
}
fn dismiss_duplicate_pairs(&mut self, pairs: Vec<(String, String)>) -> Task<Message> {
if pairs.is_empty() {
return Task::none();
}
let (Some(_), Some(data_dir)) = (&self.db, &self.data_dir) else {
return Task::none();
};
let db_path = self.db_path.clone();
let data_dir = data_dir.clone();
Task::perform(
async move {
tokio::task::spawn_blocking(move || {
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
engine::embedding::EmbeddingService::production(db.conn(), &data_dir)
.dismiss_duplicate_pairs(&pairs)
.map_err(|error| error.to_string())
})
.await
.unwrap_or_else(|error| Err(format!("task panicked: {error}")))
},
Message::DuplicatesDismissed,
)
}
fn start_embedding_reindex(&mut self) -> Task<Message> {
let Some(data_dir) = &self.data_dir else {
return Task::none();
};
if !engine::meta::read_project_json(data_dir)
.is_ok_and(|metadata| metadata.semantic_similarity_enabled)
{
self.notify(
ToastLevel::Warning,
&t(self.ui_locale, "duplicates.disabled"),
);
return Task::none();
}
let locale = self.ui_locale;
self.spawn_engine_task(
"menu.item.rebuildEmbeddingIndex",
move |db_path, project_id, data_dir, tm, tid| {
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let service = engine::embedding::EmbeddingService::production(db.conn(), &data_dir);
let indexed = service
.reindex_all_with_progress(&project_id, |current, total| {
tm.report_progress(
tid,
Some(current as f32 / total.max(1) as f32),
Some(tw(
locale,
"embeddings.indexingProgress",
&[
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
!tm.is_cancelled(tid)
})
.map_err(|error| error.to_string())?;
service
.flush_project(&project_id)
.map_err(|error| error.to_string())?;
Ok(tw(
locale,
"embeddings.reindexed",
&[("count", &indexed.len().to_string())],
))
},
)
}
fn start_embedding_backfill(&mut self) -> Task<Message> {
let Some(data_dir) = &self.data_dir else {
return Task::none();
};
if !engine::meta::read_project_json(data_dir)
.is_ok_and(|metadata| metadata.semantic_similarity_enabled)
{
return Task::none();
}
let locale = self.ui_locale;
self.spawn_engine_task(
"embeddings.indexing",
move |db_path, project_id, data_dir, tm, tid| {
let db = Database::open(&db_path).map_err(|error| error.to_string())?;
let indexed = engine::embedding::EmbeddingService::production(db.conn(), &data_dir)
.index_unindexed_with_progress(&project_id, |current, total| {
tm.report_progress(
tid,
Some(current as f32 / total.max(1) as f32),
Some(tw(
locale,
"embeddings.indexingProgress",
&[
("current", &current.to_string()),
("total", &total.to_string()),
],
)),
);
!tm.is_cancelled(tid)
})
.map_err(|error| error.to_string())?;
Ok(tw(
locale,
"embeddings.indexed",
&[("count", &indexed.len().to_string())],
))
},
)
}
pub fn view(&self) -> Element<'_, Message> {
let active_name = self.active_project.as_ref().map(|p| p.name.as_str());
let active_post_filter = match self.sidebar_view {
@@ -2706,6 +3004,7 @@ impl BdsApp {
self.settings_state.as_ref(),
self.dashboard_state.as_ref(),
&self.site_validation_state,
&self.duplicates_state,
&self.metadata_diff_state,
&self.translation_validation_state,
&self.git_state,
@@ -3397,6 +3696,11 @@ impl BdsApp {
}
MenuAction::RebuildDatabase => Task::done(Message::RebuildDatabase),
MenuAction::ReindexText => Task::done(Message::ReindexText),
MenuAction::RebuildEmbeddingIndex => Task::done(Message::EmbeddingReindex),
MenuAction::FindDuplicates => {
self.open_singleton_tab(TabType::FindDuplicates, "tabBar.findDuplicates");
Task::done(Message::DuplicatesRefresh)
}
MenuAction::MetadataDiff => Task::done(Message::RunMetadataDiff),
MenuAction::RegenerateCalendar => Task::done(Message::RegenerateCalendar),
MenuAction::ValidateTranslations => Task::done(Message::ValidateTranslations),
@@ -4777,7 +5081,7 @@ impl BdsApp {
..Default::default()
};
let ids = bds_core::db::fts::search_posts_filtered(
let mut ids = bds_core::db::fts::search_posts_filtered(
db.conn(),
query,
&self.content_language,
@@ -4786,6 +5090,19 @@ impl BdsApp {
.map(|results| results.post_ids)
.unwrap_or_default();
if let Some(data_dir) = &self.data_dir
&& let Ok(scores) = engine::embedding::EmbeddingService::production(db.conn(), data_dir)
.compute_similarities(current_post_id, &ids)
{
ids.sort_by(|a, b| {
scores
.get(b)
.copied()
.unwrap_or_default()
.total_cmp(&scores.get(a).copied().unwrap_or_default())
});
}
ids.into_iter()
.filter_map(|post_id| {
bds_core::db::queries::post::get_post_by_id(db.conn(), &post_id).ok()
@@ -6199,6 +6516,7 @@ impl BdsApp {
blog_languages: Vec::new(),
},
);
let semantic_was_enabled = meta.semantic_similarity_enabled;
meta.name = state.project_name.clone();
meta.description = {
let value = state.project_description.text();
@@ -6249,6 +6567,8 @@ impl BdsApp {
let file_result = engine::meta::write_project_json(data_dir, &meta);
match (db_result, file_result) {
(Ok(()), Ok(())) => {
let semantic_should_backfill =
state.semantic_similarity_enabled && !semantic_was_enabled;
if let Some(listing) =
self.projects.iter_mut().find(|p| p.id == project.id)
{
@@ -6258,6 +6578,9 @@ impl BdsApp {
self.blog_languages = state.blog_languages.clone();
self.dashboard_state = Some(self.hydrate_dashboard_state());
self.notify(ToastLevel::Success, &t(self.ui_locale, "editor.saved"));
if semantic_should_backfill {
return Task::done(Message::EmbeddingBackfill);
}
}
(Err(e), _) => self.notify_operation_failed("common.save", e),
(_, Err(e)) => self.notify_operation_failed("common.save", e),
@@ -7927,6 +8250,12 @@ impl BdsApp {
}
}
impl Drop for BdsApp {
fn drop(&mut self) {
let _ = engine::embedding::EmbeddingService::flush_all();
}
}
fn content_sample(content: &str, max_len: usize) -> String {
content.chars().take(max_len).collect()
}
@@ -9184,7 +9513,10 @@ mod tests {
let mut app = make_app(db, project, &tmp);
let _ = app.refresh_counts();
let dash = app.dashboard_state.expect("dashboard state should be set");
let dash = app
.dashboard_state
.clone()
.expect("dashboard state should be set");
let now = chrono::Utc::now();
assert_eq!(dash.stats.total_posts, 2);
assert_eq!(dash.stats.published_count, 1);
@@ -9552,10 +9884,10 @@ mod tests {
db_path.as_path(),
&project.id,
"en",
Some(tmp.path()),
&filter,
false,
50,
0,
(50, 0),
);
assert_eq!(posts.len(), 1);

13
crates/bds-ui/src/app/editor_handlers.rs
View File

@@ -148,6 +148,19 @@ impl BdsApp {
}
state.tags_input.clear();
}
PostEditorMsg::AddSuggestedTag(tag) => {
if !state
.tags
.iter()
.any(|current| current.eq_ignore_ascii_case(&tag))
{
state.tags.push(tag.clone());
state.mark_dirty();
}
state
.semantic_tag_suggestions
.retain(|candidate| !candidate.eq_ignore_ascii_case(&tag));
}
PostEditorMsg::RemoveTag(tag) => {
state.tags.retain(|t| t != &tag);
state.mark_dirty();

90
crates/bds-ui/src/app/search.rs
View File

@@ -1,4 +1,5 @@
use super::*;
use chrono::Datelike;
impl BdsApp {
/// Number of items to load per sidebar page.
@@ -14,6 +15,7 @@ impl BdsApp {
let db_path = self.db_path.clone();
let project_id = project.id.clone();
let content_language = self.content_language.clone();
let data_dir = self.data_dir.clone();
let filter = match self.sidebar_view {
SidebarView::Pages => self.page_filter.clone(),
_ => self.post_filter.clone(),
@@ -27,10 +29,10 @@ impl BdsApp {
&db_path,
&project_id,
&content_language,
data_dir.as_deref(),
&filter,
is_pages,
Self::SIDEBAR_PAGE_SIZE + 1,
0,
(Self::SIDEBAR_PAGE_SIZE + 1, 0),
)
})
.await
@@ -99,6 +101,7 @@ impl BdsApp {
let db_path = self.db_path.clone();
let project_id = project.id.clone();
let content_language = self.content_language.clone();
let data_dir = self.data_dir.clone();
let offset = self.sidebar_posts.len() as i64;
let filter = match self.sidebar_view {
SidebarView::Pages => self.page_filter.clone(),
@@ -113,10 +116,10 @@ impl BdsApp {
&db_path,
&project_id,
&content_language,
data_dir.as_deref(),
&filter,
is_pages,
Self::SIDEBAR_PAGE_SIZE + 1,
offset,
(Self::SIDEBAR_PAGE_SIZE + 1, offset),
)
})
.await
@@ -163,15 +166,16 @@ impl BdsApp {
db_path: &Path,
project_id: &str,
content_language: &str,
data_dir: Option<&Path>,
filter: &PostFilter,
is_pages: bool,
limit: i64,
offset: i64,
pagination: (i64, i64),
) -> Vec<Post> {
let Ok(db) = Database::open(db_path) else {
return Vec::new();
};
let (limit, offset) = pagination;
let params = Self::build_post_filter_params(filter, is_pages);
if filter.search_query.trim().is_empty() {
return bds_core::db::queries::post::list_posts_filtered(
@@ -184,6 +188,11 @@ impl BdsApp {
.unwrap_or_default();
}
let semantic_enabled = data_dir.is_some_and(|dir| {
engine::meta::read_project_json(dir)
.is_ok_and(|metadata| metadata.semantic_similarity_enabled)
});
let requested = limit.saturating_add(offset).max(0) as usize;
let fts_filters = bds_core::db::fts::PostSearchFilters {
status: params.status.as_deref(),
tags: (!params.tags.is_empty()).then_some(params.tags.as_slice()),
@@ -193,12 +202,16 @@ impl BdsApp {
month: params.month,
from: params.from,
to: params.to,
limit: Some(limit as usize),
offset: Some(offset as usize),
limit: Some(if semantic_enabled {
requested
} else {
limit as usize
}),
offset: Some(if semantic_enabled { 0 } else { offset as usize }),
..Default::default()
};
let ids = bds_core::db::fts::search_posts_filtered(
let fts_ids = bds_core::db::fts::search_posts_filtered(
db.conn(),
&params.search_query,
content_language,
@@ -207,11 +220,60 @@ impl BdsApp {
.map(|results| results.post_ids)
.unwrap_or_default();
ids.into_iter()
let mut ids = Vec::new();
if semantic_enabled
&& let Some(data_dir) = data_dir
&& let Ok(similar) =
engine::embedding::EmbeddingService::production(db.conn(), data_dir)
.semantic_search(project_id, &params.search_query, requested)
{
ids.extend(similar.into_iter().map(|item| item.post_id));
}
ids.extend(fts_ids);
let mut seen = HashSet::new();
let posts = ids
.into_iter()
.filter(|post_id| seen.insert(post_id.clone()))
.filter_map(|post_id| {
bds_core::db::queries::post::get_post_by_id(db.conn(), &post_id).ok()
})
.filter(|post| post.project_id == project_id)
.filter(|post| {
params
.status
.as_ref()
.is_none_or(|status| post.status.as_str() == status)
})
.filter(|post| {
params
.language
.as_ref()
.is_none_or(|language| post.language.as_deref() == Some(language))
})
.filter(|post| {
params
.tags
.iter()
.all(|wanted| post.tags.iter().any(|tag| tag.eq_ignore_ascii_case(wanted)))
})
.filter(|post| {
params.categories.iter().all(|wanted| {
post.categories
.iter()
.any(|category| category.eq_ignore_ascii_case(wanted))
})
})
.filter(|post| params.from.is_none_or(|from| post.created_at >= from))
.filter(|post| params.to.is_none_or(|to| post.created_at <= to))
.filter(|post| {
if params.year.is_none() && params.month.is_none() {
return true;
}
chrono::DateTime::from_timestamp_millis(post.created_at).is_some_and(|date| {
params.year.is_none_or(|year| date.year() == year)
&& params.month.is_none_or(|month| date.month() == month)
})
})
.filter(|post| {
let is_page_post = post
.categories
@@ -222,8 +284,12 @@ impl BdsApp {
} else {
!is_page_post
}
})
.collect()
});
if semantic_enabled {
posts.skip(offset as usize).take(limit as usize).collect()
} else {
posts.collect()
}
}
pub(super) fn query_sidebar_media_blocking(

19
crates/bds-ui/src/platform/menu.rs
View File

@@ -35,6 +35,8 @@ pub enum MenuAction {
EditMenu,
RebuildDatabase,
ReindexText,
RebuildEmbeddingIndex,
FindDuplicates,
MetadataDiff,
RegenerateCalendar,
ValidateTranslations,
@@ -69,6 +71,8 @@ impl MenuAction {
MenuAction::EditMenu,
MenuAction::RebuildDatabase,
MenuAction::ReindexText,
MenuAction::RebuildEmbeddingIndex,
MenuAction::FindDuplicates,
MenuAction::MetadataDiff,
MenuAction::RegenerateCalendar,
MenuAction::ValidateTranslations,
@@ -101,6 +105,8 @@ impl MenuAction {
"edit_menu" => Self::EditMenu,
"rebuild_database" => Self::RebuildDatabase,
"reindex_text" => Self::ReindexText,
"rebuild_embedding_index" => Self::RebuildEmbeddingIndex,
"find_duplicates" => Self::FindDuplicates,
"metadata_diff" => Self::MetadataDiff,
"regenerate_calendar" => Self::RegenerateCalendar,
"validate_translations" => Self::ValidateTranslations,
@@ -136,6 +142,8 @@ impl MenuAction {
Self::EditMenu => "menu.item.editMenu",
Self::RebuildDatabase => "menu.item.rebuildDatabase",
Self::ReindexText => "menu.item.reindexText",
Self::RebuildEmbeddingIndex => "menu.item.rebuildEmbeddingIndex",
Self::FindDuplicates => "menu.item.findDuplicates",
Self::MetadataDiff => "menu.item.metadataDiff",
Self::RegenerateCalendar => "menu.item.regenerateCalendar",
Self::ValidateTranslations => "menu.item.validateTranslations",
@@ -173,6 +181,8 @@ pub(crate) fn action_enabled(
| MenuAction::EditMenu
| MenuAction::RebuildDatabase
| MenuAction::ReindexText
| MenuAction::RebuildEmbeddingIndex
| MenuAction::FindDuplicates
| MenuAction::MetadataDiff
| MenuAction::RegenerateCalendar
| MenuAction::ValidateTranslations
@@ -205,6 +215,8 @@ pub(crate) fn action_enabled(
| MenuAction::EditMenu
| MenuAction::RebuildDatabase
| MenuAction::ReindexText
| MenuAction::RebuildEmbeddingIndex
| MenuAction::FindDuplicates
| MenuAction::MetadataDiff
| MenuAction::RegenerateCalendar
| MenuAction::ValidateTranslations
@@ -413,6 +425,13 @@ pub fn build_menu_bar(locale: UiLocale) -> (Menu, MenuRegistry) {
let _ = blog_menu.append(&PredefinedMenuItem::separator());
let _ = blog_menu.append(&item(&mut reg, MenuAction::RebuildDatabase, locale, None));
let _ = blog_menu.append(&item(&mut reg, MenuAction::ReindexText, locale, None));
let _ = blog_menu.append(&item(
&mut reg,
MenuAction::RebuildEmbeddingIndex,
locale,
None,
));
let _ = blog_menu.append(&item(&mut reg, MenuAction::FindDuplicates, locale, None));
let _ = blog_menu.append(&item(&mut reg, MenuAction::MetadataDiff, locale, None));
let _ = blog_menu.append(&PredefinedMenuItem::separator());
let _ = blog_menu.append(&item(

182
crates/bds-ui/src/views/duplicates.rs Normal file
View File

@@ -0,0 +1,182 @@
use std::collections::HashSet;
use bds_core::engine::embedding::DuplicateSearchResult;
use bds_core::i18n::UiLocale;
use iced::widget::text::Shaping;
use iced::widget::{Space, button, checkbox, column, container, row, scrollable, text};
use iced::{Color, Element, Length};
use crate::app::Message;
use crate::components::inputs;
use crate::i18n::{t, tw};
#[derive(Debug, Clone, Default)]
pub struct DuplicatesState {
pub enabled: bool,
pub is_loading: bool,
pub has_run: bool,
pub page: usize,
pub result: DuplicateSearchResult,
pub selected: HashSet<(String, String)>,
pub error: Option<String>,
}
pub fn view(state: &DuplicatesState, locale: UiLocale) -> Element<'_, Message> {
let refresh = if state.is_loading {
button(text(t(locale, "duplicates.searching")).size(13)).style(inputs::secondary_button)
} else {
button(text(t(locale, "common.refresh")).size(13))
.on_press(Message::DuplicatesRefresh)
.style(inputs::secondary_button)
}
.padding([6, 16]);
let dismiss_checked = if state.selected.is_empty() || state.is_loading {
button(text(t(locale, "duplicates.dismissChecked")).size(13))
.style(inputs::secondary_button)
} else {
button(
text(tw(
locale,
"duplicates.dismissCheckedCount",
&[("count", &state.selected.len().to_string())],
))
.size(13),
)
.on_press(Message::DuplicatesDismissSelected)
.style(inputs::primary_button)
}
.padding([6, 16]);
let toolbar = inputs::toolbar(
vec![
text(t(locale, "duplicates.title"))
.size(20)
.shaping(Shaping::Advanced)
.into(),
text(tw(
locale,
"duplicates.count",
&[("count", &state.result.pairs.len().to_string())],
))
.size(12)
.color(inputs::LABEL_COLOR)
.into(),
],
vec![
button(text(t(locale, "duplicates.checkAll")).size(13))
.on_press(Message::DuplicatesCheckAll)
.padding([6, 12])
.style(inputs::secondary_button)
.into(),
button(text(t(locale, "duplicates.uncheckAll")).size(13))
.on_press(Message::DuplicatesUncheckAll)
.padding([6, 12])
.style(inputs::secondary_button)
.into(),
dismiss_checked.into(),
refresh.into(),
],
);
let body: Element<'_, Message> = if !state.enabled {
inputs::card(
text(t(locale, "duplicates.disabled"))
.size(14)
.color(inputs::LABEL_COLOR),
)
.into()
} else if state.is_loading && !state.has_run {
inputs::card(
text(t(locale, "duplicates.searching"))
.size(14)
.color(inputs::LABEL_COLOR),
)
.into()
} else if let Some(error) = &state.error {
inputs::card(
text(error.clone())
.size(14)
.color(Color::from_rgb(0.90, 0.38, 0.38)),
)
.into()
} else if state.has_run && state.result.pairs.is_empty() {
inputs::card(
text(t(locale, "duplicates.empty"))
.size(14)
.color(inputs::LABEL_COLOR),
)
.into()
} else {
let mut pairs = column!().spacing(8);
for pair in &state.result.pairs {
let key = (pair.post_id_a.clone(), pair.post_id_b.clone());
let checked = state.selected.contains(&key);
let badge = if pair.exact_match {
t(locale, "duplicates.exactMatch")
} else {
format!("{:.1}%", pair.similarity * 100.0)
};
pairs = pairs.push(inputs::card(
row![
checkbox("", checked)
.on_toggle({
let a = pair.post_id_a.clone();
let b = pair.post_id_b.clone();
move |_| Message::DuplicatesToggle(a.clone(), b.clone())
})
.size(16),
button(
text(pair.title_a.clone())
.size(13)
.shaping(Shaping::Advanced)
)
.on_press(Message::DuplicatesOpenPost(pair.post_id_a.clone()))
.padding([5, 8])
.style(inputs::disclosure_button),
text("→").size(14).color(inputs::LABEL_COLOR),
button(
text(pair.title_b.clone())
.size(13)
.shaping(Shaping::Advanced)
)
.on_press(Message::DuplicatesOpenPost(pair.post_id_b.clone()))
.padding([5, 8])
.style(inputs::disclosure_button),
Space::with_width(Length::Fill),
text(badge).size(12).color(if pair.exact_match {
Color::from_rgb(0.96, 0.68, 0.28)
} else {
Color::from_rgb(0.55, 0.76, 0.92)
}),
button(text(t(locale, "duplicates.dismiss")).size(12))
.on_press(Message::DuplicatesDismiss(
pair.post_id_a.clone(),
pair.post_id_b.clone()
))
.padding([5, 12])
.style(inputs::secondary_button),
]
.spacing(8)
.align_y(iced::Alignment::Center),
));
}
if state.result.has_more {
pairs = pairs.push(
button(text(t(locale, "duplicates.showMore")).size(13))
.on_press(Message::DuplicatesShowMore)
.padding([7, 16])
.style(inputs::secondary_button),
);
}
scrollable(container(pairs).padding(2))
.height(Length::Fill)
.into()
};
container(column![toolbar, body].spacing(12))
.padding(16)
.width(Length::Fill)
.height(Length::Fill)
.into()
}

1
crates/bds-ui/src/views/mod.rs
View File

@@ -2,6 +2,7 @@ pub mod activity_bar;
pub mod chat_surfaces;
pub mod chat_view;
pub mod dashboard;
pub mod duplicates;
pub mod git;
pub mod import_editor;
pub mod media_editor;

27
crates/bds-ui/src/views/post_editor.rs
View File

@@ -76,6 +76,7 @@ pub struct PostEditorState {
pub quick_actions_open: bool,
pub tags_input: String,
pub categories_input: String,
pub semantic_tag_suggestions: Vec<String>,
pub active_language: String,
pub canonical_language: String,
pub blog_languages: Vec<String>,
@@ -123,6 +124,7 @@ impl Clone for PostEditorState {
quick_actions_open: self.quick_actions_open,
tags_input: self.tags_input.clone(),
categories_input: self.categories_input.clone(),
semantic_tag_suggestions: self.semantic_tag_suggestions.clone(),
active_language: self.active_language.clone(),
canonical_language: self.canonical_language.clone(),
blog_languages: self.blog_languages.clone(),
@@ -188,6 +190,7 @@ impl PostEditorState {
quick_actions_open: false,
tags_input: String::new(),
categories_input: String::new(),
semantic_tag_suggestions: Vec::new(),
active_language: canonical_lang.clone(),
canonical_language: canonical_lang,
blog_languages: blog_languages.to_vec(),
@@ -342,6 +345,7 @@ pub enum PostEditorMsg {
SwitchLanguage(String),
TagsInputChanged(String),
TagsInputSubmit,
AddSuggestedTag(String),
RemoveTag(String),
CategoriesInputChanged(String),
CategoriesInputSubmit,
@@ -619,6 +623,28 @@ pub fn view<'a>(
Message::PostEditor(PostEditorMsg::TagsInputSubmit),
|tag| Message::PostEditor(PostEditorMsg::RemoveTag(tag)),
);
let semantic_tags: Element<'a, Message> = if state.semantic_tag_suggestions.is_empty() {
Space::new(0, 0).into()
} else {
let mut chips = row![
text(t(locale, "editor.semanticTagSuggestions"))
.size(11)
.color(inputs::LABEL_COLOR)
]
.spacing(6)
.align_y(iced::Alignment::Center);
for tag in &state.semantic_tag_suggestions {
chips = chips.push(
button(text(format!("+ {tag}")).size(11))
.on_press(Message::PostEditor(PostEditorMsg::AddSuggestedTag(
tag.clone(),
)))
.padding([4, 8])
.style(inputs::secondary_button),
);
}
chips.into()
};
// Categories chip input
let categories_section = chip_input_field(
@@ -776,6 +802,7 @@ pub fn view<'a>(
meta_row1,
meta_row2,
tags_section,
semantic_tags,
categories_section,
outlinks_section,
backlinks_section,

40
crates/bds-ui/src/views/workspace.rs
View File

@@ -18,6 +18,7 @@ use crate::views::{
activity_bar,
chat_view::{self, ChatEditorState},
dashboard::DashboardState,
duplicates::{self, DuplicatesState},
git::{self, GitDiffState, GitUiState},
media_editor::{self, MediaEditorState},
metadata_diff::{self, MetadataDiffState},
@@ -128,6 +129,7 @@ pub fn view<'a>(
settings_state: Option<&'a SettingsViewState>,
dashboard_state: Option<&'a DashboardState>,
site_validation_state: &'a SiteValidationState,
duplicates_state: &'a DuplicatesState,
metadata_diff_state: &'a MetadataDiffState,
translation_validation_state: &'a TranslationValidationState,
git_state: &'a GitUiState,
@@ -158,6 +160,7 @@ pub fn view<'a>(
settings_state,
dashboard_state,
site_validation_state,
duplicates_state,
metadata_diff_state,
translation_validation_state,
git_diffs,
@@ -400,6 +403,7 @@ fn route_content_area<'a>(
settings_state: Option<&'a SettingsViewState>,
dashboard_state: Option<&'a DashboardState>,
site_validation_state: &'a SiteValidationState,
duplicates_state: &'a DuplicatesState,
metadata_diff_state: &'a MetadataDiffState,
translation_validation_state: &'a TranslationValidationState,
git_diffs: &'a HashMap<String, GitDiffState>,
@@ -489,6 +493,7 @@ fn route_content_area<'a>(
}
}
ContentRoute::SiteValidation => site_validation::view(site_validation_state, locale),
ContentRoute::FindDuplicates => duplicates::view(duplicates_state, locale),
ContentRoute::MetadataDiff => metadata_diff::view(metadata_diff_state, locale),
ContentRoute::TranslationValidation => {
translation_validation::view(translation_validation_state, locale)
@@ -524,6 +529,7 @@ enum ContentRoute<'a> {
Tags,
Settings,
SiteValidation,
FindDuplicates,
MetadataDiff,
TranslationValidation,
GitDiff(&'a str),
@@ -612,11 +618,11 @@ fn route_kind<'a>(
TabType::SiteValidation => ContentRoute::SiteValidation,
TabType::MetadataDiff => ContentRoute::MetadataDiff,
TabType::GitDiff => ContentRoute::GitDiff(tab_id),
TabType::FindDuplicates => ContentRoute::FindDuplicates,
TabType::Style
| TabType::MenuEditor
| TabType::Documentation
| TabType::ApiDocumentation
| TabType::FindDuplicates => ContentRoute::Placeholder(&tab.title),
| TabType::ApiDocumentation => ContentRoute::Placeholder(&tab.title),
TabType::TranslationValidation => ContentRoute::TranslationValidation,
}
}
@@ -678,7 +684,6 @@ mod tests {
TabType::MenuEditor,
TabType::Documentation,
TabType::ApiDocumentation,
TabType::FindDuplicates,
];
for tab_type in unsupported {
@@ -703,6 +708,35 @@ mod tests {
}
}
#[test]
fn duplicate_tab_routes_to_real_review_surface() {
let tabs = vec![tab(
"find_duplicates",
TabType::FindDuplicates,
"Duplicates",
)];
let empty_posts = HashMap::new();
let empty_media = HashMap::new();
let empty_templates = HashMap::new();
let empty_scripts = HashMap::new();
let empty_imports = HashMap::new();
let site_validation = SiteValidationState::default();
let route = route_kind(
&tabs,
Some("find_duplicates"),
&empty_posts,
&empty_media,
&empty_templates,
&empty_scripts,
&empty_imports,
None,
None,
None,
&site_validation,
);
assert!(matches!(route, ContentRoute::FindDuplicates));
}
#[test]
fn git_diff_tab_routes_to_real_view() {
let empty_posts = HashMap::new();