hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement semantic embeddings and duplicate review

Browse Source
This commit is contained in:
Chili Palmer
2026-07-19 18:06:47 +02:00
parent fd4fd744e0
commit e8764262a3
42 changed files with 3808 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1382
crates/bds-core/src/engine/embedding.rs Normal file
View File

File diff suppressed because it is too large Load Diff

51
crates/bds-core/src/engine/metadata_diff.rs
View File

@@ -5,6 +5,7 @@ use std::path::Path;
use crate::db::DbConnection as Connection;
use walkdir::WalkDir;
use crate::db::queries::embedding as qe;
use crate::db::queries::media as qm;
use crate::db::queries::media_translation as qmt;
use crate::db::queries::post as qp;
@@ -159,6 +160,49 @@ pub fn compute_metadata_diff(
}
// 6. Detect orphans
if crate::engine::meta::read_project_json(data_dir)
.is_ok_and(|metadata| metadata.semantic_similarity_enabled)
{
let service = crate::engine::embedding::EmbeddingService::production(conn, data_dir);
let keys = qe::list_keys(conn, project_id)?
.into_iter()
.map(|key| (key.post_id.clone(), key))
.collect::<std::collections::HashMap<_, _>>();
for post in &posts {
let expected = service.content_hash_for_post(post)?;
let key = keys.get(&post.id);
let current_hash = key.map(|key| key.content_hash.as_str()).unwrap_or("");
let vector_status = key
.filter(|key| crate::engine::embedding::decode_vector(&key.vector).is_ok())
.map(|_| "ready")
.unwrap_or("missing");
if current_hash != expected || vector_status != "ready" {
let mut fields = Vec::new();
if current_hash != expected {
fields.push(DiffField {
field_name: "content_hash".into(),
db_value: current_hash.into(),
file_value: expected,
});
}
if vector_status != "ready" {
fields.push(DiffField {
field_name: "embedding".into(),
db_value: vector_status.into(),
file_value: "ready".into(),
});
}
report.diffs.push(EntityDiff {
entity_type: "embedding".into(),
entity_id: post.id.clone(),
file_path: format!("projects/{project_id}/embeddings.usearch"),
fields,
});
}
}
}
// 7. Detect orphans
let orphans = detect_orphan_files(conn, data_dir, project_id)?;
report.orphans = orphans;
@@ -204,6 +248,11 @@ pub fn repair_metadata_diff_item(
conn, data_dir, project_id, &path,
)?;
}
"embedding" => {
let post = qp::get_post_by_id(conn, &item.entity_id)?;
crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.sync_post(&post)?;
}
other => return unsupported_repair(other),
}
}
@@ -222,6 +271,8 @@ pub fn repair_metadata_diff_item(
)?,
"script" => rewrite_script_from_database(conn, data_dir, &item.entity_id)?,
"template" => rewrite_template_from_database(conn, data_dir, &item.entity_id)?,
"embedding" => crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.flush_project(project_id)?,
other => return unsupported_repair(other),
},
}

1
crates/bds-core/src/engine/mod.rs
View File

@@ -8,6 +8,7 @@ mod chat_tools;
pub mod cli_launcher;
pub mod cli_sync;
pub mod domain_events;
pub mod embedding;
pub mod error;
pub mod gallery_import;
pub mod generation;

14
crates/bds-core/src/engine/post.rs
View File

@@ -36,7 +36,7 @@ pub struct RebuildReport {
)]
pub fn create_post(
conn: &Connection,
_data_dir: &Path,
data_dir: &Path,
project_id: &str,
title: &str,
content: Option<&str>,
@@ -91,6 +91,7 @@ pub fn create_post(
fts_index_post(conn, &post)?;
emit_post(&post, NotificationAction::Created);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
@@ -102,7 +103,7 @@ pub fn create_post(
)]
pub fn update_post(
conn: &Connection,
_data_dir: &Path,
data_dir: &Path,
post_id: &str,
title: Option<&str>,
slug: Option<&str>,
@@ -169,7 +170,7 @@ pub fn update_post(
if post.status == PostStatus::Published || post.status == PostStatus::Archived {
// Reload content from filesystem if content field is NULL (published state)
if post.content.is_none() && !post.file_path.is_empty() {
let abs_path = _data_dir.join(&post.file_path);
let abs_path = data_dir.join(&post.file_path);
if abs_path.exists()
&& let Ok(file_content) = fs::read_to_string(&abs_path)
&& let Ok((_fm, body)) = read_post_file(&file_content)
@@ -187,6 +188,7 @@ pub fn update_post(
fts_index_post(conn, &post)?;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
@@ -210,6 +212,7 @@ pub fn publish_post(conn: &Connection, data_dir: &Path, post_id: &str) -> Engine
Ok(post) => {
conn.release_savepoint()?;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
Err(error) => {
@@ -343,7 +346,9 @@ pub fn archive_post(conn: &Connection, data_dir: &Path, post_id: &str) -> Engine
}
let now = now_unix_ms();
qp::update_post_status(conn, post_id, &PostStatus::Archived, now)?;
post.status = PostStatus::Archived;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(())
}
@@ -455,6 +460,7 @@ pub fn discard_post_draft(conn: &Connection, data_dir: &Path, post_id: &str) ->
Ok(post) => {
conn.release_savepoint()?;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
Ok(post)
}
Err(error) => {
@@ -503,6 +509,8 @@ pub fn delete_post(conn: &Connection, data_dir: &Path, post_id: &str) -> EngineR
// Delete post from DB
qp::delete_post(conn, post_id)?;
crate::engine::embedding::remove_post_best_effort(conn, data_dir, &post.project_id, post_id);
emit_post(&post, NotificationAction::Deleted);
Ok(())

1
crates/bds-core/src/engine/project.rs
View File

@@ -238,6 +238,7 @@ pub fn delete_project(
let is_custom_path = project.data_path.is_some();
q::delete_project(conn, project_id)?;
crate::engine::embedding::EmbeddingService::forget_project(project_id);
// Clean up internal filesystem only (not custom external paths per spec)
if !is_custom_path

16
crates/bds-core/src/engine/rebuild.rs
View File

@@ -170,15 +170,18 @@ fn rebuild_from_filesystem_inner(
)));
}
progress(0.98, "Refreshing semantic index...");
crate::engine::embedding::EmbeddingService::production(conn, data_dir)
.index_unindexed(project_id)?;
progress(1.0, "Rebuild complete");
Ok(report)
}
fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
use crate::db::schema::{
dismissed_duplicate_pairs, embedding_keys, generated_file_hashes, import_definitions,
media, media_translations, post_links, post_media, post_translations, posts, scripts, tags,
templates,
generated_file_hashes, import_definitions, media, media_translations, post_links,
post_media, post_translations, posts, scripts, tags, templates,
};
let post_ids = crate::db::queries::post::list_posts_by_project(conn, project_id)?
@@ -219,13 +222,6 @@ fn clear_project_rows(conn: &Connection, project_id: &str) -> EngineResult<()> {
generated_file_hashes::table.filter(generated_file_hashes::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(embedding_keys::table.filter(embedding_keys::project_id.eq(project_id)))
.execute(connection)?;
diesel::delete(
dismissed_duplicate_pairs::table
.filter(dismissed_duplicate_pairs::project_id.eq(project_id)),
)
.execute(connection)?;
diesel::delete(
import_definitions::table.filter(import_definitions::project_id.eq(project_id)),
)