From d50e538fc3c8cec2a33fcba2bda5f78d6ff9cbba Mon Sep 17 00:00:00 2001 From: Georg Bauer Date: Tue, 21 Jul 2026 11:21:03 +0200 Subject: [PATCH] Index published post bodies from files --- crates/bds-core/src/engine/post.rs | 127 +++++++++++++++++++++++---- crates/bds-core/src/engine/search.rs | 112 +++++++++++++++++++++-- 2 files changed, 213 insertions(+), 26 deletions(-) diff --git a/crates/bds-core/src/engine/post.rs b/crates/bds-core/src/engine/post.rs index c52a70c..eb2c1e8 100644 --- a/crates/bds-core/src/engine/post.rs +++ b/crates/bds-core/src/engine/post.rs @@ -88,7 +88,7 @@ pub fn create_post( qp::insert_post(conn, &post)?; // Index for FTS - fts_index_post(conn, &post)?; + fts_index_post(conn, data_dir, &post)?; emit_post(&post, NotificationAction::Created); crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post); @@ -185,7 +185,7 @@ pub fn update_post( qp::update_post(conn, &post)?; // Re-index FTS - fts_index_post(conn, &post)?; + fts_index_post(conn, data_dir, &post)?; emit_post(&post, NotificationAction::Updated); crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post); @@ -319,7 +319,7 @@ fn publish_post_in_savepoint( } // Re-index FTS - fts_index_post(conn, &post)?; + fts_index_post(conn, data_dir, &post)?; Ok(post) } @@ -454,7 +454,7 @@ pub fn discard_post_draft(conn: &Connection, data_dir: &Path, post_id: &str) -> conn.begin_savepoint()?; match (|| { qp::update_post(conn, &post)?; - fts_index_post(conn, &post)?; + fts_index_post(conn, data_dir, &post)?; Ok(post) })() { Ok(post) => { @@ -571,7 +571,7 @@ pub fn upsert_translation( qt::update_post_translation(conn, &t)?; // Re-index FTS for parent post - fts_index_post(conn, &post)?; + fts_index_post(conn, data_dir, &post)?; Ok(t) } @@ -596,7 +596,7 @@ pub fn upsert_translation( qt::insert_post_translation(conn, &t)?; // Re-index FTS for parent post - fts_index_post(conn, &post)?; + fts_index_post(conn, data_dir, &post)?; Ok(t) } @@ -623,7 +623,7 @@ pub fn delete_translation( // Re-index FTS for parent post if let Ok(post) = qp::get_post_by_id(conn, &t.translation_for) { - fts_index_post(conn, &post)?; + fts_index_post(conn, data_dir, &post)?; } Ok(()) @@ -643,6 +643,7 @@ pub fn publish_post_translation( conn.begin_savepoint()?; match publish_translation(conn, data_dir, &mut translation, &post) { Ok(()) => { + fts_index_post(conn, data_dir, &post)?; conn.release_savepoint()?; Ok(translation) } @@ -747,7 +748,7 @@ pub fn rebuild_posts_from_filesystem_with_progress( // Re-index FTS for all posts in this project let posts = qp::list_posts_by_project(conn, project_id)?; for post in &posts { - fts_index_post(conn, post)?; + fts_index_post(conn, data_dir, post)?; } Ok(report) @@ -895,25 +896,32 @@ fn publish_translation( } /// Index a post in FTS, gathering translation texts. -fn fts_index_post(conn: &Connection, post: &Post) -> EngineResult<()> { +fn fts_index_post(conn: &Connection, data_dir: &Path, post: &Post) -> EngineResult<()> { let translations = qt::list_post_translations_by_post(conn, &post.id).unwrap_or_default(); let translation_data: Vec = translations .iter() - .map(|t| fts::PostTranslationFts { - title: t.title.clone(), - excerpt: t.excerpt.clone(), - content: t.content.clone(), - language: t.language.clone(), + .map(|t| { + Ok(fts::PostTranslationFts { + title: t.title.clone(), + excerpt: t.excerpt.clone(), + content: resolve_translation_fts_content(data_dir, t)?, + language: t.language.clone(), + }) }) - .collect(); + .collect::>()?; - let lang = post.language.as_deref().unwrap_or("en"); + let main_language = crate::engine::meta::read_project_json(data_dir) + .ok() + .and_then(|metadata| metadata.main_language) + .unwrap_or_else(|| "en".to_string()); + let content = resolve_post_fts_content(data_dir, post)?; + let lang = post.language.as_deref().unwrap_or(&main_language); fts::index_post( conn, &post.id, &post.title, post.excerpt.as_deref(), - post.content.as_deref(), + content.as_deref(), &post.tags, &post.categories, &translation_data, @@ -922,6 +930,33 @@ fn fts_index_post(conn: &Connection, post: &Post) -> EngineResult<()> { Ok(()) } +fn resolve_post_fts_content(data_dir: &Path, post: &Post) -> EngineResult> { + if post.content.is_some() { + return Ok(post.content.clone()); + } + if post.file_path.is_empty() { + return Ok(None); + } + let raw = fs::read_to_string(data_dir.join(&post.file_path))?; + let (_fm, body) = read_post_file(&raw).map_err(EngineError::Parse)?; + Ok(Some(body)) +} + +fn resolve_translation_fts_content( + data_dir: &Path, + translation: &PostTranslation, +) -> EngineResult> { + if translation.content.is_some() { + return Ok(translation.content.clone()); + } + if translation.file_path.is_empty() { + return Ok(None); + } + let raw = fs::read_to_string(data_dir.join(&translation.file_path))?; + let (_fm, body) = read_translation_file(&raw).map_err(EngineError::Parse)?; + Ok(Some(body)) +} + /// Check if a file stem looks like a translation filename: `{slug}.{lang}` /// where lang is a 2-letter code. We look for a dot followed by exactly 2 lowercase letters. pub(crate) fn is_translation_filename(stem: &str) -> bool { @@ -1190,6 +1225,64 @@ mod tests { ); } + #[test] + fn publish_indexes_post_body_from_file_when_db_content_is_cleared() { + let (db, dir) = setup(); + let post = create_post( + db.conn(), + dir.path(), + "p1", + "Published Search", + Some("distinctive quokkafire body"), + vec![], + vec![], + None, + Some("en"), + None, + ) + .unwrap(); + publish_post(db.conn(), dir.path(), &post.id).unwrap(); + let stored = qp::get_post_by_id(db.conn(), &post.id).unwrap(); + assert!(stored.content.is_none()); + assert_eq!( + crate::db::fts::search_posts(db.conn(), "quokkafire", "en").unwrap(), + vec![post.id] + ); + } + + #[test] + fn publish_indexes_translation_body_from_file_when_db_content_is_cleared() { + let (db, dir) = setup(); + let post = create_post( + db.conn(), + dir.path(), + "p1", + "Translated Search", + Some("canonical body"), + vec![], + vec![], + None, + Some("en"), + None, + ) + .unwrap(); + upsert_translation( + db.conn(), + dir.path(), + &post.id, + "de", + "Übersetzte Suche", + None, + Some("markantes drachenfalter wort"), + ) + .unwrap(); + publish_post(db.conn(), dir.path(), &post.id).unwrap(); + assert_eq!( + crate::db::fts::search_posts(db.conn(), "drachenfalter", "de").unwrap(), + vec![post.id] + ); + } + #[test] fn create_post_empty_title_uses_untitled() { let (db, dir) = setup(); diff --git a/crates/bds-core/src/engine/search.rs b/crates/bds-core/src/engine/search.rs index 2225e15..5e0623e 100644 --- a/crates/bds-core/src/engine/search.rs +++ b/crates/bds-core/src/engine/search.rs @@ -1,5 +1,6 @@ //! Full-text search reindexing engine functions. +use std::fs; use std::path::Path; use crate::db::DbConnection as Connection; @@ -9,7 +10,8 @@ use crate::db::queries::{ media as media_q, media_translation, post as post_q, post_translation, project as project_q, setting, }; -use crate::engine::{EngineResult, domain_events}; +use crate::engine::{EngineError, EngineResult, domain_events}; +use crate::util::frontmatter::{read_post_file, read_translation_file}; use crate::util::now_unix_ms; const REBUILD_REQUIRED_SETTING: &str = "app.search-index-rebuild-required"; @@ -218,19 +220,30 @@ fn index_project( let translations = post_translation::list_post_translations_by_post(conn, &post.id)?; let translation_data = translations .iter() - .map(|translation| fts::PostTranslationFts { - title: translation.title.clone(), - excerpt: translation.excerpt.clone(), - content: translation.content.clone(), - language: translation.language.clone(), + .map(|translation| { + Ok(fts::PostTranslationFts { + title: translation.title.clone(), + excerpt: translation.excerpt.clone(), + content: data_dir + .map(|dir| resolve_translation_fts_content(dir, translation)) + .transpose()? + .flatten() + .or_else(|| translation.content.clone()), + language: translation.language.clone(), + }) }) - .collect::>(); + .collect::>>()?; + let content = data_dir + .map(|dir| resolve_post_fts_content(dir, &post)) + .transpose()? + .flatten() + .or_else(|| post.content.clone()); fts::index_post( conn, &post.id, &post.title, post.excerpt.as_deref(), - post.content.as_deref(), + content.as_deref(), &post.tags, &post.categories, &translation_data, @@ -271,11 +284,41 @@ fn index_project( Ok(report) } +fn resolve_post_fts_content( + data_dir: &Path, + post: &crate::model::Post, +) -> EngineResult> { + if post.content.is_some() { + return Ok(post.content.clone()); + } + if post.file_path.is_empty() { + return Ok(None); + } + let raw = fs::read_to_string(data_dir.join(&post.file_path))?; + let (_fm, body) = read_post_file(&raw).map_err(EngineError::Parse)?; + Ok(Some(body)) +} + +fn resolve_translation_fts_content( + data_dir: &Path, + translation: &crate::model::PostTranslation, +) -> EngineResult> { + if translation.content.is_some() { + return Ok(translation.content.clone()); + } + if translation.file_path.is_empty() { + return Ok(None); + } + let raw = fs::read_to_string(data_dir.join(&translation.file_path))?; + let (_fm, body) = read_translation_file(&raw).map_err(EngineError::Parse)?; + Ok(Some(body)) +} + #[cfg(test)] mod tests { use super::*; use crate::db::Database; - use crate::db::fts::ensure_fts_tables; + use crate::db::fts::{self, ensure_fts_tables}; use crate::engine; fn setup() -> (Database, String) { @@ -330,6 +373,57 @@ mod tests { assert_eq!(results.len(), 1); } + #[test] + fn rebuild_indexes_published_post_and_translation_bodies_from_files() { + let db = Database::open_in_memory().unwrap(); + let _ = db.migrate(); + ensure_fts_tables(db.conn()).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let project = engine::project::create_project( + db.conn(), + "Published Project", + Some(dir.path().to_str().unwrap()), + ) + .unwrap(); + let post = engine::post::create_post( + db.conn(), + dir.path(), + &project.id, + "Published Rebuild", + Some("distinctive platypusnova body"), + vec![], + vec![], + None, + Some("en"), + None, + ) + .unwrap(); + engine::post::upsert_translation( + db.conn(), + dir.path(), + &post.id, + "de", + "Veröffentlichter Neuaufbau", + None, + Some("markantes schmetterlingskomet wort"), + ) + .unwrap(); + engine::post::publish_post(db.conn(), dir.path(), &post.id).unwrap(); + fts::remove_post_from_index(db.conn(), &post.id).unwrap(); + + let report = reindex_project(db.conn(), &project.id, None).unwrap(); + + assert_eq!(report.posts_indexed, 1); + assert_eq!( + fts::search_posts(db.conn(), "platypusnova", "en").unwrap(), + vec![post.id.clone()] + ); + assert_eq!( + fts::search_posts(db.conn(), "schmetterlingskomet", "de").unwrap(), + vec![post.id] + ); + } + #[test] fn shared_rebuild_indexes_every_project() { let (db, first_project_id) = setup();