hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix search indexing data directory

Browse Source
This commit is contained in:
Georg Bauer
2026-07-21 13:27:09 +02:00
parent 184d6e4845
commit 18d96a10b8
2 changed files with 216 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
crates/bds-core/src/engine/post.rs
View File

@@ -88,7 +88,7 @@ pub fn create_post(
qp::insert_post(conn, &post)?;
// Index for FTS
fts_index_post(conn, &post)?;
fts_index_post(conn, data_dir, &post)?;
emit_post(&post, NotificationAction::Created);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
@@ -185,7 +185,7 @@ pub fn update_post(
qp::update_post(conn, &post)?;
// Re-index FTS
fts_index_post(conn, &post)?;
fts_index_post(conn, data_dir, &post)?;
emit_post(&post, NotificationAction::Updated);
crate::engine::embedding::sync_post_best_effort(conn, data_dir, &post);
@@ -319,7 +319,7 @@ fn publish_post_in_savepoint(
}
// Re-index FTS
fts_index_post(conn, &post)?;
fts_index_post(conn, data_dir, &post)?;
Ok(post)
}
@@ -454,7 +454,7 @@ pub fn discard_post_draft(conn: &Connection, data_dir: &Path, post_id: &str) ->
conn.begin_savepoint()?;
match (|| {
qp::update_post(conn, &post)?;
fts_index_post(conn, &post)?;
fts_index_post(conn, data_dir, &post)?;
Ok(post)
})() {
Ok(post) => {
@@ -571,7 +571,7 @@ pub fn upsert_translation(
qt::update_post_translation(conn, &t)?;
// Re-index FTS for parent post
fts_index_post(conn, &post)?;
fts_index_post(conn, data_dir, &post)?;
Ok(t)
}
@@ -596,7 +596,7 @@ pub fn upsert_translation(
qt::insert_post_translation(conn, &t)?;
// Re-index FTS for parent post
fts_index_post(conn, &post)?;
fts_index_post(conn, data_dir, &post)?;
Ok(t)
}
@@ -623,7 +623,7 @@ pub fn delete_translation(
// Re-index FTS for parent post
if let Ok(post) = qp::get_post_by_id(conn, &t.translation_for) {
fts_index_post(conn, &post)?;
fts_index_post(conn, data_dir, &post)?;
}
Ok(())
@@ -643,6 +643,7 @@ pub fn publish_post_translation(
conn.begin_savepoint()?;
match publish_translation(conn, data_dir, &mut translation, &post) {
Ok(()) => {
fts_index_post(conn, data_dir, &post)?;
conn.release_savepoint()?;
Ok(translation)
}
@@ -747,7 +748,7 @@ pub fn rebuild_posts_from_filesystem_with_progress(
// Re-index FTS for all posts in this project
let posts = qp::list_posts_by_project(conn, project_id)?;
for post in &posts {
fts_index_post(conn, post)?;
fts_index_post(conn, data_dir, post)?;
}
Ok(report)
@@ -895,25 +896,32 @@ fn publish_translation(
}
/// Index a post in FTS, gathering translation texts.
fn fts_index_post(conn: &Connection, post: &Post) -> EngineResult<()> {
fn fts_index_post(conn: &Connection, data_dir: &Path, post: &Post) -> EngineResult<()> {
let translations = qt::list_post_translations_by_post(conn, &post.id).unwrap_or_default();
let translation_data: Vec<fts::PostTranslationFts> = translations
.iter()
.map(|t| fts::PostTranslationFts {
title: t.title.clone(),
excerpt: t.excerpt.clone(),
content: t.content.clone(),
language: t.language.clone(),
.map(|t| {
Ok(fts::PostTranslationFts {
title: t.title.clone(),
excerpt: t.excerpt.clone(),
content: resolve_translation_fts_content(data_dir, t)?,
language: t.language.clone(),
})
})
.collect();
.collect::<EngineResult<_>>()?;
let lang = post.language.as_deref().unwrap_or("en");
let main_language = crate::engine::meta::read_project_json(data_dir)
.ok()
.and_then(|metadata| metadata.main_language)
.unwrap_or_else(|| "en".to_string());
let content = resolve_post_fts_content(data_dir, post)?;
let lang = post.language.as_deref().unwrap_or(&main_language);
fts::index_post(
conn,
&post.id,
&post.title,
post.excerpt.as_deref(),
post.content.as_deref(),
content.as_deref(),
&post.tags,
&post.categories,
&translation_data,
@@ -922,6 +930,33 @@ fn fts_index_post(conn: &Connection, post: &Post) -> EngineResult<()> {
Ok(())
}
fn resolve_post_fts_content(data_dir: &Path, post: &Post) -> EngineResult<Option<String>> {
if post.content.is_some() {
return Ok(post.content.clone());
}
if post.file_path.is_empty() {
return Ok(None);
}
let raw = fs::read_to_string(data_dir.join(&post.file_path))?;
let (_fm, body) = read_post_file(&raw).map_err(EngineError::Parse)?;
Ok(Some(body))
}
fn resolve_translation_fts_content(
data_dir: &Path,
translation: &PostTranslation,
) -> EngineResult<Option<String>> {
if translation.content.is_some() {
return Ok(translation.content.clone());
}
if translation.file_path.is_empty() {
return Ok(None);
}
let raw = fs::read_to_string(data_dir.join(&translation.file_path))?;
let (_fm, body) = read_translation_file(&raw).map_err(EngineError::Parse)?;
Ok(Some(body))
}
/// Check if a file stem looks like a translation filename: `{slug}.{lang}`
/// where lang is a 2-letter code. We look for a dot followed by exactly 2 lowercase letters.
pub(crate) fn is_translation_filename(stem: &str) -> bool {
@@ -1190,6 +1225,64 @@ mod tests {
);
}
#[test]
fn publish_indexes_post_body_from_file_when_db_content_is_cleared() {
let (db, dir) = setup();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Published Search",
Some("distinctive quokkafire body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
publish_post(db.conn(), dir.path(), &post.id).unwrap();
let stored = qp::get_post_by_id(db.conn(), &post.id).unwrap();
assert!(stored.content.is_none());
assert_eq!(
crate::db::fts::search_posts(db.conn(), "quokkafire", "en").unwrap(),
vec![post.id]
);
}
#[test]
fn publish_indexes_translation_body_from_file_when_db_content_is_cleared() {
let (db, dir) = setup();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Translated Search",
Some("canonical body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
upsert_translation(
db.conn(),
dir.path(),
&post.id,
"de",
"Übersetzte Suche",
None,
Some("markantes drachenfalter wort"),
)
.unwrap();
publish_post(db.conn(), dir.path(), &post.id).unwrap();
assert_eq!(
crate::db::fts::search_posts(db.conn(), "drachenfalter", "de").unwrap(),
vec![post.id]
);
}
#[test]
fn create_post_empty_title_uses_untitled() {
let (db, dir) = setup();

117
crates/bds-core/src/engine/search.rs
View File

@@ -1,5 +1,6 @@
//! Full-text search reindexing engine functions.
use std::fs;
use std::path::Path;
use crate::db::DbConnection as Connection;
@@ -9,7 +10,8 @@ use crate::db::queries::{
media as media_q, media_translation, post as post_q, post_translation, project as project_q,
setting,
};
use crate::engine::{EngineResult, domain_events};
use crate::engine::{EngineError, EngineResult, domain_events};
use crate::util::frontmatter::{read_post_file, read_translation_file};
use crate::util::now_unix_ms;
const REBUILD_REQUIRED_SETTING: &str = "app.search-index-rebuild-required";
@@ -201,8 +203,9 @@ fn index_project(
current: &mut usize,
on_item: Option<&ItemProgressFn>,
) -> EngineResult<ReindexReport> {
let main_language = data_path
.and_then(|path| crate::engine::meta::read_project_json(Path::new(path)).ok())
let data_dir = data_path.map(Path::new);
let main_language = data_dir
.and_then(|path| crate::engine::meta::read_project_json(path).ok())
.and_then(|metadata| metadata.main_language)
.unwrap_or_else(|| "en".to_string());
let mut report = ReindexReport {
@@ -218,19 +221,30 @@ fn index_project(
let translations = post_translation::list_post_translations_by_post(conn, &post.id)?;
let translation_data = translations
.iter()
.map(|translation| fts::PostTranslationFts {
title: translation.title.clone(),
excerpt: translation.excerpt.clone(),
content: translation.content.clone(),
language: translation.language.clone(),
.map(|translation| {
Ok(fts::PostTranslationFts {
title: translation.title.clone(),
excerpt: translation.excerpt.clone(),
content: data_dir
.map(|dir| resolve_translation_fts_content(dir, translation))
.transpose()?
.flatten()
.or_else(|| translation.content.clone()),
language: translation.language.clone(),
})
})
.collect::<Vec<_>>();
.collect::<EngineResult<Vec<_>>>()?;
let content = data_dir
.map(|dir| resolve_post_fts_content(dir, &post))
.transpose()?
.flatten()
.or_else(|| post.content.clone());
fts::index_post(
conn,
&post.id,
&post.title,
post.excerpt.as_deref(),
post.content.as_deref(),
content.as_deref(),
&post.tags,
&post.categories,
&translation_data,
@@ -271,11 +285,41 @@ fn index_project(
Ok(report)
}
fn resolve_post_fts_content(
data_dir: &Path,
post: &crate::model::Post,
) -> EngineResult<Option<String>> {
if post.content.is_some() {
return Ok(post.content.clone());
}
if post.file_path.is_empty() {
return Ok(None);
}
let raw = fs::read_to_string(data_dir.join(&post.file_path))?;
let (_fm, body) = read_post_file(&raw).map_err(EngineError::Parse)?;
Ok(Some(body))
}
fn resolve_translation_fts_content(
data_dir: &Path,
translation: &crate::model::PostTranslation,
) -> EngineResult<Option<String>> {
if translation.content.is_some() {
return Ok(translation.content.clone());
}
if translation.file_path.is_empty() {
return Ok(None);
}
let raw = fs::read_to_string(data_dir.join(&translation.file_path))?;
let (_fm, body) = read_translation_file(&raw).map_err(EngineError::Parse)?;
Ok(Some(body))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::db::fts::ensure_fts_tables;
use crate::db::fts::{self, ensure_fts_tables};
use crate::engine;
fn setup() -> (Database, String) {
@@ -330,6 +374,57 @@ mod tests {
assert_eq!(results.len(), 1);
}
#[test]
fn rebuild_indexes_published_post_and_translation_bodies_from_files() {
let db = Database::open_in_memory().unwrap();
let _ = db.migrate();
ensure_fts_tables(db.conn()).unwrap();
let dir = tempfile::tempdir().unwrap();
let project = engine::project::create_project(
db.conn(),
"Published Project",
Some(dir.path().to_str().unwrap()),
)
.unwrap();
let post = engine::post::create_post(
db.conn(),
dir.path(),
&project.id,
"Published Rebuild",
Some("distinctive platypusnova body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
engine::post::upsert_translation(
db.conn(),
dir.path(),
&post.id,
"de",
"Veröffentlichter Neuaufbau",
None,
Some("markantes schmetterlingskomet wort"),
)
.unwrap();
engine::post::publish_post(db.conn(), dir.path(), &post.id).unwrap();
fts::remove_post_from_index(db.conn(), &post.id).unwrap();
let report = reindex_project(db.conn(), &project.id, None).unwrap();
assert_eq!(report.posts_indexed, 1);
assert_eq!(
fts::search_posts(db.conn(), "platypusnova", "en").unwrap(),
vec![post.id.clone()]
);
assert_eq!(
fts::search_posts(db.conn(), "schmetterlingskomet", "de").unwrap(),
vec![post.id]
);
}
#[test]
fn shared_rebuild_indexes_every_project() {
let (db, first_project_id) = setup();