hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: database migration

Browse Source
This commit is contained in:
Chili Palmer
2026-04-05 12:47:58 +02:00
parent e0c0c8a31e
commit c7a939736c
59 changed files with 766 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
crates/bds-core/src/db/fts.rs
View File

@@ -63,9 +63,25 @@ pub fn stem_text(text: &str, language: &str) -> String {
.join(" ")
}
/// Structured translation data for FTS indexing of posts.
pub struct PostTranslationFts {
pub title: String,
pub excerpt: Option<String>,
pub content: Option<String>,
pub language: String,
}
/// Structured translation data for FTS indexing of media.
pub struct MediaTranslationFts {
pub title: Option<String>,
pub alt: Option<String>,
pub caption: Option<String>,
pub language: String,
}
/// Index a post in the FTS table with separate columns per spec.
///
/// Concatenates translation text into the content column (stemmed per-language).
/// Translation titles go to the title column, excerpts to excerpt, content to content.
pub fn index_post(
conn: &Connection,
post_id: &str,
@@ -74,22 +90,37 @@ pub fn index_post(
content: Option<&str>,
tags: &[String],
categories: &[String],
translations: &[(String, String)], // (text, language) pairs
translations: &[PostTranslationFts],
language: &str,
) -> rusqlite::Result<()> {
// Remove existing entry
remove_post_from_index(conn, post_id)?;
let stemmed_title = stem_text(title, language);
let stemmed_excerpt = stem_text(excerpt.unwrap_or(""), language);
// Title column: post title + all translation titles
let mut title_parts = vec![stem_text(title, language)];
for t in translations {
title_parts.push(stem_text(&t.title, &t.language));
}
let stemmed_title = title_parts.join(" ");
// Content column: post content + all translation texts
// Excerpt column: post excerpt + all translation excerpts
let mut excerpt_parts = vec![stem_text(excerpt.unwrap_or(""), language)];
for t in translations {
if let Some(ref exc) = t.excerpt {
excerpt_parts.push(stem_text(exc, &t.language));
}
}
let stemmed_excerpt = excerpt_parts.join(" ");
// Content column: post content + all translation content
let mut content_parts = Vec::new();
if let Some(cnt) = content {
content_parts.push(stem_text(cnt, language));
}
for (text, trans_lang) in translations {
content_parts.push(stem_text(text, trans_lang));
for t in translations {
if let Some(ref cnt) = t.content {
content_parts.push(stem_text(cnt, &t.language));
}
}
let stemmed_content = content_parts.join(" ");
@@ -104,6 +135,8 @@ pub fn index_post(
}
/// Index a media item in the FTS table with separate columns per spec.
///
/// Translation titles go to the title column, alts to alt, captions to caption.
pub fn index_media(
conn: &Connection,
media_id: &str,
@@ -112,21 +145,38 @@ pub fn index_media(
caption: Option<&str>,
original_name: &str,
tags: &[String],
translations: &[(String, String)], // (text, language) pairs
translations: &[MediaTranslationFts],
language: &str,
) -> rusqlite::Result<()> {
remove_media_from_index(conn, media_id)?;
let stemmed_title = stem_text(title.unwrap_or(""), language);
let stemmed_alt = stem_text(alt.unwrap_or(""), language);
// Title column: media title + all translation titles
let mut title_parts = vec![stem_text(title.unwrap_or(""), language)];
for t in translations {
if let Some(ref ttl) = t.title {
title_parts.push(stem_text(ttl, &t.language));
}
}
let stemmed_title = title_parts.join(" ");
// Caption column: media caption + all translation texts
// Alt column: media alt + all translation alts
let mut alt_parts = vec![stem_text(alt.unwrap_or(""), language)];
for t in translations {
if let Some(ref a) = t.alt {
alt_parts.push(stem_text(a, &t.language));
}
}
let stemmed_alt = alt_parts.join(" ");
// Caption column: media caption + all translation captions
let mut caption_parts = Vec::new();
if let Some(c) = caption {
caption_parts.push(stem_text(c, language));
}
for (text, trans_lang) in translations {
caption_parts.push(stem_text(text, trans_lang));
for t in translations {
if let Some(ref cap) = t.caption {
caption_parts.push(stem_text(cap, &t.language));
}
}
let stemmed_caption = caption_parts.join(" ");
@@ -176,8 +226,12 @@ pub struct PostSearchFilters<'a> {
pub status: Option<&'a str>,
pub tags: Option<&'a [String]>,
pub categories: Option<&'a [String]>,
pub language: Option<&'a str>,
pub missing_translation_language: Option<&'a str>,
pub year: Option<i32>,
pub month: Option<u32>,
pub from: Option<i64>,
pub to: Option<i64>,
pub limit: Option<usize>,
pub offset: Option<usize>,
}
@@ -263,6 +317,32 @@ pub fn search_posts_filtered(
}
}
if let Some(lang) = filters.language {
sql.push_str(&format!("AND (language = ?{param_idx} OR language IS NULL) "));
params.push(Box::new(lang.to_string()));
param_idx += 1;
}
if let Some(missing_lang) = filters.missing_translation_language {
sql.push_str(&format!(
"AND NOT EXISTS (SELECT 1 FROM post_translations WHERE post_translations.translation_for = posts.id AND post_translations.language = ?{param_idx}) "
));
params.push(Box::new(missing_lang.to_string()));
param_idx += 1;
}
if let Some(from_ts) = filters.from {
sql.push_str(&format!("AND created_at >= ?{param_idx} "));
params.push(Box::new(from_ts));
param_idx += 1;
}
if let Some(to_ts) = filters.to {
sql.push_str(&format!("AND created_at <= ?{param_idx} "));
params.push(Box::new(to_ts));
param_idx += 1;
}
let _ = param_idx; // suppress unused warning
// First get total count (without LIMIT/OFFSET)
@@ -439,7 +519,12 @@ mod tests {
Some("Beautiful spider web"),
&["fotografie".into(), "natur".into()],
&["picture".into()],
&[("Esmeralda English".into(), "en".into())],
&[PostTranslationFts {
title: "Esmeralda English".into(),
excerpt: None,
content: None,
language: "en".into(),
}],
"en",
).unwrap();
@@ -503,7 +588,12 @@ mod tests {
index_post(
db.conn(), "p1", "Programmierung", None, Some("Deutsche Entwicklung"),
&[], &[],
&[("English development programming".into(), "en".into())],
&[PostTranslationFts {
title: "English development programming".into(),
excerpt: None,
content: Some("English development programming".into()),
language: "en".into(),
}],
"de",
).unwrap();

4
crates/bds-core/src/db/migrations.rs
View File

@@ -745,7 +745,7 @@ mod tests {
assert_eq!(kind, "post", "default kind must be 'post'");
assert_eq!(enabled, 1, "default enabled must be 1");
assert_eq!(version, 1, "default version must be 1");
assert_eq!(status, "published", "default status must be 'published'");
assert_eq!(status, "draft", "default status must be 'draft'");
}
#[test]
@@ -766,6 +766,6 @@ mod tests {
assert_eq!(ep, "render", "default entrypoint must be 'render'");
assert_eq!(enabled, 1, "default enabled must be 1");
assert_eq!(version, 1, "default version must be 1");
assert_eq!(status, "published", "default status must be 'published'");
assert_eq!(status, "draft", "default status must be 'draft'");
}
}

19
crates/bds-core/src/engine/media.rs
View File

@@ -444,20 +444,13 @@ pub fn rebuild_media_from_filesystem_with_progress(
/// Index a media item in FTS, gathering translation texts.
fn fts_index_media(conn: &Connection, media: &Media) -> EngineResult<()> {
let translations = qmt::list_media_translations_by_media(conn, &media.id).unwrap_or_default();
let translation_data: Vec<(String, String)> = translations
let translation_data: Vec<fts::MediaTranslationFts> = translations
.iter()
.map(|t| {
let mut parts = Vec::new();
if let Some(ref title) = t.title {
parts.push(title.clone());
}
if let Some(ref alt) = t.alt {
parts.push(alt.clone());
}
if let Some(ref caption) = t.caption {
parts.push(caption.clone());
}
(parts.join(" "), t.language.clone())
.map(|t| fts::MediaTranslationFts {
title: t.title.clone(),
alt: t.alt.clone(),
caption: t.caption.clone(),
language: t.language.clone(),
})
.collect();

59
crates/bds-core/src/engine/post.rs
View File

@@ -156,8 +156,19 @@ pub fn update_post(
post.do_not_translate = dnt;
}
// Auto-transition published post back to draft on content/metadata change
if post.status == PostStatus::Published {
// Auto-transition published or archived post back to draft on content/metadata change
if post.status == PostStatus::Published || post.status == PostStatus::Archived {
// Reload content from filesystem if content field is NULL (published state)
if post.content.is_none() && !post.file_path.is_empty() {
let abs_path = _data_dir.join(&post.file_path);
if abs_path.exists() {
if let Ok(file_content) = fs::read_to_string(&abs_path) {
if let Ok((_fm, body)) = read_post_file(&file_content) {
post.content = Some(body);
}
}
}
}
post.status = PostStatus::Draft;
}
@@ -294,10 +305,24 @@ pub fn publish_post(
}
/// Archive a post.
pub fn archive_post(conn: &Connection, post_id: &str) -> EngineResult<()> {
let post = qp::get_post_by_id(conn, post_id)?;
pub fn archive_post(conn: &Connection, data_dir: &Path, post_id: &str) -> EngineResult<()> {
let mut post = qp::get_post_by_id(conn, post_id)?;
if post.status == PostStatus::Archived {
return Ok(());
return Err(EngineError::Conflict(
"post is already archived".to_string(),
));
}
// Reload content from filesystem if transitioning from published (content is NULL)
if post.status == PostStatus::Published && post.content.is_none() && !post.file_path.is_empty() {
let abs_path = data_dir.join(&post.file_path);
if abs_path.exists() {
if let Ok(file_content) = fs::read_to_string(&abs_path) {
if let Ok((_fm, body)) = read_post_file(&file_content) {
post.content = Some(body);
qp::update_post(conn, &post)?;
}
}
}
}
let now = now_unix_ms();
qp::update_post_status(conn, post_id, &PostStatus::Archived, now)?;
@@ -646,17 +671,13 @@ fn publish_translation(
/// Index a post in FTS, gathering translation texts.
fn fts_index_post(conn: &Connection, post: &Post) -> EngineResult<()> {
let translations = qt::list_post_translations_by_post(conn, &post.id).unwrap_or_default();
let translation_data: Vec<(String, String)> = translations
let translation_data: Vec<fts::PostTranslationFts> = translations
.iter()
.map(|t| {
let mut parts = vec![t.title.clone()];
if let Some(ref exc) = t.excerpt {
parts.push(exc.clone());
}
if let Some(ref cnt) = t.content {
parts.push(cnt.clone());
}
(parts.join(" "), t.language.clone())
.map(|t| fts::PostTranslationFts {
title: t.title.clone(),
excerpt: t.excerpt.clone(),
content: t.content.clone(),
language: t.language.clone(),
})
.collect();
@@ -1031,7 +1052,7 @@ mod tests {
publish_post(db.conn(), dir.path(), &post.id).unwrap();
// Archive so we can try updating
archive_post(db.conn(), &post.id).unwrap();
archive_post(db.conn(), dir.path(), &post.id).unwrap();
// Try to change slug - should fail
let result = update_post(
@@ -1124,7 +1145,7 @@ mod tests {
let original_published_at = published.published_at.unwrap();
// Archive then re-publish
archive_post(db.conn(), &post.id).unwrap();
archive_post(db.conn(), dir.path(), &post.id).unwrap();
// Brief delay to ensure now() is different
let republished = publish_post(db.conn(), dir.path(), &post.id).unwrap();
@@ -1406,7 +1427,7 @@ mod tests {
publish_post(db.conn(), dir.path(), &post.id).unwrap();
// Archive then update to test auto-draft
archive_post(db.conn(), &post.id).unwrap();
archive_post(db.conn(), dir.path(), &post.id).unwrap();
// Re-publish
let _ = publish_post(db.conn(), dir.path(), &post.id).unwrap();
@@ -1475,7 +1496,7 @@ mod tests {
vec![], vec![], None, None, None,
).unwrap();
publish_post(db.conn(), dir.path(), &post.id).unwrap();
archive_post(db.conn(), &post.id).unwrap();
archive_post(db.conn(), dir.path(), &post.id).unwrap();
let from_db = qp::get_post_by_id(db.conn(), &post.id).unwrap();
assert_eq!(from_db.status, PostStatus::Archived);

28
crates/bds-core/src/engine/project.rs
View File

@@ -298,16 +298,24 @@ mod tests {
#[test]
fn delete_project_removes_row() {
let (db, dir) = setup();
// Project with no custom data_path → uses internal directory
let p = create_project(
db.conn(),
"P",
None,
)
.unwrap();
let internal_dir = dir.path().join("projects").join(&p.id);
let _ = std::fs::create_dir_all(&internal_dir);
delete_project(db.conn(), &p.id, Some(&internal_dir)).unwrap();
// Simulate an internal project (no custom data_path) by inserting directly
let now = crate::util::now_unix_ms();
let project = Project {
id: uuid::Uuid::new_v4().to_string(),
name: "P".to_string(),
slug: "p".to_string(),
description: None,
data_path: None,
is_active: false,
created_at: now,
updated_at: now,
};
crate::db::queries::project::insert_project(db.conn(), &project).unwrap();
// Create the internal directory structure under the temp dir
let internal_dir = dir.path().join("projects").join(&project.id);
create_directory_structure(&internal_dir).unwrap();
assert!(internal_dir.join("posts").is_dir());
delete_project(db.conn(), &project.id, Some(&internal_dir)).unwrap();
assert!(list_projects(db.conn()).unwrap().is_empty());
assert!(!internal_dir.exists(), "internal directory should be cleaned up");
}

34
crates/bds-core/src/engine/search.rs
View File

@@ -50,16 +50,13 @@ pub fn reindex_all_with_progress(
}
let translations = post_translation::list_post_translations_by_post(conn, &post.id)?;
let trans_pairs: Vec<(String, String)> = translations
let trans_data: Vec<fts::PostTranslationFts> = translations
.iter()
.map(|t| {
let text = [
t.title.as_str(),
t.excerpt.as_deref().unwrap_or(""),
t.content.as_deref().unwrap_or(""),
]
.join(" ");
(text, t.language.clone())
.map(|t| fts::PostTranslationFts {
title: t.title.clone(),
excerpt: t.excerpt.clone(),
content: t.content.clone(),
language: t.language.clone(),
})
.collect();
@@ -72,7 +69,7 @@ pub fn reindex_all_with_progress(
post.content.as_deref(),
&post.tags,
&post.categories,
&trans_pairs,
&trans_data,
language,
)?;
@@ -87,16 +84,13 @@ pub fn reindex_all_with_progress(
}
let translations = media_translation::list_media_translations_by_media(conn, &m.id)?;
let trans_pairs: Vec<(String, String)> = translations
let trans_data: Vec<fts::MediaTranslationFts> = translations
.iter()
.map(|t| {
let text = [
t.title.as_deref().unwrap_or(""),
t.alt.as_deref().unwrap_or(""),
t.caption.as_deref().unwrap_or(""),
]
.join(" ");
(text, t.language.clone())
.map(|t| fts::MediaTranslationFts {
title: t.title.clone(),
alt: t.alt.clone(),
caption: t.caption.clone(),
language: t.language.clone(),
})
.collect();
@@ -109,7 +103,7 @@ pub fn reindex_all_with_progress(
m.caption.as_deref(),
&m.original_name,
&m.tags,
&trans_pairs,
&trans_data,
language,
)?;

53
crates/bds-core/src/engine/task.rs
View File

@@ -8,7 +8,7 @@ pub type TaskId = u64;
/// Task status.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TaskStatus {
Queued,
Pending,
Running,
Completed,
Failed(String),
@@ -28,11 +28,14 @@ pub struct TaskProgress {
struct TaskEntry {
id: TaskId,
label: String,
group_id: Option<String>,
group_name: Option<String>,
status: TaskStatus,
cancel_flag: Arc<AtomicBool>,
progress: Option<f32>,
message: Option<String>,
created_at: Instant,
last_progress_report: Option<Instant>,
}
/// Manages concurrent tasks with a max concurrency limit and FIFO queue.
@@ -61,11 +64,14 @@ impl TaskManager {
let entry = TaskEntry {
id,
label: label.to_owned(),
status: TaskStatus::Queued,
group_id: None,
group_name: None,
status: TaskStatus::Pending,
cancel_flag: Arc::new(AtomicBool::new(false)),
progress: None,
message: None,
created_at: Instant::now(),
last_progress_report: None,
};
let mut tasks = self.tasks.lock().unwrap();
@@ -73,13 +79,24 @@ impl TaskManager {
// Auto-start if under capacity
let running = tasks.iter().filter(|t| t.status == TaskStatus::Running).count();
if running < self.max_concurrent {
if let Some(t) = tasks.iter_mut().find(|t| t.id == id && t.status == TaskStatus::Queued) {
if let Some(t) = tasks.iter_mut().find(|t| t.id == id && t.status == TaskStatus::Pending) {
t.status = TaskStatus::Running;
}
}
id
}
/// Submit a new task within a group. Returns its unique identifier.
pub fn submit_grouped(&self, label: &str, group_id: &str, group_name: &str) -> TaskId {
let id = self.submit(label);
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == id) {
entry.group_id = Some(group_id.to_owned());
entry.group_name = Some(group_name.to_owned());
}
id
}
/// Try to start a queued task. Returns true if the task was moved to
/// Running, false if concurrency is at capacity or the task is not Queued.
pub fn try_start(&self, task_id: TaskId) -> bool {
@@ -89,7 +106,7 @@ impl TaskManager {
return false;
}
if let Some(entry) = tasks.iter_mut().find(|t| t.id == task_id) {
if entry.status == TaskStatus::Queued {
if entry.status == TaskStatus::Pending {
entry.status = TaskStatus::Running;
return true;
}
@@ -125,7 +142,7 @@ impl TaskManager {
pub fn cancel(&self, task_id: TaskId) {
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == task_id) {
if matches!(entry.status, TaskStatus::Running | TaskStatus::Queued) {
if matches!(entry.status, TaskStatus::Running | TaskStatus::Pending) {
entry.cancel_flag.store(true, Ordering::Release);
entry.status = TaskStatus::Cancelled;
}
@@ -152,7 +169,7 @@ impl TaskManager {
/// Count tasks that are still queued.
pub fn pending_count(&self) -> usize {
let tasks = self.tasks.lock().unwrap();
tasks.iter().filter(|t| t.status == TaskStatus::Queued).count()
tasks.iter().filter(|t| t.status == TaskStatus::Pending).count()
}
/// Count tasks that are currently running.
@@ -164,7 +181,7 @@ impl TaskManager {
/// Remove all completed, failed, and cancelled tasks.
pub fn drain_completed(&self) {
let mut tasks = self.tasks.lock().unwrap();
tasks.retain(|t| matches!(t.status, TaskStatus::Queued | TaskStatus::Running));
tasks.retain(|t| matches!(t.status, TaskStatus::Pending | TaskStatus::Running));
}
/// Return the label of a task.
@@ -176,16 +193,24 @@ impl TaskManager {
/// Return the id of the first queued task (FIFO order).
pub fn next_queued(&self) -> Option<TaskId> {
let tasks = self.tasks.lock().unwrap();
tasks.iter().find(|t| t.status == TaskStatus::Queued).map(|t| t.id)
tasks.iter().find(|t| t.status == TaskStatus::Pending).map(|t| t.id)
}
/// Update progress for a running task.
/// Update progress for a running task. Throttled to at most once per 250ms.
pub fn report_progress(&self, task_id: TaskId, progress: Option<f32>, message: Option<String>) {
let mut tasks = self.tasks.lock().unwrap();
if let Some(entry) = tasks.iter_mut().find(|t| t.id == task_id) {
if entry.status == TaskStatus::Running {
entry.progress = progress;
entry.message = message;
let now = Instant::now();
let should_report = match entry.last_progress_report {
Some(prev) => now.duration_since(prev).as_millis() >= PROGRESS_THROTTLE_MS as u128,
None => true,
};
if should_report {
entry.progress = progress;
entry.message = message;
entry.last_progress_report = Some(now);
}
}
}
}
@@ -215,7 +240,7 @@ impl TaskManager {
fn promote_next(tasks: &mut Vec<TaskEntry>, max_concurrent: usize) {
let running = tasks.iter().filter(|t| t.status == TaskStatus::Running).count();
if running < max_concurrent {
if let Some(t) = tasks.iter_mut().find(|t| t.status == TaskStatus::Queued) {
if let Some(t) = tasks.iter_mut().find(|t| t.status == TaskStatus::Pending) {
t.status = TaskStatus::Running;
}
}
@@ -279,7 +304,7 @@ mod tests {
// First 3 auto-started, 4th stays queued
assert_eq!(mgr.running_count(), 3);
assert_eq!(mgr.status(ids[3]), Some(TaskStatus::Queued));
assert_eq!(mgr.status(ids[3]), Some(TaskStatus::Pending));
}
#[test]
@@ -354,7 +379,7 @@ mod tests {
let b = mgr.submit("second"); // queued
assert_eq!(mgr.status(a), Some(TaskStatus::Running));
assert_eq!(mgr.status(b), Some(TaskStatus::Queued));
assert_eq!(mgr.status(b), Some(TaskStatus::Pending));
mgr.complete(a);
assert_eq!(mgr.status(b), Some(TaskStatus::Running));

5
crates/bds-core/src/model/generation.rs
View File

@@ -58,5 +58,10 @@ pub struct PublishingPreferences {
pub ssh_user: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ssh_remote_path: Option<String>,
#[serde(default = "default_ssh_mode")]
pub ssh_mode: SshMode,
}
fn default_ssh_mode() -> SshMode {
SshMode::Scp
}