hugo/RuDS
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: completed base feature set for the app

Browse Source
This commit is contained in:
Chili Palmer
2026-07-18 20:25:17 +02:00
parent e9d6c70eb0
commit 638202b96c
46 changed files with 4486 additions and 541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

441
crates/bds-core/src/engine/auto_translation.rs Normal file
View File

@@ -0,0 +1,441 @@
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use serde_json::json;
use crate::db::DbConnection as Connection;
use crate::db::queries::{
media as qm, media_translation as qmt, post as qp, post_media, post_translation,
};
use crate::engine::ai::{
self, MediaTranslationResult, OneShotOperation, OneShotRequest, OneShotResponse,
TranslationResult,
};
use crate::engine::{EngineError, EngineResult};
use crate::model::{Media, Post, PostStatus};
use crate::util::frontmatter::read_post_file;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FillMissingTranslationsReport {
pub translated_posts: usize,
pub translated_media: usize,
pub failed_count: usize,
pub warned_count: usize,
pub nothing_to_do: bool,
pub errors: Vec<String>,
}
pub fn configured_languages(main_language: &str, blog_languages: &[String]) -> Vec<String> {
let mut seen = HashSet::new();
std::iter::once(main_language.to_string())
.chain(blog_languages.iter().cloned())
.map(|language| normalize_language(&language))
.filter(|language| !language.is_empty() && seen.insert(language.clone()))
.collect()
}
pub fn missing_languages(
conn: &Connection,
post: &Post,
configured: &[String],
) -> EngineResult<Vec<String>> {
if post.do_not_translate {
return Ok(Vec::new());
}
let source = normalize_language(post.language.as_deref().unwrap_or("en"));
let existing = post_translation::list_post_translations_by_post(conn, &post.id)?
.into_iter()
.map(|translation| normalize_language(&translation.language))
.collect::<HashSet<_>>();
Ok(configured
.iter()
.filter(|language| **language != source && !existing.contains(*language))
.cloned()
.collect())
}
/// Batch maintenance path. Generated post translations are published, while
/// per-item failures are accumulated and never abort the batch.
pub fn fill_missing_translations(
conn: &Connection,
data_dir: &Path,
project_id: &str,
main_language: &str,
blog_languages: &[String],
offline_mode: bool,
mut on_progress: impl FnMut(f32, &str),
) -> EngineResult<FillMissingTranslationsReport> {
fill_missing_translations_with(
conn,
data_dir,
project_id,
main_language,
blog_languages,
&mut |post, language| translate_post_ai(conn, offline_mode, post, language),
&mut |media, language| translate_media_ai(conn, offline_mode, media, language),
&mut on_progress,
)
}
#[expect(
clippy::too_many_arguments,
reason = "testable translation orchestration dependencies"
)]
fn fill_missing_translations_with(
conn: &Connection,
data_dir: &Path,
project_id: &str,
main_language: &str,
blog_languages: &[String],
post_translator: &mut dyn FnMut(&Post, &str) -> EngineResult<TranslationResult>,
media_translator: &mut dyn FnMut(&Media, &str) -> EngineResult<MediaTranslationResult>,
on_progress: &mut dyn FnMut(f32, &str),
) -> EngineResult<FillMissingTranslationsReport> {
let configured = configured_languages(main_language, blog_languages);
if configured.len() <= 1 {
return Ok(FillMissingTranslationsReport {
nothing_to_do: true,
..Default::default()
});
}
let posts = qp::list_posts_by_project(conn, project_id)?;
on_progress(0.0, "Scanning published posts");
let mut work = Vec::new();
for post in posts
.into_iter()
.filter(|post| post.status == PostStatus::Published && !post.do_not_translate)
{
for language in missing_languages(conn, &post, &configured)? {
work.push((post.clone(), language));
}
}
if work.is_empty() {
return Ok(FillMissingTranslationsReport {
nothing_to_do: true,
..Default::default()
});
}
let mut report = FillMissingTranslationsReport::default();
for (index, (post, language)) in work.iter().enumerate() {
on_progress(
0.15 + (index as f32 / work.len() as f32) * 0.85,
&format!("{} → {language}", post.title),
);
match translate_one_post(
conn,
data_dir,
post,
language,
true,
post_translator,
media_translator,
) {
Ok(media_count) => {
report.translated_posts += 1;
report.translated_media += media_count;
}
Err(error) => {
report.failed_count += 1;
report
.errors
.push(format!("{} ({language}): {error}", post.title));
}
}
}
on_progress(1.0, "Translation batch complete");
Ok(report)
}
/// Reactive manual-save path. Generated translations remain drafts.
pub fn translate_missing_for_post(
conn: &Connection,
data_dir: &Path,
post_id: &str,
main_language: &str,
blog_languages: &[String],
offline_mode: bool,
) -> EngineResult<FillMissingTranslationsReport> {
let post = qp::get_post_by_id(conn, post_id)?;
let configured = configured_languages(main_language, blog_languages);
let targets = missing_languages(conn, &post, &configured)?;
let mut report = FillMissingTranslationsReport {
nothing_to_do: targets.is_empty(),
..Default::default()
};
for language in targets {
let result = translate_one_post(
conn,
data_dir,
&post,
&language,
false,
&mut |post, language| translate_post_ai(conn, offline_mode, post, language),
&mut |media, language| translate_media_ai(conn, offline_mode, media, language),
);
match result {
Ok(media_count) => {
report.translated_posts += 1;
report.translated_media += media_count;
}
Err(error) => {
report.failed_count += 1;
report
.errors
.push(format!("{} ({language}): {error}", post.title));
}
}
}
Ok(report)
}
fn translate_one_post(
conn: &Connection,
data_dir: &Path,
post: &Post,
language: &str,
auto_publish: bool,
post_translator: &mut dyn FnMut(&Post, &str) -> EngineResult<TranslationResult>,
media_translator: &mut dyn FnMut(&Media, &str) -> EngineResult<MediaTranslationResult>,
) -> EngineResult<usize> {
if post.do_not_translate {
return Ok(0);
}
let body = post_body(data_dir, post)?;
if body.trim().is_empty() {
return Err(EngineError::Validation("no content to translate".into()));
}
let mut input = post.clone();
input.content = Some(body);
let translated = post_translator(&input, language)?;
let translation = crate::engine::post::upsert_translation(
conn,
data_dir,
&post.id,
language,
&translated.title,
Some(&translated.excerpt),
Some(&translated.content),
)?;
if auto_publish {
crate::engine::post::publish_post_translation(conn, data_dir, &translation.id)?;
}
let mut translated_media = 0;
for link in post_media::list_post_media_by_post(conn, &post.id)? {
let media = qm::get_media_by_id(conn, &link.media_id)?;
let source = normalize_language(media.language.as_deref().unwrap_or(""));
if source.is_empty() || source == language {
continue;
}
if qmt::get_media_translation_by_media_and_language(conn, &media.id, language).is_ok() {
continue;
}
let translated = media_translator(&media, language)?;
crate::engine::media::upsert_media_translation(
conn,
data_dir,
&media.id,
language,
Some(&translated.title),
Some(&translated.alt),
Some(&translated.caption),
)?;
translated_media += 1;
}
Ok(translated_media)
}
fn post_body(data_dir: &Path, post: &Post) -> EngineResult<String> {
if let Some(content) = &post.content {
return Ok(content.clone());
}
if post.file_path.is_empty() {
return Ok(String::new());
}
let raw = fs::read_to_string(data_dir.join(&post.file_path))?;
read_post_file(&raw)
.map(|(_, body)| body)
.map_err(EngineError::Parse)
}
fn translate_post_ai(
conn: &Connection,
offline_mode: bool,
post: &Post,
language: &str,
) -> EngineResult<TranslationResult> {
match ai::run_one_shot(
conn,
offline_mode,
&OneShotRequest {
operation: OneShotOperation::TranslatePost {
target_language: language.to_string(),
},
content: json!({
"title": post.title,
"excerpt": post.excerpt,
"content": post.content,
}),
},
)? {
OneShotResponse::Translation(result) => Ok(result),
_ => Err(EngineError::Parse(
"unexpected post translation response".into(),
)),
}
}
fn translate_media_ai(
conn: &Connection,
offline_mode: bool,
media: &Media,
language: &str,
) -> EngineResult<MediaTranslationResult> {
match ai::run_one_shot(
conn,
offline_mode,
&OneShotRequest {
operation: OneShotOperation::TranslateMedia {
target_language: language.to_string(),
},
content: json!({
"title": media.title,
"alt": media.alt,
"caption": media.caption,
}),
},
)? {
OneShotResponse::MediaTranslation(result) => Ok(result),
_ => Err(EngineError::Parse(
"unexpected media translation response".into(),
)),
}
}
fn normalize_language(language: &str) -> String {
language
.split(['-', '_'])
.next()
.unwrap_or("")
.trim()
.to_lowercase()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::db::fts::ensure_fts_tables;
use crate::db::queries::project::{insert_project, make_test_project};
use crate::engine::post::{create_post, publish_post};
use tempfile::TempDir;
#[test]
fn batch_translates_only_missing_languages_and_publishes() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
ensure_fts_tables(db.conn()).unwrap();
insert_project(db.conn(), &make_test_project("p1", "blog")).unwrap();
let dir = TempDir::new().unwrap();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Hello",
Some("Body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
publish_post(db.conn(), dir.path(), &post.id).unwrap();
let mut requested = Vec::new();
let report = fill_missing_translations_with(
db.conn(),
dir.path(),
"p1",
"en",
&["de".into(), "fr".into(), "de-DE".into()],
&mut |_post, language| {
requested.push(language.to_string());
Ok(TranslationResult {
title: format!("Title {language}"),
excerpt: format!("Excerpt {language}"),
content: format!("Body {language}"),
})
},
&mut |_media, _language| unreachable!(),
&mut |_, _| {},
)
.unwrap();
assert_eq!(requested, vec!["de", "fr"]);
assert_eq!(report.translated_posts, 2);
for language in ["de", "fr"] {
let translation = post_translation::get_post_translation_by_post_and_language(
db.conn(),
&post.id,
language,
)
.unwrap();
assert_eq!(translation.status, PostStatus::Published);
assert!(dir.path().join(&translation.file_path).is_file());
assert!(translation.content.is_none());
}
}
#[test]
fn skips_do_not_translate_posts() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
ensure_fts_tables(db.conn()).unwrap();
insert_project(db.conn(), &make_test_project("p1", "blog")).unwrap();
let dir = TempDir::new().unwrap();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Private",
Some("Body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
let post = crate::engine::post::update_post(
db.conn(),
dir.path(),
&post.id,
None,
None,
None,
None,
None,
None,
None,
None,
None,
Some(true),
)
.unwrap();
publish_post(db.conn(), dir.path(), &post.id).unwrap();
let report = fill_missing_translations_with(
db.conn(),
dir.path(),
"p1",
"en",
&["de".into()],
&mut |_, _| panic!("translator must not run"),
&mut |_, _| panic!("translator must not run"),
&mut |_, _| {},
)
.unwrap();
assert!(report.nothing_to_do);
}
}

343
crates/bds-core/src/engine/blogmark.rs Normal file
View File

@@ -0,0 +1,343 @@
use std::fs;
use std::path::Path;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use url::Url;
use crate::db::DbConnection as Connection;
use crate::db::queries::script as script_queries;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Post, Script, ScriptKind};
use crate::scripting::{self, ExecutionControl, ExecutionKind};
const MAX_TITLE_LENGTH: usize = 200;
const MAX_URL_LENGTH: usize = 2_048;
const MAX_TOASTS_TOTAL: usize = 20;
const MAX_TOAST_LENGTH: usize = 300;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BlogmarkCandidate {
pub title: String,
pub url: Option<String>,
pub content: Option<String>,
pub tags: Vec<String>,
pub categories: Vec<String>,
pub project_id: Option<String>,
}
#[derive(Debug, Clone)]
pub struct BlogmarkImportResult {
pub post: Post,
pub toasts: Vec<String>,
pub transform_errors: Vec<String>,
}
pub fn parse_deep_link(raw: &str) -> EngineResult<BlogmarkCandidate> {
let parsed =
Url::parse(raw).map_err(|_| EngineError::Validation("invalid blogmark URL".into()))?;
if parsed.scheme() != "bds2" {
return Err(EngineError::Validation(
"unsupported blogmark scheme".into(),
));
}
if parsed.host_str() != Some("new-post") {
return Err(EngineError::Validation(
"unsupported blogmark action".into(),
));
}
let params = parsed
.query_pairs()
.into_owned()
.collect::<std::collections::HashMap<_, _>>();
let url = params.get("url").and_then(|value| sanitize_http_url(value));
let fallback = url
.as_ref()
.and_then(|value| Url::parse(value).ok())
.and_then(|value| value.host_str().map(str::to_string));
let title = sanitize_title(params.get("title").map(String::as_str), fallback.as_deref());
Ok(BlogmarkCandidate {
title,
url,
content: nonempty(params.get("content")),
tags: list_param(params.get("tags")),
categories: list_param(params.get("categories")),
project_id: nonempty(params.get("project_id")),
})
}
pub fn receive_deep_link(
conn: &Connection,
data_dir: &Path,
project_id: &str,
raw: &str,
) -> EngineResult<BlogmarkImportResult> {
let mut candidate = parse_deep_link(raw)?;
if let Some(target) = &candidate.project_id
&& target != project_id
{
return Err(EngineError::Validation(
"blogmark targets a different project".into(),
));
}
if candidate.content.is_none()
&& let Some(url) = &candidate.url
{
candidate.content = Some(format!(
"[{}]({url})",
escape_markdown_link_text(&candidate.title)
));
}
let (mut candidate, toasts, transform_errors) =
run_transforms(conn, data_dir, project_id, candidate)?;
if candidate.categories.is_empty() {
let metadata = crate::engine::meta::read_project_json(data_dir)?;
if let Some(category) = metadata
.blogmark_category
.filter(|value| !value.trim().is_empty())
{
candidate.categories.push(category);
}
}
let metadata = crate::engine::meta::read_project_json(data_dir)?;
let post = crate::engine::post::create_post(
conn,
data_dir,
project_id,
&candidate.title,
candidate.content.as_deref(),
candidate.tags,
candidate.categories,
metadata.default_author.as_deref(),
metadata.main_language.as_deref(),
None,
)?;
Ok(BlogmarkImportResult {
post,
toasts,
transform_errors,
})
}
fn run_transforms(
conn: &Connection,
data_dir: &Path,
project_id: &str,
candidate: BlogmarkCandidate,
) -> EngineResult<(BlogmarkCandidate, Vec<String>, Vec<String>)> {
let mut transforms = script_queries::list_scripts_by_project(conn, project_id)?
.into_iter()
.filter(|script| script.kind == ScriptKind::Transform && script.enabled)
.collect::<Vec<_>>();
transforms.sort_by(|left, right| {
left.updated_at
.cmp(&right.updated_at)
.then_with(|| left.slug.cmp(&right.slug))
.then_with(|| left.id.cmp(&right.id))
});
let mut current =
serde_json::to_value(candidate).map_err(|error| EngineError::Parse(error.to_string()))?;
let mut toasts = Vec::new();
let mut errors = Vec::new();
for script in transforms {
if script.entrypoint.trim().is_empty() {
continue;
}
let source = resolved_script_content(data_dir, &script)?;
let context = json!({
"source": "blogmark",
"url": current.get("url").cloned().unwrap_or(Value::Null),
});
match scripting::execute_many(
&source,
&script.entrypoint,
&[current.clone(), context],
ExecutionKind::Transform,
&ExecutionControl::default(),
) {
Ok(execution) => {
let (next, returned_toasts) = split_transform_result(execution.value, &current);
current = next;
accept_toasts(
&mut toasts,
execution.toasts.into_iter().chain(returned_toasts),
);
}
Err(error) => errors.push(format!("{}: {error}", script.slug)),
}
}
let candidate = serde_json::from_value(current).map_err(|error| {
EngineError::Validation(format!("transform returned invalid post data: {error}"))
})?;
Ok((candidate, toasts, errors))
}
fn split_transform_result(value: Value, previous: &Value) -> (Value, Vec<String>) {
if let Some(object) = value.as_object() {
if let Some(data) = object.get("data").filter(|value| value.is_object()) {
let toasts = object
.get("toasts")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.map(str::to_string)
.collect();
return (data.clone(), toasts);
}
return (value, Vec::new());
}
(previous.clone(), Vec::new())
}
fn accept_toasts(target: &mut Vec<String>, source: impl IntoIterator<Item = String>) {
for message in source.into_iter().take(5) {
if target.len() >= MAX_TOASTS_TOTAL {
break;
}
target.push(message.chars().take(MAX_TOAST_LENGTH).collect());
}
}
fn resolved_script_content(data_dir: &Path, script: &Script) -> EngineResult<String> {
if let Some(content) = &script.content {
return Ok(content.clone());
}
let raw = fs::read_to_string(data_dir.join(&script.file_path))?;
crate::util::frontmatter::read_script_file(&raw)
.map(|(_, body)| body)
.map_err(EngineError::Parse)
}
fn sanitize_title(value: Option<&str>, fallback: Option<&str>) -> String {
let cleaned = value
.unwrap_or_default()
.chars()
.filter(|ch| !ch.is_control())
.collect::<String>();
let cleaned = cleaned.trim();
let selected = if cleaned.is_empty() {
fallback.unwrap_or_default()
} else {
cleaned
};
selected.chars().take(MAX_TITLE_LENGTH).collect()
}
fn sanitize_http_url(value: &str) -> Option<String> {
let cleaned = value
.chars()
.filter(|ch| !ch.is_control())
.collect::<String>();
let mut parsed = Url::parse(cleaned.trim()).ok()?;
if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() {
return None;
}
parsed.set_fragment(None);
parsed.set_username("").ok()?;
parsed.set_password(None).ok()?;
let normalized = parsed.to_string();
(normalized.chars().count() <= MAX_URL_LENGTH).then_some(normalized)
}
fn nonempty(value: Option<&String>) -> Option<String> {
value
.map(|value| value.trim())
.filter(|value| !value.is_empty())
.map(str::to_string)
}
fn list_param(value: Option<&String>) -> Vec<String> {
value
.map(|value| {
value
.split(',')
.map(str::trim)
.filter(|item| !item.is_empty())
.map(str::to_string)
.collect()
})
.unwrap_or_default()
}
fn escape_markdown_link_text(value: &str) -> String {
value
.trim()
.replace('\\', "\\\\")
.replace('[', "\\[")
.replace(']', "\\]")
.replace('(', "\\(")
.replace(')', "\\)")
.replace(['\r', '\n'], " ")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db::Database;
use crate::model::ScriptKind;
use tempfile::TempDir;
#[test]
fn parses_and_hardens_blogmark_links() {
let candidate = parse_deep_link(
"bds2://new-post?title=%00Hello&url=https%3A%2F%2Fuser%3Apass%40example.com%2Fa%23frag&tags=one%2Ctwo",
)
.unwrap();
assert_eq!(candidate.title, "Hello");
assert_eq!(candidate.url.as_deref(), Some("https://example.com/a"));
assert_eq!(candidate.tags, vec!["one", "two"]);
}
#[test]
fn rejects_other_schemes_and_actions() {
assert!(parse_deep_link("bds://new-post?title=x").is_err());
assert!(parse_deep_link("bds2://other?title=x").is_err());
}
#[test]
fn invalid_source_url_never_reaches_candidate() {
let candidate =
parse_deep_link("bds2://new-post?title=x&url=javascript%3Aalert%281%29").unwrap();
assert!(candidate.url.is_none());
}
#[test]
fn imports_draft_after_ordered_transform_pipeline() {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
crate::db::fts::ensure_fts_tables(db.conn()).unwrap();
let directory = TempDir::new().unwrap();
let project = crate::engine::project::create_project(
db.conn(),
"Blog",
Some(directory.path().to_str().unwrap()),
)
.unwrap();
crate::engine::script::create_script(
db.conn(),
&project.id,
"Add tag",
ScriptKind::Transform,
"function main(data, context) data.tags = {context.source}; bds.app.toast('done'); return data end",
None,
)
.unwrap();
let result = receive_deep_link(
db.conn(),
directory.path(),
&project.id,
"bds2://new-post?title=Example&url=https%3A%2F%2Fexample.com",
)
.unwrap();
assert_eq!(result.post.tags, vec!["blogmark"]);
assert_eq!(
result.post.content.as_deref(),
Some("[Example](https://example.com/)")
);
assert_eq!(result.toasts, vec!["done"]);
assert!(result.transform_errors.is_empty());
}
}

24
crates/bds-core/src/engine/generation.rs
View File

@@ -72,6 +72,26 @@ pub fn generate_starter_site(
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
_language: &str,
) -> EngineResult<GenerationReport> {
generate_starter_site_with_progress(
conn,
output_dir,
project_id,
metadata,
posts,
_language,
|_current, _total, _path| {},
)
}
pub fn generate_starter_site_with_progress(
conn: &Connection,
output_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
posts: &[PublishedPostSource],
_language: &str,
mut on_page: impl FnMut(usize, usize, &str),
) -> EngineResult<GenerationReport> {
let mut report = GenerationReport::default();
let data_dir = project_data_dir(output_dir);
@@ -85,7 +105,8 @@ pub fn generate_starter_site(
build_site_render_artifacts(conn, &data_dir, project_id, metadata, &input_posts)
.map_err(|error| EngineError::Parse(error.to_string()))?;
for page in &artifacts.pages {
let total_pages = artifacts.pages.len();
for (index, page) in artifacts.pages.iter().enumerate() {
write_out(
conn,
output_dir,
@@ -94,6 +115,7 @@ pub fn generate_starter_site(
&page.html,
&mut report,
)?;
on_page(index + 1, total_pages, &page.url_path);
}
write_bundled_site_assets(conn, output_dir, project_id, &mut report)?;

209
crates/bds-core/src/engine/media.rs
View File

@@ -198,6 +198,128 @@ pub fn update_media(
Ok(media)
}
/// Replace a media item's binary while preserving its identity, public path,
/// metadata, translations, and post links. Returns `None` when the new file is
/// byte-for-byte identical to the stored binary.
pub fn replace_media_file(
conn: &Connection,
data_dir: &Path,
media_id: &str,
new_source_path: &Path,
) -> EngineResult<Option<Media>> {
let mut media = qm::get_media_by_id(conn, media_id)
.map_err(|_| EngineError::NotFound(format!("media {media_id}")))?;
if !new_source_path.is_file() {
return Err(EngineError::Validation(format!(
"replacement file does not exist: {}",
new_source_path.display()
)));
}
let replacement_checksum = crate::util::file_hash(new_source_path)?;
if media.checksum.as_deref() == Some(replacement_checksum.as_str()) {
return Ok(None);
}
let destination = data_dir.join(&media.file_path);
if !destination.is_file() {
return Err(EngineError::NotFound(format!(
"stored media file {}",
destination.display()
)));
}
let (width, height) = image_dimensions(new_source_path)
.map(|(w, h)| (Some(w as i32), Some(h as i32)))
.map_err(|error| {
EngineError::Validation(format!(
"replacement is not a readable image '{}': {error}",
new_source_path.display(),
))
})?;
let replacement_size = fs::metadata(new_source_path)?.len() as i64;
// Generate all derived files before touching the canonical binary.
let staged_thumbnails = data_dir
.join("thumbnails")
.join(format!(".replace-{media_id}"));
if staged_thumbnails.exists() {
fs::remove_dir_all(&staged_thumbnails)?;
}
let staged_paths = generate_all_thumbnails(new_source_path, &staged_thumbnails, media_id)
.map_err(EngineError::Parse)?;
let backup = destination.with_extension(format!(
"{}.bak",
destination
.extension()
.and_then(|extension| extension.to_str())
.unwrap_or("media")
));
if backup.exists() {
fs::remove_file(&backup)?;
}
fs::rename(&destination, &backup)?;
if let Err(error) = fs::copy(new_source_path, &destination) {
let _ = fs::rename(&backup, &destination);
let _ = fs::remove_dir_all(&staged_thumbnails);
return Err(error.into());
}
let previous_media = media.clone();
let previous_sidecar = fs::read_to_string(data_dir.join(&media.sidecar_path)).ok();
media.size = replacement_size;
media.width = width;
media.height = height;
media.checksum = Some(replacement_checksum);
media.updated_at = now_unix_ms();
let apply_result = (|| -> EngineResult<()> {
conn.begin_savepoint()?;
qm::update_media(conn, &media)?;
let linked = qpm::list_post_media_by_media(conn, media_id).unwrap_or_default();
let linked_post_ids: Vec<String> = linked.iter().map(|item| item.post_id.clone()).collect();
let sidecar = MediaSidecar::from_media(&media, &linked_post_ids);
atomic_write_str(&data_dir.join(&media.sidecar_path), &sidecar.to_string())?;
fts_index_media(conn, &media)?;
for staged in &staged_paths {
let staged = Path::new(staged);
let relative = staged.strip_prefix(&staged_thumbnails).map_err(|error| {
EngineError::Validation(format!("invalid staged thumbnail path: {error}"))
})?;
let target = data_dir.join("thumbnails").join(relative);
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
fs::copy(staged, target)?;
}
conn.release_savepoint()?;
Ok(())
})();
let _ = fs::remove_dir_all(&staged_thumbnails);
match apply_result {
Ok(()) => {
fs::remove_file(backup)?;
Ok(Some(media))
}
Err(error) => {
let _ = conn.rollback_savepoint();
let _ = fs::remove_file(&destination);
let _ = fs::rename(&backup, &destination);
let _ = qm::update_media(conn, &previous_media);
if let Some(previous_sidecar) = previous_sidecar {
let _ = atomic_write_str(
&data_dir.join(&previous_media.sidecar_path),
&previous_sidecar,
);
}
Err(error)
}
}
}
/// Delete a media item and all related artifacts.
pub fn delete_media(conn: &Connection, data_dir: &Path, media_id: &str) -> EngineResult<()> {
let media = qm::get_media_by_id(conn, media_id)?;
@@ -499,7 +621,7 @@ fn is_translation_sidecar(file_name: &str) -> bool {
}
/// Rebuild a canonical media from its sidecar file. Returns true if created, false if updated.
fn rebuild_canonical_media(
pub(crate) fn rebuild_canonical_media(
conn: &Connection,
data_dir: &Path,
project_id: &str,
@@ -612,7 +734,7 @@ fn rebuild_canonical_media(
}
/// Rebuild a translation from a `*.{lang}.meta` sidecar. Returns true if created, false if updated.
fn rebuild_translation_sidecar(
pub(crate) fn rebuild_translation_sidecar(
conn: &Connection,
_data_dir: &Path,
project_id: &str,
@@ -800,6 +922,89 @@ mod tests {
assert_eq!(sc.tags, vec!["updated-tag"]);
}
#[test]
fn replace_media_file_preserves_identity_and_regenerates_artifacts() {
let (db, dir) = setup();
let source = create_test_png(dir.path());
let media = import_media(
db.conn(),
dir.path(),
"p1",
&source,
"photo.png",
Some("Kept title"),
None,
None,
None,
None,
vec!["kept".into()],
)
.unwrap();
let replacement = dir.path().join("replacement.png");
DynamicImage::new_rgb8(320, 180).save(&replacement).unwrap();
let old_checksum = media.checksum.clone();
let updated = replace_media_file(db.conn(), dir.path(), &media.id, &replacement)
.unwrap()
.unwrap();
assert_eq!(updated.id, media.id);
assert_eq!(updated.file_path, media.file_path);
assert_eq!(updated.original_name, media.original_name);
assert_eq!(updated.title, media.title);
assert_eq!(updated.tags, media.tags);
assert_eq!((updated.width, updated.height), (Some(320), Some(180)));
assert_ne!(updated.checksum, old_checksum);
assert!(!dir.path().join(format!("{}.bak", media.file_path)).exists());
let sidecar =
read_sidecar(&fs::read_to_string(dir.path().join(&updated.sidecar_path)).unwrap())
.unwrap();
assert_eq!((sidecar.width, sidecar.height), (Some(320), Some(180)));
let prefix = &media.id[..2];
let ai_thumb = dir
.path()
.join("thumbnails")
.join(prefix)
.join(format!("{}-ai.jpg", media.id));
assert!(ai_thumb.is_file());
}
#[test]
fn replace_media_file_is_noop_for_identical_content() {
let (db, dir) = setup();
let source = create_test_png(dir.path());
let media = import_media(
db.conn(),
dir.path(),
"p1",
&source,
"photo.png",
None,
None,
None,
None,
None,
vec![],
)
.unwrap();
let stored = dir.path().join(&media.file_path);
let updated_at = media.updated_at;
assert!(
replace_media_file(db.conn(), dir.path(), &media.id, &stored)
.unwrap()
.is_none()
);
assert_eq!(
qm::get_media_by_id(db.conn(), &media.id)
.unwrap()
.updated_at,
updated_at
);
}
#[test]
fn delete_media_removes_everything() {
let (db, dir) = setup();

14
crates/bds-core/src/engine/meta.rs
View File

@@ -177,6 +177,18 @@ pub fn update_project_metadata(data_dir: &Path, changes: &serde_json::Value) ->
if let Some(max) = changes.get("maxPostsPerPage").and_then(|v| v.as_i64()) {
meta.max_posts_per_page = max as i32;
}
if let Some(concurrency) = changes.get("imageImportConcurrency") {
meta.image_import_concurrency = concurrency
.as_i64()
.map(|value| value as i32)
.or_else(|| {
concurrency
.as_str()
.and_then(|value| value.parse::<i32>().ok())
})
.unwrap_or(4)
.clamp(1, 8);
}
if let Some(cat) = changes.get("blogmarkCategory") {
meta.blogmark_category = cat.as_str().map(|s| s.to_string());
}
@@ -218,6 +230,7 @@ pub fn startup_sync(data_dir: &Path) -> EngineResult<()> {
main_language: None,
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
@@ -286,6 +299,7 @@ mod tests {
main_language: Some("en".into()),
default_author: None,
max_posts_per_page: 25,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,

439
crates/bds-core/src/engine/metadata_diff.rs
View File

@@ -7,16 +7,23 @@ use walkdir::WalkDir;
use crate::db::from_row::{script_kind_to_str, template_kind_to_str};
use crate::db::queries::media as qm;
use crate::db::queries::media_translation as qmt;
use crate::db::queries::post as qp;
use crate::db::queries::post_translation as qt;
use crate::db::queries::project as qproject;
use crate::db::queries::script as qs;
use crate::db::queries::template as qtpl;
use crate::engine::{EngineError, EngineResult};
use crate::model::{Media, Post, PostStatus, PostTranslation, Script, Template};
use crate::model::{Media, MediaTranslation, Post, PostStatus, PostTranslation, Script, Template};
use crate::util::frontmatter::{
read_post_file, read_script_file, read_template_file, read_translation_file,
ScriptFrontmatter, TemplateFrontmatter, read_post_file, read_script_file, read_template_file,
read_translation_file, write_post_file, write_script_file, write_template_file,
write_translation_file,
};
use crate::util::sidecar::read_sidecar;
use crate::util::sidecar::{
MediaSidecar, MediaTranslationSidecar, read_sidecar, read_translation_sidecar,
};
use crate::util::{atomic_write_str, media_translation_sidecar_path};
/// A single field difference.
#[derive(Debug, Clone)]
@@ -43,13 +50,19 @@ pub struct OrphanFile {
}
/// Complete diff report.
#[derive(Debug, Default)]
#[derive(Debug, Clone, Default)]
pub struct DiffReport {
pub diffs: Vec<EntityDiff>,
pub orphans: Vec<OrphanFile>,
pub errors: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RepairDirection {
FileToDatabase,
DatabaseToFile,
}
/// Compare DB state vs filesystem files and report all differences.
///
/// This function does NOT modify anything -- it only reports differences.
@@ -60,6 +73,14 @@ pub fn compute_metadata_diff(
) -> EngineResult<DiffReport> {
let mut report = DiffReport::default();
if let Ok(project) = qproject::get_project_by_id(conn, project_id) {
match diff_project(data_dir, &project) {
Ok(Some(diff)) => report.diffs.push(diff),
Ok(None) => {}
Err(error) => report.errors.push(format!("project {project_id}: {error}")),
}
}
// 1. Diff posts
let posts = qp::list_posts_by_project(conn, project_id)?;
for post in &posts {
@@ -94,6 +115,17 @@ pub fn compute_metadata_diff(
if m.sidecar_path.is_empty() {
continue;
}
let translations = qmt::list_media_translations_by_media(conn, &m.id)?;
for translation in &translations {
match diff_media_translation(data_dir, m, translation) {
Ok(Some(diff)) => report.diffs.push(diff),
Ok(None) => {}
Err(error) => report
.errors
.push(format!("media translation {}: {error}", translation.id)),
}
}
match diff_media(data_dir, m) {
Ok(Some(d)) => report.diffs.push(d),
Ok(None) => {}
@@ -134,6 +166,255 @@ pub fn compute_metadata_diff(
Ok(report)
}
/// Resolve one reported difference in the selected direction.
pub fn repair_metadata_diff_item(
conn: &Connection,
data_dir: &Path,
project_id: &str,
direction: RepairDirection,
item: &EntityDiff,
) -> EngineResult<()> {
match direction {
RepairDirection::FileToDatabase => {
let path = data_dir.join(&item.file_path);
match item.entity_type.as_str() {
"project" => sync_project_from_file(conn, data_dir, project_id)?,
"post" => {
crate::engine::post::rebuild_canonical_post(conn, data_dir, project_id, &path)?;
}
"post_translation" => {
crate::engine::post::rebuild_translation(conn, data_dir, project_id, &path)?;
}
"media" => {
crate::engine::media::rebuild_canonical_media(
conn, data_dir, project_id, &path,
)?;
}
"media_translation" => {
crate::engine::media::rebuild_translation_sidecar(
conn, data_dir, project_id, &path,
)?;
}
"script" => {
crate::engine::script_rebuild::rebuild_single_script(
conn, data_dir, project_id, &path,
)?;
}
"template" => {
crate::engine::template_rebuild::rebuild_single_template(
conn, data_dir, project_id, &path,
)?;
}
other => return unsupported_repair(other),
}
}
RepairDirection::DatabaseToFile => match item.entity_type.as_str() {
"project" => rewrite_project_from_database(conn, data_dir, project_id)?,
"post" => rewrite_post_from_database(conn, data_dir, &item.entity_id)?,
"post_translation" => {
rewrite_post_translation_from_database(conn, data_dir, &item.entity_id)?
}
"media" => rewrite_media_from_database(conn, data_dir, &item.entity_id)?,
"media_translation" => rewrite_media_translation_from_database(
conn,
data_dir,
project_id,
&item.entity_id,
)?,
"script" => rewrite_script_from_database(conn, data_dir, &item.entity_id)?,
"template" => rewrite_template_from_database(conn, data_dir, &item.entity_id)?,
other => return unsupported_repair(other),
},
}
Ok(())
}
fn diff_project(
data_dir: &Path,
project: &crate::model::Project,
) -> EngineResult<Option<EntityDiff>> {
let metadata = crate::engine::meta::read_project_json(data_dir)?;
let mut fields = Vec::new();
compare_field(&mut fields, "name", &project.name, &metadata.name);
compare_field(
&mut fields,
"description",
project.description.as_deref().unwrap_or(""),
metadata.description.as_deref().unwrap_or(""),
);
Ok((!fields.is_empty()).then_some(EntityDiff {
entity_type: "project".into(),
entity_id: project.id.clone(),
file_path: "meta/project.json".into(),
fields,
}))
}
fn sync_project_from_file(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
let metadata = crate::engine::meta::read_project_json(data_dir)?;
let mut project = qproject::get_project_by_id(conn, project_id)?;
project.name = metadata.name;
project.description = metadata.description;
project.updated_at = crate::util::now_unix_ms();
qproject::update_project(conn, &project)?;
Ok(())
}
fn rewrite_project_from_database(
conn: &Connection,
data_dir: &Path,
project_id: &str,
) -> EngineResult<()> {
let project = qproject::get_project_by_id(conn, project_id)?;
let mut metadata = crate::engine::meta::read_project_json(data_dir)?;
metadata.name = project.name;
metadata.description = project.description;
crate::engine::meta::write_project_json(data_dir, &metadata)
}
fn unsupported_repair<T>(entity_type: &str) -> EngineResult<T> {
Err(EngineError::Validation(format!(
"unsupported metadata diff entity type: {entity_type}"
)))
}
fn rewrite_post_from_database(
conn: &Connection,
data_dir: &Path,
entity_id: &str,
) -> EngineResult<()> {
let post = qp::get_post_by_id(conn, entity_id)?;
let path = data_dir.join(&post.file_path);
let body = fs::read_to_string(&path)
.ok()
.and_then(|content| read_post_file(&content).ok().map(|(_, body)| body))
.unwrap_or_default();
atomic_write_str(&path, &write_post_file(&post, &body))?;
Ok(())
}
fn rewrite_post_translation_from_database(
conn: &Connection,
data_dir: &Path,
entity_id: &str,
) -> EngineResult<()> {
let translation = qt::get_post_translation_by_id(conn, entity_id)?;
let path = data_dir.join(&translation.file_path);
let body = fs::read_to_string(&path)
.ok()
.and_then(|content| read_translation_file(&content).ok().map(|(_, body)| body))
.unwrap_or_default();
atomic_write_str(&path, &write_translation_file(&translation, &body))?;
Ok(())
}
fn rewrite_media_from_database(
conn: &Connection,
data_dir: &Path,
entity_id: &str,
) -> EngineResult<()> {
let media = qm::get_media_by_id(conn, entity_id)?;
let linked = crate::db::queries::post_media::list_post_media_by_media(conn, entity_id)
.unwrap_or_default();
let post_ids = linked
.into_iter()
.map(|link| link.post_id)
.collect::<Vec<_>>();
atomic_write_str(
&data_dir.join(&media.sidecar_path),
&MediaSidecar::from_media(&media, &post_ids).to_string(),
)?;
Ok(())
}
fn rewrite_media_translation_from_database(
conn: &Connection,
data_dir: &Path,
project_id: &str,
entity_id: &str,
) -> EngineResult<()> {
let media_items = qm::list_media_by_project(conn, project_id)?;
for media in media_items {
if let Some(translation) = qmt::list_media_translations_by_media(conn, &media.id)?
.into_iter()
.find(|translation| translation.id == entity_id)
{
let sidecar = MediaTranslationSidecar {
translation_for: translation.translation_for,
language: translation.language.clone(),
title: translation.title,
alt: translation.alt,
caption: translation.caption,
};
let path = media_translation_sidecar_path(&media.file_path, &translation.language);
atomic_write_str(&data_dir.join(path), &sidecar.to_string())?;
return Ok(());
}
}
Err(EngineError::NotFound(format!(
"media translation {entity_id}"
)))
}
fn rewrite_script_from_database(
conn: &Connection,
data_dir: &Path,
entity_id: &str,
) -> EngineResult<()> {
let script = qs::get_script_by_id(conn, entity_id)?;
let path = data_dir.join(&script.file_path);
let body = fs::read_to_string(&path)
.ok()
.and_then(|content| read_script_file(&content).ok().map(|(_, body)| body))
.or(script.content.clone())
.unwrap_or_default();
let frontmatter = ScriptFrontmatter {
id: script.id,
project_id: Some(script.project_id),
slug: script.slug,
title: script.title,
kind: script_kind_to_str(&script.kind).to_owned(),
entrypoint: script.entrypoint,
enabled: script.enabled,
version: script.version,
created_at: script.created_at,
updated_at: script.updated_at,
};
atomic_write_str(&path, &write_script_file(&frontmatter, &body))?;
Ok(())
}
fn rewrite_template_from_database(
conn: &Connection,
data_dir: &Path,
entity_id: &str,
) -> EngineResult<()> {
let template = qtpl::get_template_by_id(conn, entity_id)?;
let path = data_dir.join(&template.file_path);
let body = fs::read_to_string(&path)
.ok()
.and_then(|content| read_template_file(&content).ok().map(|(_, body)| body))
.or(template.content.clone())
.unwrap_or_default();
let frontmatter = TemplateFrontmatter {
id: template.id,
project_id: Some(template.project_id),
slug: template.slug,
title: template.title,
kind: template_kind_to_str(&template.kind).to_owned(),
enabled: template.enabled,
version: template.version,
created_at: template.created_at,
updated_at: template.updated_at,
};
atomic_write_str(&path, &write_template_file(&frontmatter, &body))?;
Ok(())
}
// --- Internal helpers ---
fn opt_to_str(opt: &Option<String>) -> String {
@@ -361,6 +642,58 @@ fn diff_media(data_dir: &Path, media: &Media) -> EngineResult<Option<EntityDiff>
}
}
fn diff_media_translation(
data_dir: &Path,
media: &Media,
translation: &MediaTranslation,
) -> EngineResult<Option<EntityDiff>> {
let relative_path = media_translation_sidecar_path(&media.file_path, &translation.language);
let absolute_path = data_dir.join(&relative_path);
if !absolute_path.is_file() {
return Ok(None);
}
let sidecar = read_translation_sidecar(&fs::read_to_string(&absolute_path)?)
.map_err(EngineError::Parse)?;
let mut fields = Vec::new();
compare_field(
&mut fields,
"translationFor",
&translation.translation_for,
&sidecar.translation_for,
);
compare_field(
&mut fields,
"language",
&translation.language,
&sidecar.language,
);
compare_field(
&mut fields,
"title",
&opt_to_str(&translation.title),
&opt_to_str(&sidecar.title),
);
compare_field(
&mut fields,
"alt",
&opt_to_str(&translation.alt),
&opt_to_str(&sidecar.alt),
);
compare_field(
&mut fields,
"caption",
&opt_to_str(&translation.caption),
&opt_to_str(&sidecar.caption),
);
Ok((!fields.is_empty()).then_some(EntityDiff {
entity_type: "media_translation".into(),
entity_id: translation.id.clone(),
file_path: relative_path,
fields,
}))
}
fn diff_template(data_dir: &Path, tpl: &Template) -> EngineResult<Option<EntityDiff>> {
let abs_path = data_dir.join(&tpl.file_path);
if !abs_path.exists() {
@@ -484,6 +817,12 @@ fn detect_orphan_files(
if !m.sidecar_path.is_empty() {
db_file_paths.insert(m.sidecar_path.clone());
}
for translation in qmt::list_media_translations_by_media(conn, &m.id)? {
db_file_paths.insert(media_translation_sidecar_path(
&m.file_path,
&translation.language,
));
}
}
let templates = qtpl::list_templates_by_project(conn, project_id)?;
@@ -578,6 +917,15 @@ fn detect_orphan_files(
});
}
}
for translation in qmt::list_media_translations_by_media(conn, &m.id)? {
let path = media_translation_sidecar_path(&m.file_path, &translation.language);
if !data_dir.join(&path).is_file() {
orphans.push(OrphanFile {
file_path: path,
reason: "db_entry_without_file".to_string(),
});
}
}
}
for t in &templates {
@@ -719,6 +1067,89 @@ mod tests {
assert_eq!(title_diffs[0].file_value, "Tampered Title");
}
#[test]
fn repairs_one_post_diff_from_file_to_database() {
let (db, dir) = setup();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Database Title",
Some("body"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
let published = publish_post(db.conn(), dir.path(), &post.id).unwrap();
let path = dir.path().join(&published.file_path);
let content = fs::read_to_string(&path).unwrap();
fs::write(&path, content.replace("Database Title", "Filesystem Title")).unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
let item = report
.diffs
.iter()
.find(|item| item.entity_type == "post")
.unwrap();
repair_metadata_diff_item(
db.conn(),
dir.path(),
"p1",
RepairDirection::FileToDatabase,
item,
)
.unwrap();
assert_eq!(
qp::get_post_by_id(db.conn(), &post.id).unwrap().title,
"Filesystem Title"
);
}
#[test]
fn repairs_one_post_diff_from_database_to_file_without_losing_body() {
let (db, dir) = setup();
let post = create_post(
db.conn(),
dir.path(),
"p1",
"Database Title",
Some("body that must survive"),
vec![],
vec![],
None,
Some("en"),
None,
)
.unwrap();
let published = publish_post(db.conn(), dir.path(), &post.id).unwrap();
let path = dir.path().join(&published.file_path);
let content = fs::read_to_string(&path).unwrap();
fs::write(&path, content.replace("Database Title", "Filesystem Title")).unwrap();
let report = compute_metadata_diff(db.conn(), dir.path(), "p1").unwrap();
let item = report
.diffs
.iter()
.find(|item| item.entity_type == "post")
.unwrap();
repair_metadata_diff_item(
db.conn(),
dir.path(),
"p1",
RepairDirection::DatabaseToFile,
item,
)
.unwrap();
let repaired = fs::read_to_string(path).unwrap();
assert!(repaired.contains("title: Database Title"));
assert!(repaired.contains("body that must survive"));
}
#[test]
fn detects_media_sidecar_drift() {
let (db, dir) = setup();

4
crates/bds-core/src/engine/mod.rs
View File

@@ -1,4 +1,6 @@
pub mod ai;
pub mod auto_translation;
pub mod blogmark;
pub mod calendar;
pub mod context;
pub mod error;
@@ -11,6 +13,7 @@ pub mod post;
pub mod post_media;
pub mod preview;
pub mod project;
pub mod publishing;
pub mod rebuild;
pub mod script;
pub mod script_rebuild;
@@ -20,7 +23,6 @@ pub mod tag;
pub mod task;
pub mod template;
pub mod template_rebuild;
pub mod validate_content;
pub mod validate_media;
pub mod validate_site;
pub mod validate_translations;

28
crates/bds-core/src/engine/post.rs
View File

@@ -662,6 +662,30 @@ pub fn delete_translation(
Ok(())
}
/// Publish one draft translation without republishing its canonical post.
pub fn publish_post_translation(
conn: &Connection,
data_dir: &Path,
translation_id: &str,
) -> EngineResult<PostTranslation> {
let mut translation = qt::get_post_translation_by_id(conn, translation_id)?;
if translation.status == PostStatus::Published {
return Ok(translation);
}
let post = qp::get_post_by_id(conn, &translation.translation_for)?;
conn.begin_savepoint()?;
match publish_translation(conn, data_dir, &mut translation, &post) {
Ok(()) => {
conn.release_savepoint()?;
Ok(translation)
}
Err(error) => {
let _ = conn.rollback_savepoint();
Err(error)
}
}
}
/// Rebuild posts from filesystem. Walk posts/ dir, parse .md files, upsert into DB.
pub fn rebuild_posts_from_filesystem(
conn: &Connection,
@@ -943,7 +967,7 @@ fn is_translation_filename(stem: &str) -> bool {
}
/// Rebuild a canonical post from a .md file. Returns true if created, false if updated.
fn rebuild_canonical_post(
pub(crate) fn rebuild_canonical_post(
conn: &Connection,
data_dir: &Path,
project_id: &str,
@@ -1033,7 +1057,7 @@ fn rebuild_canonical_post(
}
/// Rebuild a translation from a .{lang}.md file. Returns true if created, false if updated.
fn rebuild_translation(
pub(crate) fn rebuild_translation(
conn: &Connection,
data_dir: &Path,
project_id: &str,

1
crates/bds-core/src/engine/preview.rs
View File

@@ -480,6 +480,7 @@ mod tests {
main_language: Some("en".into()),
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,

204
crates/bds-core/src/engine/project.rs
View File

@@ -28,7 +28,7 @@ pub fn create_project(
});
let now = now_unix_ms();
let project = Project {
let mut project = Project {
id: id.clone(),
name: name.to_string(),
slug: slug.clone(),
@@ -49,12 +49,98 @@ pub fn create_project(
// Create directory structure
create_directory_structure(&data_dir)?;
// Store a stable, absolute machine-local pointer after the folder exists.
if data_path.is_some() {
project.data_path = Some(data_dir.canonicalize()?.to_string_lossy().to_string());
q::update_project(conn, &project)?;
}
// Write default meta files
write_default_meta_files(&data_dir, name)?;
Ok(project)
}
/// Open an existing portable project folder and remember its current location
/// in the machine-local project registry (the application database).
///
/// The folder itself is authoritative: `data_path` is derived from the
/// location of `meta/project.json` and is never written into that file.
pub fn open_project(conn: &Connection, folder_path: &Path) -> EngineResult<Project> {
let folder_path = folder_path.canonicalize().map_err(|error| {
EngineError::Validation(format!(
"cannot open project folder '{}': {error}",
folder_path.display()
))
})?;
let metadata_path = folder_path.join("meta/project.json");
if !metadata_path.is_file() {
return Err(EngineError::Validation(format!(
"project folder must contain meta/project.json: {}",
folder_path.display()
)));
}
let metadata: ProjectMetadata = serde_json::from_str(&fs::read_to_string(&metadata_path)?)?;
metadata.validate().map_err(EngineError::Validation)?;
let resolved_path = folder_path.to_string_lossy().to_string();
let projects = q::list_projects(conn)?;
// Reopening a registered folder is idempotent. If a registered folder no
// longer exists and its portable metadata name matches uniquely, treat the
// selected folder as that project's new location.
let exact = projects.iter().find(|project| {
project.data_path.as_deref().is_some_and(|path| {
Path::new(path)
.canonicalize()
.map(|registered| registered == folder_path)
.unwrap_or(false)
})
});
let moved_matches: Vec<&Project> = projects
.iter()
.filter(|project| {
project.name == metadata.name
&& project
.data_path
.as_deref()
.is_some_and(|path| !Path::new(path).join("meta/project.json").is_file())
})
.collect();
let moved = if moved_matches.len() == 1 {
Some(moved_matches[0])
} else {
None
};
if let Some(existing) = exact.or(moved) {
let mut project = existing.clone();
project.name = metadata.name;
project.description = metadata.description;
project.data_path = Some(resolved_path);
project.updated_at = now_unix_ms();
q::update_project(conn, &project)?;
return Ok(project);
}
let now = now_unix_ms();
let base_slug = slugify(&metadata.name);
let project = Project {
id: Uuid::new_v4().to_string(),
name: metadata.name,
slug: ensure_unique(&base_slug, |candidate| {
q::get_project_by_slug(conn, candidate).is_ok()
}),
description: metadata.description,
data_path: Some(resolved_path),
is_active: false,
created_at: now,
updated_at: now,
};
q::insert_project(conn, &project)?;
Ok(project)
}
/// Ensure the default project (id="default") exists.
/// Creates it on first launch if missing, per the DefaultProjectExists invariant.
/// Returns the project (existing or newly created).
@@ -182,6 +268,7 @@ fn write_default_meta_files(data_dir: &Path, project_name: &str) -> EngineResult
main_language: None,
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
@@ -210,69 +297,11 @@ fn write_default_meta_files(data_dir: &Path, project_name: &str) -> EngineResult
let default_opml = crate::engine::menu::default_menu_opml();
atomic_write_str(&meta_dir.join("menu.opml"), &default_opml)?;
// Starter templates — per project.allium StarterTemplatesCopied
copy_starter_templates(data_dir)?;
copy_bundled_site_assets(data_dir)?;
Ok(())
}
/// Copy bundled starter templates into the project templates directory.
/// Per project.allium: "Bundled starter templates are copied into the new project."
fn copy_starter_templates(data_dir: &Path) -> EngineResult<()> {
let templates_dir = data_dir.join("templates");
let partials_dir = templates_dir.join("partials");
fs::create_dir_all(&partials_dir)?;
// Starter templates embedded at compile time from assets/starter-templates/
let templates: &[(&str, &str)] = &[
(
"single-post.liquid",
include_str!("../../../../assets/starter-templates/single-post.liquid"),
),
(
"post-list.liquid",
include_str!("../../../../assets/starter-templates/post-list.liquid"),
),
(
"not-found.liquid",
include_str!("../../../../assets/starter-templates/not-found.liquid"),
),
];
let partials: &[(&str, &str)] = &[
(
"head.liquid",
include_str!("../../../../assets/starter-templates/partials/head.liquid"),
),
(
"menu.liquid",
include_str!("../../../../assets/starter-templates/partials/menu.liquid"),
),
(
"menu-items.liquid",
include_str!("../../../../assets/starter-templates/partials/menu-items.liquid"),
),
(
"language-switcher.liquid",
include_str!("../../../../assets/starter-templates/partials/language-switcher.liquid"),
),
];
for (name, content) in templates {
let path = templates_dir.join(name);
if !path.exists() {
atomic_write_str(&path, content)?;
}
}
for (name, content) in partials {
let path = partials_dir.join(name);
if !path.exists() {
atomic_write_str(&path, content)?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -308,6 +337,15 @@ mod tests {
assert!(data_path.join("assets/pico.min.css").is_file());
assert!(data_path.join("assets/tag-cloud.js").is_file());
// Bundled defaults stay in the application. Project templates are
// reserved for user-managed overrides.
assert!(
fs::read_dir(data_path.join("templates"))
.unwrap()
.next()
.is_none()
);
// Verify meta files
let project_json: serde_json::Value =
serde_json::from_str(&fs::read_to_string(data_path.join("meta/project.json")).unwrap())
@@ -343,6 +381,54 @@ mod tests {
assert_eq!(p2.slug, "blog-2");
}
#[test]
fn open_project_registers_existing_folder_without_rewriting_metadata() {
let (db, dir) = setup();
let data_path = dir.path().join("portable-blog");
fs::create_dir_all(data_path.join("meta")).unwrap();
let metadata = r#"{"name":"Portable Blog","description":"Moved safely"}"#;
fs::write(data_path.join("meta/project.json"), metadata).unwrap();
let project = open_project(db.conn(), &data_path).unwrap();
assert_eq!(project.name, "Portable Blog");
assert_eq!(project.description.as_deref(), Some("Moved safely"));
assert_eq!(
project.data_path.as_deref(),
data_path.canonicalize().unwrap().to_str()
);
assert_eq!(
fs::read_to_string(data_path.join("meta/project.json")).unwrap(),
metadata
);
}
#[test]
fn open_project_updates_moved_matching_project_location() {
let (db, dir) = setup();
let old_path = dir.path().join("old-location");
let project =
create_project(db.conn(), "Movable Blog", Some(old_path.to_str().unwrap())).unwrap();
let new_path = dir.path().join("new-location");
fs::rename(&old_path, &new_path).unwrap();
let reopened = open_project(db.conn(), &new_path).unwrap();
assert_eq!(reopened.id, project.id);
assert_eq!(
reopened.data_path.as_deref(),
new_path.canonicalize().unwrap().to_str()
);
assert_eq!(list_projects(db.conn()).unwrap().len(), 1);
}
#[test]
fn open_project_rejects_folder_without_project_metadata() {
let (db, dir) = setup();
let error = open_project(db.conn(), dir.path()).unwrap_err();
assert!(error.to_string().contains("meta/project.json"));
}
#[test]
fn get_active_project_none() {
let (db, _dir) = setup();

462
crates/bds-core/src/engine/publishing.rs Normal file
View File

@@ -0,0 +1,462 @@
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use serde::{Deserialize, Serialize};
use walkdir::WalkDir;
use crate::engine::{EngineError, EngineResult};
use crate::model::{PublishingPreferences, SshMode};
use crate::util::atomic_write_str;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublishJobStatus {
Pending,
Running,
Completed,
Failed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UploadTargetKind {
Html,
Thumbnails,
Media,
}
impl UploadTargetKind {
fn as_str(self) -> &'static str {
match self {
Self::Html => "html",
Self::Thumbnails => "thumbnails",
Self::Media => "media",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UploadTarget {
pub kind: UploadTargetKind,
pub local_dir: PathBuf,
pub remote_dir: String,
}
#[derive(Debug, Clone)]
pub struct PublishJob {
pub ssh_host: String,
pub ssh_user: String,
pub ssh_remote_path: String,
pub ssh_mode: SshMode,
pub status: PublishJobStatus,
pub completed_targets: Vec<UploadTargetKind>,
pub error: Option<String>,
}
impl PublishJob {
fn start(&mut self) -> EngineResult<()> {
if self.status != PublishJobStatus::Pending {
return Err(EngineError::Conflict(
"publish job can only start while pending".into(),
));
}
self.status = PublishJobStatus::Running;
Ok(())
}
fn complete(&mut self) -> EngineResult<()> {
if self.status != PublishJobStatus::Running {
return Err(EngineError::Conflict(
"publish job can only complete while running".into(),
));
}
self.status = PublishJobStatus::Completed;
Ok(())
}
fn fail(&mut self, error: impl Into<String>) {
if self.status == PublishJobStatus::Running {
self.status = PublishJobStatus::Failed;
self.error = Some(error.into());
}
}
}
#[derive(Debug, Clone)]
struct Credentials {
host: String,
user: String,
remote_path: String,
mode: SshMode,
}
impl Credentials {
fn from_preferences(preferences: &PublishingPreferences) -> EngineResult<Self> {
let required = |value: &Option<String>, field: &str| {
value
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_owned)
.ok_or_else(|| EngineError::Validation(format!("missing {field}")))
};
Ok(Self {
host: required(&preferences.ssh_host, "SSH host")?,
user: required(&preferences.ssh_user, "SSH user")?,
remote_path: required(&preferences.ssh_remote_path, "SSH remote path")?,
mode: preferences.ssh_mode.clone(),
})
}
fn remote_base(&self) -> String {
format!("{}@{}", self.user, self.host)
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
struct ScpMtimeCache(BTreeMap<String, u64>);
type CommandRunner<'a> = dyn FnMut(&str, &[String]) -> Result<(), String> + 'a;
/// Upload the generated site using non-interactive SSH-agent authentication.
pub fn upload_site(
data_dir: &Path,
private_cache_dir: &Path,
preferences: &PublishingPreferences,
mut on_progress: impl FnMut(usize, usize, UploadTargetKind),
) -> EngineResult<PublishJob> {
if std::env::var_os("SSH_AUTH_SOCK").is_none() {
return Err(EngineError::Validation(
"SSH agent is unavailable (SSH_AUTH_SOCK is not set)".into(),
));
}
upload_site_with_runner(
data_dir,
private_cache_dir,
preferences,
&mut |program, args| run_command(program, args),
&mut on_progress,
)
}
fn upload_site_with_runner(
data_dir: &Path,
private_cache_dir: &Path,
preferences: &PublishingPreferences,
runner: &mut CommandRunner<'_>,
on_progress: &mut dyn FnMut(usize, usize, UploadTargetKind),
) -> EngineResult<PublishJob> {
let credentials = Credentials::from_preferences(preferences)?;
let targets = build_upload_targets(data_dir, &credentials);
let mut job = PublishJob {
ssh_host: credentials.host.clone(),
ssh_user: credentials.user.clone(),
ssh_remote_path: credentials.remote_path.clone(),
ssh_mode: credentials.mode.clone(),
status: PublishJobStatus::Pending,
completed_targets: Vec::new(),
error: None,
};
job.start()?;
let cache_path = private_cache_dir.join("publishing-scp-mtimes.json");
let mut cache = read_cache(&cache_path);
for (index, target) in targets.iter().enumerate() {
on_progress(index + 1, targets.len(), target.kind);
let result = match credentials.mode {
SshMode::Rsync => upload_rsync(target, &credentials, runner),
SshMode::Scp => upload_scp(target, &credentials, &mut cache, runner),
};
if let Err(error) = result {
job.fail(error.clone());
return Err(EngineError::Parse(error));
}
job.completed_targets.push(target.kind);
if matches!(credentials.mode, SshMode::Scp) {
write_cache(&cache_path, &cache)?;
}
}
job.complete()?;
Ok(job)
}
fn build_upload_targets(data_dir: &Path, credentials: &Credentials) -> Vec<UploadTarget> {
let root = credentials.remote_path.trim_end_matches('/');
vec![
UploadTarget {
kind: UploadTargetKind::Html,
local_dir: data_dir.join("html"),
remote_dir: root.to_owned(),
},
UploadTarget {
kind: UploadTargetKind::Thumbnails,
local_dir: data_dir.join("thumbnails"),
remote_dir: format!("{root}/thumbnails"),
},
UploadTarget {
kind: UploadTargetKind::Media,
local_dir: data_dir.join("media"),
remote_dir: format!("{root}/media"),
},
]
}
fn upload_rsync(
target: &UploadTarget,
credentials: &Credentials,
runner: &mut CommandRunner<'_>,
) -> Result<(), String> {
if !target.local_dir.is_dir() {
return Ok(());
}
let mut args = vec![
"--update".into(),
"--compress".into(),
"--verbose".into(),
"--recursive".into(),
"--times".into(),
];
if target.kind == UploadTargetKind::Media {
args.push("--exclude=*.meta".into());
}
args.push(format!("{}/", target.local_dir.display()));
args.push(format!(
"{}:{}/",
credentials.remote_base(),
target.remote_dir.trim_end_matches('/')
));
runner("rsync", &args)
}
fn upload_scp(
target: &UploadTarget,
credentials: &Credentials,
cache: &mut ScpMtimeCache,
runner: &mut CommandRunner<'_>,
) -> Result<(), String> {
let files = list_target_files(target).map_err(|error| error.to_string())?;
let mut remote_dirs = BTreeSet::new();
let mut pending = Vec::new();
for relative in files {
let local = target.local_dir.join(&relative);
let mtime = modified_seconds(&local).map_err(|error| error.to_string())?;
let key = cache_key(credentials, target, &relative);
if cache.0.get(&key).is_some_and(|recorded| *recorded >= mtime) {
continue;
}
let remote_file = format!("{}/{}", target.remote_dir, relative.to_string_lossy());
if let Some(parent) = Path::new(&remote_file).parent() {
remote_dirs.insert(parent.to_string_lossy().to_string());
}
pending.push((local, remote_file, key, mtime));
}
if pending.is_empty() {
return Ok(());
}
let mut mkdir_args = vec![credentials.remote_base(), "mkdir".into(), "-p".into()];
mkdir_args.extend(remote_dirs);
runner("ssh", &mkdir_args)?;
for (local, remote_file, key, mtime) in pending {
runner(
"scp",
&[
"-q".into(),
"-p".into(),
local.to_string_lossy().to_string(),
format!("{}:{remote_file}", credentials.remote_base()),
],
)?;
cache.0.insert(key, mtime);
}
Ok(())
}
fn list_target_files(target: &UploadTarget) -> EngineResult<Vec<PathBuf>> {
if !target.local_dir.is_dir() {
return Ok(Vec::new());
}
let mut files = WalkDir::new(&target.local_dir)
.into_iter()
.filter_map(Result::ok)
.filter(|entry| entry.file_type().is_file())
.filter_map(|entry| {
entry
.path()
.strip_prefix(&target.local_dir)
.ok()
.map(PathBuf::from)
})
.filter(|relative| {
target.kind != UploadTargetKind::Media || !relative.to_string_lossy().ends_with(".meta")
})
.collect::<Vec<_>>();
files.sort();
Ok(files)
}
fn cache_key(credentials: &Credentials, target: &UploadTarget, relative: &Path) -> String {
format!(
"{}|{}|{}|{}|{}|{}",
credentials.host,
credentials.user,
credentials.remote_path,
target.kind.as_str(),
target.remote_dir,
relative.to_string_lossy()
)
}
fn modified_seconds(path: &Path) -> std::io::Result<u64> {
Ok(fs::metadata(path)?
.modified()?
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs())
}
fn read_cache(path: &Path) -> ScpMtimeCache {
fs::read_to_string(path)
.ok()
.and_then(|content| serde_json::from_str(&content).ok())
.unwrap_or_default()
}
fn write_cache(path: &Path, cache: &ScpMtimeCache) -> EngineResult<()> {
atomic_write_str(path, &serde_json::to_string(cache)?)?;
Ok(())
}
fn run_command(program: &str, args: &[String]) -> Result<(), String> {
let output = Command::new(program)
.args(args)
.output()
.map_err(|error| format!("failed to start {program}: {error}"))?;
if output.status.success() {
Ok(())
} else {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
Err(if stderr.is_empty() {
format!("{program} exited with {}", output.status)
} else {
stderr
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
fn preferences(mode: SshMode) -> PublishingPreferences {
PublishingPreferences {
ssh_host: Some("example.test".into()),
ssh_user: Some("alice".into()),
ssh_remote_path: Some("/srv/blog".into()),
ssh_mode: mode,
}
}
#[test]
fn rsync_uses_required_flags_and_excludes_media_sidecars() {
let dir = TempDir::new().unwrap();
let cache = TempDir::new().unwrap();
for subdir in ["html", "thumbnails", "media"] {
fs::create_dir_all(dir.path().join(subdir)).unwrap();
fs::write(dir.path().join(subdir).join("file.txt"), "x").unwrap();
}
let mut commands = Vec::<(String, Vec<String>)>::new();
let job = upload_site_with_runner(
dir.path(),
cache.path(),
&preferences(SshMode::Rsync),
&mut |program, args| {
commands.push((program.to_owned(), args.to_vec()));
Ok(())
},
&mut |_, _, _| {},
)
.unwrap();
assert_eq!(job.status, PublishJobStatus::Completed);
assert_eq!(job.completed_targets.len(), 3);
assert!(commands.iter().all(|(program, _)| program == "rsync"));
assert!(commands.iter().all(|(_, args)| {
args.contains(&"--update".into())
&& args.contains(&"--compress".into())
&& args.contains(&"--verbose".into())
}));
assert_eq!(
commands
.iter()
.filter(|(_, args)| args.contains(&"--exclude=*.meta".into()))
.count(),
1
);
}
#[test]
fn scp_excludes_sidecars_and_skips_unchanged_files() {
let dir = TempDir::new().unwrap();
let cache = TempDir::new().unwrap();
fs::create_dir_all(dir.path().join("media/nested")).unwrap();
fs::write(dir.path().join("media/nested/photo.jpg"), "image").unwrap();
fs::write(dir.path().join("media/nested/photo.jpg.meta"), "metadata").unwrap();
let prefs = preferences(SshMode::Scp);
let mut first = Vec::<(String, Vec<String>)>::new();
upload_site_with_runner(
dir.path(),
cache.path(),
&prefs,
&mut |program, args| {
first.push((program.to_owned(), args.to_vec()));
Ok(())
},
&mut |_, _, _| {},
)
.unwrap();
assert_eq!(
first.iter().filter(|(program, _)| program == "scp").count(),
1
);
assert!(
first
.iter()
.all(|(_, args)| !args.iter().any(|arg| arg.ends_with(".meta")))
);
let mut second = Vec::<String>::new();
upload_site_with_runner(
dir.path(),
cache.path(),
&prefs,
&mut |program, _| {
second.push(program.to_owned());
Ok(())
},
&mut |_, _, _| {},
)
.unwrap();
assert!(second.is_empty());
}
#[test]
fn failed_target_does_not_complete_job() {
let dir = TempDir::new().unwrap();
let cache = TempDir::new().unwrap();
fs::create_dir_all(dir.path().join("html")).unwrap();
fs::write(dir.path().join("html/index.html"), "x").unwrap();
let error = upload_site_with_runner(
dir.path(),
cache.path(),
&preferences(SshMode::Rsync),
&mut |_, _| Err("network down".into()),
&mut |_, _, _| {},
)
.unwrap_err();
assert!(error.to_string().contains("network down"));
}
}

45
crates/bds-core/src/engine/script.rs
View File

@@ -32,13 +32,17 @@ pub fn create_script(
let now = now_unix_ms();
let id = Uuid::new_v4().to_string();
let entrypoint = entrypoint.unwrap_or(match &kind {
ScriptKind::Macro => "render",
ScriptKind::Utility | ScriptKind::Transform => "main",
});
let script = Script {
id,
project_id: project_id.to_string(),
slug,
title: title.to_string(),
kind,
entrypoint: entrypoint.unwrap_or("render").to_string(),
entrypoint: entrypoint.to_string(),
enabled: true,
version: 1,
file_path: String::new(),
@@ -127,9 +131,7 @@ pub fn save_script(conn: &Connection, script_id: &str, content: &str) -> EngineR
/// Validate Lua script syntax. Returns Ok(()) or a parse error message.
pub fn validate_script_syntax(content: &str) -> Result<(), String> {
// Basic validation: balanced block structures
validate_lua_blocks(content)?;
Ok(())
crate::scripting::validate(content)
}
/// Discover entrypoint function names from Lua source.
@@ -262,41 +264,6 @@ fn script_kind_to_frontmatter(kind: &ScriptKind) -> String {
}
}
fn validate_lua_blocks(content: &str) -> Result<(), String> {
// Track block-level nesting for function/end, if/end, for/end, while/end, do/end
let mut depth: i32 = 0;
for line in content.lines() {
let trimmed = line.trim();
// Skip comments
if trimmed.starts_with("--") {
continue;
}
// Count block openers
for word in trimmed.split_whitespace() {
match word {
"function" | "if" | "for" | "while" | "do" => depth += 1,
"end" | "end," | "end;" => {
depth -= 1;
if depth < 0 {
return Err("unexpected 'end' without matching block opener".to_string());
}
}
_ => {}
}
}
// Handle "then" on if lines (don't double-count)
// Handle inline "function() ... end"
}
if depth > 0 {
return Err(format!("{depth} unclosed block(s) — missing 'end'"));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;

48
crates/bds-core/src/engine/script_rebuild.rs
View File

@@ -30,7 +30,7 @@ fn parse_script_kind(s: &str) -> Result<ScriptKind, String> {
/// Rebuild scripts from the filesystem into the database.
///
/// Walks the `scripts/` directory for `*.lua` and `*.py` files, parses each
/// Walks the `scripts/` directory for `*.lua` files, parses each
/// via frontmatter, and either creates or updates the corresponding DB row.
/// Published scripts (those present on disk) have `content = None` in the DB.
pub fn rebuild_scripts_from_filesystem(
@@ -54,7 +54,7 @@ pub fn rebuild_scripts_from_filesystem(
continue;
}
let ext = path.extension().and_then(|e| e.to_str());
if ext != Some("lua") && ext != Some("py") {
if ext != Some("lua") {
continue;
}
@@ -75,9 +75,9 @@ pub fn rebuild_scripts_from_filesystem(
Ok(report)
}
/// Rebuild a single script from a `.lua` or `.py` file.
/// Rebuild a single script from a `.lua` file.
/// Returns `true` if created, `false` if updated.
fn rebuild_single_script(
pub(crate) fn rebuild_single_script(
conn: &Connection,
data_dir: &Path,
project_id: &str,
@@ -197,46 +197,6 @@ end
assert_eq!(script.file_path, "scripts/my-macro.lua");
}
#[test]
fn rebuild_creates_script_from_py() {
let (db, dir) = setup();
let scripts_dir = dir.path().join("scripts");
fs::create_dir_all(&scripts_dir).unwrap();
let content = "\"\"\"\n\
---
id: \"py-script-1\"
slug: \"my-utility\"
title: \"My Utility\"
kind: \"utility\"
entrypoint: \"main\"
enabled: true
version: 1
createdAt: \"2024-01-01T00:00:00.000Z\"
updatedAt: \"2024-01-01T00:00:00.000Z\"
---
\"\"\"
def main():
print(\"hello\")
";
fs::write(scripts_dir.join("my-utility.py"), content).unwrap();
let report = rebuild_scripts_from_filesystem(db.conn(), dir.path(), "p1").unwrap();
assert_eq!(report.created, 1);
assert_eq!(report.updated, 0);
assert!(report.errors.is_empty(), "errors: {:?}", report.errors);
let script = qs::get_script_by_id(db.conn(), "py-script-1").unwrap();
assert_eq!(script.slug, "my-utility");
assert_eq!(script.title, "My Utility");
assert_eq!(script.kind, ScriptKind::Utility);
assert_eq!(script.entrypoint, "main");
assert_eq!(script.status, ScriptStatus::Published);
assert!(script.content.is_none());
assert_eq!(script.file_path, "scripts/my-utility.py");
}
#[test]
fn rebuild_updates_existing() {
let (db, dir) = setup();

2
crates/bds-core/src/engine/template_rebuild.rs
View File

@@ -78,7 +78,7 @@ pub fn rebuild_templates_from_filesystem(
/// Rebuild a single template from a `.liquid` file.
/// Returns `true` if created, `false` if updated.
fn rebuild_single_template(
pub(crate) fn rebuild_single_template(
conn: &Connection,
data_dir: &Path,
project_id: &str,

220
crates/bds-core/src/engine/validate_content.rs
View File

@@ -1,220 +0,0 @@
//! Validation functions for template (Liquid) and script (Lua/Python) content.
//! Per template.allium and script.allium, these are pre-publish gates.
//!
//! Current implementation: basic structural checks.
//! When a full Liquid parser crate is added, upgrade `validate_liquid` to
//! attempt a real parse. Similarly for `validate_script` with a Lua parser.
/// Result of a validation check.
#[derive(Debug, Clone)]
pub struct ValidationResult {
pub valid: bool,
pub errors: Vec<String>,
}
impl ValidationResult {
pub fn ok() -> Self {
Self {
valid: true,
errors: Vec::new(),
}
}
pub fn fail(errors: Vec<String>) -> Self {
Self {
valid: false,
errors,
}
}
}
/// Validate Liquid template content.
/// Per template.allium: "LiquidJS parser must accept the template".
/// Currently performs structural tag-matching. Upgrade to full liquid crate parse later.
pub fn validate_liquid(content: &str) -> ValidationResult {
let mut errors = Vec::new();
// Check for unmatched Liquid block tags
let block_tags = [
"if", "unless", "for", "case", "capture", "comment", "raw", "paginate", "tablerow",
"block", "schema",
];
for tag in &block_tags {
let open_pattern = format!("{{% {tag}");
let close_pattern = format!("{{% end{tag}");
let opens = content.matches(&open_pattern).count();
let closes = content.matches(&close_pattern).count();
if opens != closes {
errors.push(format!(
"Unmatched {{% {tag} %}}: {opens} opens, {closes} closes"
));
}
}
// Check for unclosed {{ }}
let double_opens = content.matches("{{").count();
let double_closes = content.matches("}}").count();
if double_opens != double_closes {
errors.push(format!(
"Unmatched {{}}: {double_opens} opens, {double_closes} closes"
));
}
// Check for unclosed {% %}
let tag_opens = content.matches("{%").count();
let tag_closes = content.matches("%}").count();
if tag_opens != tag_closes {
errors.push(format!(
"Unmatched {{% %}}: {tag_opens} opens, {tag_closes} closes"
));
}
if errors.is_empty() {
ValidationResult::ok()
} else {
ValidationResult::fail(errors)
}
}
/// Validate script content (Lua syntax).
/// Per script.allium: "AST parsing must succeed".
/// Currently performs basic bracket-matching. Upgrade to full Lua parser later.
pub fn validate_script(content: &str) -> ValidationResult {
let mut errors = Vec::new();
// Basic bracket matching
let mut paren_depth: i32 = 0;
let mut brace_depth: i32 = 0;
let mut bracket_depth: i32 = 0;
for (line_num, line) in content.lines().enumerate() {
// Skip comment lines
let trimmed = line.trim();
if trimmed.starts_with("--") {
continue;
}
for ch in line.chars() {
match ch {
'(' => paren_depth += 1,
')' => paren_depth -= 1,
'{' => brace_depth += 1,
'}' => brace_depth -= 1,
'[' => bracket_depth += 1,
']' => bracket_depth -= 1,
_ => {}
}
if paren_depth < 0 {
errors.push(format!("Unexpected ')' at line {}", line_num + 1));
paren_depth = 0;
}
if brace_depth < 0 {
errors.push(format!("Unexpected '}}' at line {}", line_num + 1));
brace_depth = 0;
}
if bracket_depth < 0 {
errors.push(format!("Unexpected ']' at line {}", line_num + 1));
bracket_depth = 0;
}
}
}
if paren_depth != 0 {
errors.push(format!("Unclosed parentheses: depth {paren_depth}"));
}
if brace_depth != 0 {
errors.push(format!("Unclosed braces: depth {brace_depth}"));
}
if bracket_depth != 0 {
errors.push(format!("Unclosed brackets: depth {bracket_depth}"));
}
// Check for unmatched Lua block keywords.
// In Lua, `for ... do ... end` and `while ... do ... end` each form a
// single block, so `do` on such lines must NOT be counted as a separate
// opener.
let mut block_opens: usize = 0;
let mut block_closes: usize = 0;
for line in content.lines() {
let trimmed = line.trim();
if trimmed.starts_with("--") {
continue;
}
let words: Vec<&str> = trimmed.split_whitespace().collect();
let has_for_or_while = words.iter().any(|w| matches!(*w, "for" | "while"));
for w in &words {
match *w {
"function" | "if" | "for" | "while" | "repeat" => block_opens += 1,
"do" if !has_for_or_while => block_opens += 1,
"end" | "until" => block_closes += 1,
_ => {}
}
}
}
if block_opens > block_closes {
errors.push(format!(
"Possible unclosed blocks: {block_opens} openers, {block_closes} closers"
));
}
if errors.is_empty() {
ValidationResult::ok()
} else {
ValidationResult::fail(errors)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn valid_liquid_template() {
let content = r#"
<html>
{% if user %}
<p>{{ user.name }}</p>
{% endif %}
</html>"#;
let result = validate_liquid(content);
assert!(result.valid, "Errors: {:?}", result.errors);
}
#[test]
fn unmatched_liquid_if() {
let content = "{% if true %}<p>Hello</p>";
let result = validate_liquid(content);
assert!(!result.valid);
assert!(result.errors.iter().any(|e| e.contains("if")));
}
#[test]
fn unmatched_liquid_output() {
let content = "{{ user.name";
let result = validate_liquid(content);
assert!(!result.valid);
}
#[test]
fn valid_lua_script() {
let content = r#"
function render(ctx)
local result = {}
for i = 1, 10 do
result[i] = i * 2
end
return result
end
"#;
let result = validate_script(content);
assert!(result.valid, "Errors: {:?}", result.errors);
}
#[test]
fn unmatched_lua_parens() {
let content = "function test(\n local x = 1\nend";
let result = validate_script(content);
assert!(!result.valid);
}
}

2
crates/bds-core/src/engine/validate_translations.rs
View File

@@ -44,7 +44,7 @@ pub enum TranslationIssueKind {
}
/// Result of translation validation.
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct TranslationValidationReport {
pub db_issues: Vec<TranslationIssue>,
pub fs_issues: Vec<TranslationIssue>,

31
crates/bds-core/src/model/metadata.rs
View File

@@ -4,6 +4,23 @@ fn default_max_posts() -> i32 {
50
}
fn default_image_import_concurrency() -> i32 {
4
}
fn deserialize_image_import_concurrency<'de, D>(deserializer: D) -> Result<i32, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
let parsed = value
.as_i64()
.map(|value| value as i32)
.or_else(|| value.as_str().and_then(|value| value.parse::<i32>().ok()))
.unwrap_or_else(default_image_import_concurrency);
Ok(parsed.clamp(1, 8))
}
fn default_true() -> bool {
true
}
@@ -22,6 +39,11 @@ pub struct ProjectMetadata {
pub default_author: Option<String>,
#[serde(default = "default_max_posts")]
pub max_posts_per_page: i32,
#[serde(
default = "default_image_import_concurrency",
deserialize_with = "deserialize_image_import_concurrency"
)]
pub image_import_concurrency: i32,
#[serde(skip_serializing_if = "Option::is_none")]
pub blogmark_category: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -41,6 +63,12 @@ impl ProjectMetadata {
self.max_posts_per_page
));
}
if !(1..=8).contains(&self.image_import_concurrency) {
return Err(format!(
"imageImportConcurrency must be 1..8, got {}",
self.image_import_concurrency
));
}
Ok(())
}
}
@@ -80,6 +108,7 @@ mod tests {
main_language: Some("en".into()),
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,
@@ -101,6 +130,7 @@ mod tests {
let json = r#"{"name": "Minimal"}"#;
let meta: ProjectMetadata = serde_json::from_str(json).unwrap();
assert_eq!(meta.max_posts_per_page, 50);
assert_eq!(meta.image_import_concurrency, 4);
assert!(!meta.semantic_similarity_enabled);
assert!(meta.blog_languages.is_empty());
}
@@ -159,6 +189,7 @@ mod tests {
main_language: None,
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,

62
crates/bds-core/src/render/macros.rs
View File

@@ -52,7 +52,44 @@ fn render_macro(invocation: &str, context: &MacroRenderContext) -> Option<String
"vimeo" => Some(render_vimeo(&args)),
"photo_archive" => Some(render_photo_archive(&args, context)),
"tag_cloud" => Some(render_tag_cloud(&args, context)),
_ => None,
_ => render_script_macro(name, &args, context),
}
}
fn render_script_macro(
name: &str,
args: &HashMap<String, JsonValue>,
context: &MacroRenderContext,
) -> Option<String> {
let definition = context.roots.get("macro_scripts")?.get(name)?;
let source = definition.get("source")?.as_str()?;
let entrypoint = definition
.get("entrypoint")
.and_then(JsonValue::as_str)
.unwrap_or("render");
let env = serde_json::json!({
"isPreview": context.roots.get("is_preview").and_then(JsonValue::as_bool).unwrap_or(false),
"mainLanguage": context.roots.get("main_language").and_then(JsonValue::as_str).unwrap_or("en"),
"language": context.roots.get("language").and_then(JsonValue::as_str).unwrap_or("en"),
"languagePrefix": context.roots.get("language_prefix").and_then(JsonValue::as_str).unwrap_or(""),
"hook": "markdown",
"source": { "kind": if context.post_id.is_some() { "post" } else { "page" } },
"translations": context.roots.get("translations").cloned().unwrap_or(JsonValue::Array(Vec::new())),
});
let params = serde_json::to_value(args).ok()?;
match crate::scripting::execute_many(
source,
entrypoint,
&[params, env],
crate::scripting::ExecutionKind::Macro,
&crate::scripting::ExecutionControl::default(),
) {
Ok(result) => Some(match result.value {
JsonValue::Null => String::new(),
JsonValue::String(value) => value,
value => value.to_string(),
}),
Err(_) => Some(String::new()),
}
}
@@ -499,4 +536,27 @@ mod tests {
markdown
);
}
#[test]
fn script_macro_receives_named_params_and_environment() {
let mut roots = serde_json::Map::new();
roots.insert(
"macro_scripts".into(),
serde_json::json!({
"notice": {
"source": "function render(params, env) return '<aside>' .. params.text .. ':' .. env.language .. '</aside>' end",
"entrypoint": "render"
}
}),
);
roots.insert("language".into(), serde_json::Value::String("de".into()));
let rendered = expand_builtin_macros(
"[[notice text=Hallo]]",
&MacroRenderContext {
roots,
post_id: Some("post-1".into()),
},
);
assert_eq!(rendered, "<aside>Hallo:de</aside>");
}
}

14
crates/bds-core/src/render/page_renderer.rs
View File

@@ -172,7 +172,19 @@ impl Filter for MarkdownFilter {
fn collect_macro_roots(runtime: &dyn Runtime) -> JsonMap<String, JsonValue> {
let mut roots = JsonMap::new();
for key in ["post", "post_tags", "tag_color_by_name", "project", "Tags"] {
for key in [
"post",
"post_tags",
"tag_color_by_name",
"project",
"Tags",
"macro_scripts",
"language",
"language_prefix",
"main_language",
"is_preview",
"translations",
] {
if let Some(value) = runtime.try_get(&[ScalarCow::new(key)]) {
roots.insert(key.to_string(), liquid_value_to_json(value.as_view()));
}

1
crates/bds-core/src/render/routes.rs
View File

@@ -492,6 +492,7 @@ mod tests {
main_language: Some("en".into()),
default_author: None,
max_posts_per_page: 50,
image_import_concurrency: 4,
blogmark_category: None,
pico_theme: None,
semantic_similarity_enabled: false,

69
crates/bds-core/src/render/site.rs
View File

@@ -11,13 +11,14 @@ use serde_json::{Value, json};
use crate::db::queries;
use crate::engine::menu::{self, MenuItemKind};
use crate::model::{
CategorySettings, Media, Post, ProjectMetadata, Tag, Template, TemplateKind, TemplateStatus,
CategorySettings, Media, Post, ProjectMetadata, ScriptKind, Tag, Template, TemplateKind,
TemplateStatus,
};
use crate::render::{
RenderCategorySettings, RenderTemplateLookup, build_canonical_post_path,
render_liquid_template, resolve_post_template,
};
use crate::util::frontmatter::{read_template_file, read_translation_file};
use crate::util::frontmatter::{read_script_file, read_template_file, read_translation_file};
use crate::util::slugify;
const STARTER_SINGLE_POST_TEMPLATE: &str =
@@ -69,6 +70,7 @@ struct TemplateBundle {
default_list_template: String,
not_found_template: String,
partials: HashMap<String, String>,
macro_scripts: HashMap<String, Value>,
}
#[derive(Debug, Clone)]
@@ -97,6 +99,24 @@ pub fn build_site_render_artifacts(
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
false,
)
}
fn build_site_render_artifacts_with_mode(
conn: &Connection,
data_dir: &Path,
project_id: &str,
metadata: &ProjectMetadata,
published_posts: &[(Post, String)],
is_preview: bool,
) -> Result<SiteRenderArtifacts, Box<dyn Error + Send + Sync>> {
let bundle = load_template_bundle(conn, data_dir, project_id)?;
let main_language = main_language(metadata).to_string();
@@ -136,6 +156,7 @@ pub fn build_site_render_artifacts(
&menu_items,
&post_data_json_by_id,
&bundle,
is_preview,
)
.map(|html| SitePage {
language: language.clone(),
@@ -173,6 +194,7 @@ pub fn build_site_render_artifacts(
&menu_items,
&post_data_json_by_id,
&bundle,
is_preview,
)?;
let canonical_path = build_canonical_post_path(&record.post, &language, &main_language);
let relative_path = format!("{}/index.html", canonical_path.trim_start_matches('/'));
@@ -202,8 +224,14 @@ pub fn build_preview_response(
published_posts: &[(Post, String)],
requested_path: &str,
) -> Result<PreviewRenderResult, Box<dyn Error + Send + Sync>> {
let artifacts =
build_site_render_artifacts(conn, data_dir, project_id, metadata, published_posts)?;
let artifacts = build_site_render_artifacts_with_mode(
conn,
data_dir,
project_id,
metadata,
published_posts,
true,
)?;
let normalized = normalize_request_path(requested_path);
if let Some(page) = artifacts
.pages
@@ -239,6 +267,30 @@ fn load_template_bundle(
let mut partials = starter_partials();
let mut default_list_template = STARTER_POST_LIST_TEMPLATE.to_string();
let mut not_found_template = STARTER_NOT_FOUND_TEMPLATE.to_string();
let mut macro_scripts = HashMap::new();
for script in queries::script::list_scripts_by_project(conn, project_id).unwrap_or_default() {
if script.kind != ScriptKind::Macro
|| !script.enabled
|| script.entrypoint.trim().is_empty()
{
continue;
}
let source = if let Some(content) = script.content {
content
} else if script.file_path.is_empty() {
String::new()
} else {
fs::read_to_string(data_dir.join(&script.file_path))
.ok()
.and_then(|raw| read_script_file(&raw).ok().map(|(_, body)| body))
.unwrap_or_default()
};
macro_scripts.insert(
script.slug,
json!({ "source": source, "entrypoint": script.entrypoint }),
);
}
for template in templates {
if !template.enabled {
@@ -315,6 +367,7 @@ fn load_template_bundle(
default_list_template,
not_found_template,
partials,
macro_scripts,
})
}
@@ -557,6 +610,7 @@ fn render_list_route(
menu_items: &[Value],
post_data_json_by_id: &HashMap<String, Value>,
bundle: &TemplateBundle,
is_preview: bool,
) -> Result<String, Box<dyn Error + Send + Sync>> {
let main_language = main_language(metadata);
let canonical_map = canonical_post_path_by_slug(posts, language, main_language);
@@ -569,6 +623,9 @@ fn render_list_route(
let context = json!({
"language": language,
"language_prefix": language_prefix(language, main_language),
"main_language": main_language,
"is_preview": is_preview,
"macro_scripts": bundle.macro_scripts,
"html_theme_attribute": serde_json::Value::Null,
"page_title": route.page_title,
"pico_stylesheet_href": pico_stylesheet_href(metadata),
@@ -626,6 +683,7 @@ fn render_post_route(
menu_items: &[Value],
post_data_json_by_id: &HashMap<String, Value>,
bundle: &TemplateBundle,
is_preview: bool,
) -> Result<String, Box<dyn Error + Send + Sync>> {
let render_categories = category_settings
.iter()
@@ -683,6 +741,9 @@ fn render_post_route(
let context = json!({
"language": language,
"language_prefix": language_prefix(language, main_language),
"main_language": main_language,
"is_preview": is_preview,
"macro_scripts": bundle.macro_scripts,
"page_title": record.post.title,
"pico_stylesheet_href": pico_stylesheet_href(metadata),
"html_theme_attribute": serde_json::Value::Null,

350
crates/bds-core/src/scripting/mod.rs
View File

@@ -1 +1,349 @@
// Scripting (Lua) — stubs for M0, implemented in M5/Wave 6.
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use mlua::{
Function, HookTriggers, Lua, LuaOptions, LuaSerdeExt, MultiValue, StdLib, Value, VmState,
};
use serde_json::Value as JsonValue;
const MACRO_TIMEOUT: Duration = Duration::from_secs(10);
const MEMORY_LIMIT_BYTES: usize = 32 * 1024 * 1024;
const MAX_TOASTS_PER_SCRIPT: usize = 5;
const MAX_TOAST_LENGTH: usize = 300;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExecutionKind {
Macro,
Utility,
Transform,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ScriptProgress {
pub current: f64,
pub total: Option<f64>,
pub message: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ScriptExecution {
pub value: JsonValue,
pub output: Vec<String>,
pub progress: Vec<ScriptProgress>,
pub toasts: Vec<String>,
}
#[derive(Clone)]
pub struct ExecutionControl {
cancelled: Arc<AtomicBool>,
}
impl Default for ExecutionControl {
fn default() -> Self {
Self {
cancelled: Arc::new(AtomicBool::new(false)),
}
}
}
impl ExecutionControl {
pub fn cancel(&self) {
self.cancelled.store(true, Ordering::Release);
}
}
/// Parse Lua source with the same runtime used for execution.
pub fn validate(source: &str) -> Result<(), String> {
let lua = sandboxed_lua()?;
lua.load(source)
.set_name("script")
.into_function()
.map(|_| ())
.map_err(|error| error.to_string())
}
/// Execute a named Lua entrypoint with one JSON-compatible argument.
pub fn execute(
source: &str,
entrypoint: &str,
args: &JsonValue,
kind: ExecutionKind,
control: &ExecutionControl,
) -> Result<ScriptExecution, String> {
execute_many(
source,
entrypoint,
std::slice::from_ref(args),
kind,
control,
)
}
/// Execute a named Lua entrypoint with positional JSON-compatible arguments.
pub fn execute_many(
source: &str,
entrypoint: &str,
args: &[JsonValue],
kind: ExecutionKind,
control: &ExecutionControl,
) -> Result<ScriptExecution, String> {
if entrypoint.trim().is_empty() {
return Err("script entrypoint is empty".into());
}
let lua = sandboxed_lua()?;
let output = Arc::new(Mutex::new(Vec::<String>::new()));
let progress = Arc::new(Mutex::new(Vec::<ScriptProgress>::new()));
let toasts = Arc::new(Mutex::new(Vec::<String>::new()));
install_host_api(&lua, &output, &progress, &toasts)?;
let started = Instant::now();
let cancelled = Arc::clone(&control.cancelled);
lua.set_hook(
HookTriggers::new().every_nth_instruction(5_000),
move |_lua, _debug| {
if cancelled.load(Ordering::Acquire) {
return Err(mlua::Error::RuntimeError("script cancelled".into()));
}
if kind == ExecutionKind::Macro && started.elapsed() >= MACRO_TIMEOUT {
return Err(mlua::Error::RuntimeError(
"macro exceeded 10 second timeout".into(),
));
}
Ok(VmState::Continue)
},
)
.map_err(|error| error.to_string())?;
lua.load(source)
.set_name("script")
.exec()
.map_err(|error| error.to_string())?;
let function: Function = lua
.globals()
.get(entrypoint)
.map_err(|_| format!("entrypoint '{entrypoint}' was not found"))?;
let arguments = args
.iter()
.map(|argument| lua.to_value(argument))
.collect::<Result<MultiValue, _>>()
.map_err(|error| error.to_string())?;
let value: Value = function
.call(arguments)
.map_err(|error| error.to_string())?;
let value = lua.from_value(value).map_err(|error| error.to_string())?;
Ok(ScriptExecution {
value,
output: output.lock().map_err(|_| "output sink poisoned")?.clone(),
progress: progress
.lock()
.map_err(|_| "progress sink poisoned")?
.clone(),
toasts: toasts.lock().map_err(|_| "toast sink poisoned")?.clone(),
})
}
fn sandboxed_lua() -> Result<Lua, String> {
let libraries = StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8;
let lua = Lua::new_with(libraries, LuaOptions::default()).map_err(|error| error.to_string())?;
lua.set_memory_limit(MEMORY_LIMIT_BYTES)
.map_err(|error| error.to_string())?;
let globals = lua.globals();
for dangerous in [
"collectgarbage",
"debug",
"dofile",
"io",
"load",
"loadfile",
"os",
"package",
"require",
] {
globals
.set(dangerous, Value::Nil)
.map_err(|error| error.to_string())?;
}
drop(globals);
Ok(lua)
}
fn install_host_api(
lua: &Lua,
output: &Arc<Mutex<Vec<String>>>,
progress: &Arc<Mutex<Vec<ScriptProgress>>>,
toasts: &Arc<Mutex<Vec<String>>>,
) -> Result<(), String> {
let globals = lua.globals();
let print_output = Arc::clone(output);
globals
.set(
"print",
lua.create_function(move |_lua, values: MultiValue| {
let line = values
.iter()
.map(lua_value_text)
.collect::<Vec<_>>()
.join("\t");
print_output.lock().unwrap().push(line);
Ok(())
})
.map_err(|error| error.to_string())?,
)
.map_err(|error| error.to_string())?;
let bds = lua.create_table().map_err(|error| error.to_string())?;
let app = lua.create_table().map_err(|error| error.to_string())?;
let log_output = Arc::clone(output);
app.set(
"log",
lua.create_function(move |_lua, values: MultiValue| {
let line = values
.iter()
.map(lua_value_text)
.collect::<Vec<_>>()
.join("\t");
log_output.lock().unwrap().push(line);
Ok(())
})
.map_err(|error| error.to_string())?,
)
.map_err(|error| error.to_string())?;
let progress_sink = Arc::clone(progress);
app.set(
"progress",
lua.create_function(
move |_lua, (current, total, message): (f64, Option<f64>, Option<String>)| {
progress_sink.lock().unwrap().push(ScriptProgress {
current,
total,
message,
});
Ok(())
},
)
.map_err(|error| error.to_string())?,
)
.map_err(|error| error.to_string())?;
let toast_sink = Arc::clone(toasts);
app.set(
"toast",
lua.create_function(move |_lua, message: String| {
let mut sink = toast_sink.lock().unwrap();
if sink.len() < MAX_TOASTS_PER_SCRIPT {
sink.push(message.chars().take(MAX_TOAST_LENGTH).collect());
}
Ok(())
})
.map_err(|error| error.to_string())?,
)
.map_err(|error| error.to_string())?;
bds.set("app", app).map_err(|error| error.to_string())?;
globals.set("bds", bds).map_err(|error| error.to_string())?;
Ok(())
}
fn lua_value_text(value: &Value) -> String {
match value {
Value::Nil => "nil".into(),
Value::Boolean(value) => value.to_string(),
Value::Integer(value) => value.to_string(),
Value::Number(value) => value.to_string(),
Value::String(value) => value.to_string_lossy(),
other => format!("{other:?}"),
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn executes_entrypoint_and_routes_output_and_progress() {
let result = execute(
r#"
function main(input)
print("starting", input.name)
bds.app.log("working")
bds.app.progress(1, 2, "half")
return { title = input.name, complete = true }
end
"#,
"main",
&json!({"name": "Example"}),
ExecutionKind::Utility,
&ExecutionControl::default(),
)
.unwrap();
assert_eq!(result.value["title"], "Example");
assert_eq!(result.value["complete"], true);
assert_eq!(result.output, vec!["starting\tExample", "working"]);
assert_eq!(result.progress[0].message.as_deref(), Some("half"));
}
#[test]
fn sandbox_hides_ambient_host_capabilities() {
let result = execute(
"function main(_) return { io = io, os = os, package = package, load = load } end",
"main",
&JsonValue::Null,
ExecutionKind::Utility,
&ExecutionControl::default(),
)
.unwrap();
assert_eq!(result.value, serde_json::json!({}));
}
#[test]
fn cancellation_stops_managed_execution() {
let control = ExecutionControl::default();
control.cancel();
let error = execute(
"function main(_) while true do end end",
"main",
&JsonValue::Null,
ExecutionKind::Utility,
&control,
)
.unwrap_err();
assert!(error.contains("cancelled"));
}
#[test]
fn validates_with_real_lua_parser() {
assert!(validate("function main() return 1 end").is_ok());
assert!(validate("function main( return 1 end").is_err());
}
#[test]
fn enforces_per_script_toast_budget_and_length() {
let result = execute(
r#"
function main(_)
for i = 1, 8 do bds.app.toast(string.rep("x", 400)) end
end
"#,
"main",
&JsonValue::Null,
ExecutionKind::Transform,
&ExecutionControl::default(),
)
.unwrap();
assert_eq!(result.toasts.len(), 5);
assert!(
result
.toasts
.iter()
.all(|toast| toast.chars().count() == 300)
);
}
}

64
crates/bds-core/src/util/frontmatter.rs
View File

@@ -17,26 +17,6 @@ pub fn split_frontmatter(input: &str) -> Option<(&str, &str)> {
Some((yaml, body))
}
/// Split content at Python docstring `"""` delimiters (for legacy .py scripts).
/// The YAML frontmatter is inside: `"""\n---\n{yaml}\n---\n"""`
pub fn split_docstring_frontmatter(input: &str) -> Option<(&str, &str)> {
let trimmed = input.trim_start_matches('\u{feff}');
if !trimmed.starts_with("\"\"\"") {
return None;
}
let after_open = &trimmed[3..];
let after_open = after_open.strip_prefix('\n').unwrap_or(after_open);
// Find closing """
let close_pos = after_open.find("\"\"\"")?;
let inside = &after_open[..close_pos].trim();
// Inside should be ---\n{yaml}\n---
let yaml_content = split_frontmatter(inside)?;
let body_start = close_pos + 3;
let rest = &after_open[body_start..];
let body = rest.strip_prefix('\n').unwrap_or(rest);
Some((yaml_content.0, body))
}
/// Format frontmatter + body into a complete file string.
pub fn format_frontmatter(yaml: &str, body: &str) -> String {
format!("---\n{yaml}\n---\n{body}")
@@ -485,14 +465,8 @@ impl ScriptFrontmatter {
}
}
/// Read a script file. Supports both `---` (Lua) and `"""` (Python) delimiters.
/// Read a Lua script file with standard YAML frontmatter.
pub fn read_script_file(content: &str) -> Result<(ScriptFrontmatter, String), String> {
// Try docstring format first (Python scripts)
if let Some((yaml, body)) = split_docstring_frontmatter(content) {
let fm = ScriptFrontmatter::from_yaml(yaml)?;
return Ok((fm, body.to_string()));
}
// Fall back to standard --- format (Lua scripts)
let (yaml, body) = split_frontmatter(content).ok_or("no frontmatter delimiters found")?;
let fm = ScriptFrontmatter::from_yaml(yaml)?;
Ok((fm, body.to_string()))
@@ -577,14 +551,6 @@ mod tests {
assert!(split_frontmatter("no frontmatter here").is_none());
}
#[test]
fn split_docstring() {
let input = "\"\"\"\n---\nfoo: bar\n---\n\"\"\"\nbody\n";
let (yaml, body) = split_docstring_frontmatter(input).unwrap();
assert_eq!(yaml, "foo: bar");
assert_eq!(body, "body\n");
}
#[test]
fn parse_esmeralda() {
let path = fixture_dir().join("posts/2005/11/esmeralda.md");
@@ -822,34 +788,6 @@ mod tests {
);
}
// --- Script frontmatter tests ---
#[test]
fn parse_fixture_script_bgg() {
let path = fixture_dir().join("scripts/bgg_link.py");
let content = fs::read_to_string(path).unwrap();
let (fm, body) = read_script_file(&content).unwrap();
assert_eq!(fm.id, "2b393cae-84b9-426f-b4cf-4902aea79d7d");
assert_eq!(fm.slug, "bgg_link");
assert_eq!(fm.title, "bgg link");
assert_eq!(fm.kind, "transform");
assert_eq!(fm.entrypoint, "normalize_blogmark");
assert!(fm.enabled);
assert_eq!(fm.version, 12);
assert!(body.contains("def normalize_blogmark"));
}
#[test]
fn parse_fixture_script_test() {
let path = fixture_dir().join("scripts/test_script.py");
let content = fs::read_to_string(path).unwrap();
let (fm, body) = read_script_file(&content).unwrap();
assert_eq!(fm.slug, "test_script");
assert_eq!(fm.kind, "utility");
assert_eq!(fm.entrypoint, "main");
assert!(body.contains("print"));
}
#[test]
fn template_frontmatter_roundtrip() {
let fm = TemplateFrontmatter {