From 429f471e4ef55e6caca13cbc83b0c474d3168289 Mon Sep 17 00:00:00 2001 From: Chili Palmer Date: Sun, 19 Jul 2026 13:17:45 +0200 Subject: [PATCH] Implement complete WordPress WXR import workflow. --- Cargo.lock | 37 + Cargo.toml | 3 + README.md | 1 + RUST_PLAN_CORE.md | 2 +- RUST_PLAN_EXTENSION.md | 25 +- crates/bds-core/Cargo.toml | 2 + .../src/db/queries/import_definition.rs | 62 + crates/bds-core/src/db/queries/mod.rs | 1 + crates/bds-core/src/engine/ai.rs | 66 + crates/bds-core/src/engine/media.rs | 45 +- crates/bds-core/src/engine/mod.rs | 1 + .../bds-core/src/engine/wordpress_import.rs | 1827 +++++++++++++++++ crates/bds-core/src/model/import.rs | 224 ++ crates/bds-core/src/model/mod.rs | 6 + crates/bds-core/src/scripting/core_host.rs | 1 + crates/bds-core/tests/wordpress_import.rs | 807 ++++++++ crates/bds-ui/Cargo.toml | 1 + crates/bds-ui/src/app.rs | 656 +++++- crates/bds-ui/src/platform/dialog.rs | 41 + crates/bds-ui/src/views/import_editor.rs | 777 +++++++ crates/bds-ui/src/views/mod.rs | 1 + crates/bds-ui/src/views/modal.rs | 1 + crates/bds-ui/src/views/sidebar.rs | 55 +- crates/bds-ui/src/views/workspace.rs | 36 +- locales/ui/de.ftl | 55 + locales/ui/en.ftl | 55 + locales/ui/es.ftl | 55 + locales/ui/fr.ftl | 55 + locales/ui/it.ftl | 55 + 29 files changed, 4927 insertions(+), 26 deletions(-) create mode 100644 crates/bds-core/src/db/queries/import_definition.rs create mode 100644 crates/bds-core/src/engine/wordpress_import.rs create mode 100644 crates/bds-core/src/model/import.rs create mode 100644 crates/bds-core/tests/wordpress_import.rs create mode 100644 crates/bds-ui/src/views/import_editor.rs diff --git a/Cargo.lock b/Cargo.lock index d761592..03281fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -804,6 +804,7 @@ dependencies = [ "diesel_migrations", "fluent-bundle", "fluent-syntax", + "htmd", "image 0.25.10", "keyring", "libsqlite3-sys", @@ -814,6 +815,7 @@ dependencies = [ "pulldown-cmark", "quick-xml 0.41.0", "rayon", + "regex", "reqwest", "rust-stemmers", "serde", @@ -849,6 +851,7 @@ dependencies = [ "chrono", "dirs 5.0.1", "fluent-syntax", + "futures", "iced", "muda", "objc2 0.6.4", @@ -3090,6 +3093,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "htmd" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eee9b00ee2e599b4f86507157e3db786e7a3319fc225f0e9584151dbea2291d" +dependencies = [ + "html5ever", + "markup5ever_rcdom", + "phf 0.13.1", +] + [[package]] name = "html-escape" version = "0.2.13" @@ -4341,6 +4355,18 @@ dependencies = [ "web_atoms", ] +[[package]] +name = "markup5ever_rcdom" +version = "0.38.0+unofficial" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "333171ccdf66e915257740d44e38ea5b1b19ce7b45d33cc35cb6f118fbd981ff" +dependencies = [ + "html5ever", + "markup5ever", + "tendril", + "xml5ever", +] + [[package]] name = "matchit" version = "0.8.4" @@ -7103,6 +7129,7 @@ dependencies = [ "parking_lot 0.12.5", "phf_shared 0.13.1", "precomputed-hash", + "serde", ] [[package]] @@ -9443,6 +9470,16 @@ version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" +[[package]] +name = "xml5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dc9559429edf0cd3f327cc0afd9d6b36fa8cec6d93107b7fbe64f806b5f2d9" +dependencies = [ + "log", + "markup5ever", +] + [[package]] name = "xmlwriter" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 0e6317d..c411b88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ anyhow = "1" tokio = { version = "1", features = ["full"] } axum = "0.8" walkdir = "2" +futures = "0.3" image = "0.25" rust-stemmers = "1" sys-locale = "0.3" @@ -38,6 +39,8 @@ pulldown-cmark = "0.13" liquid = "0.26" liquid-core = { version = "0.26", features = ["derive"] } quick-xml = "0.41" +htmd = "0.5.4" +regex = "1" rayon = "1.10" pagefind = "1.5.2" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } diff --git a/README.md b/README.md index 4ac8e33..f020e08 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ The project is under active development. Core blogging workflows are broadly ava - Native Iced desktop workspace with localized menus, tabs, sidebars, dialogs, tasks, and embedded Wry previews. - Post and translation authoring with draft/published lifecycle, metadata, tags, categories, links, media, and batch gallery-image import. - Media import, thumbnails, metadata translations, filters, validation, and post assignment. +- WordPress WXR migration with saved analyses, HTML-to-Markdown and shortcode conversion, conflict/taxonomy review, recoverable 500-item execution batches, media-parent linking, progress reporting, and optional AI-assisted taxonomy mapping. - Template and Lua script management with explicit syntax-check feedback, using a custom Ropey/Syntect/Cosmic Text editor and the documented, bDS2-signature-compatible project-scoped [`bds` host API](docs/scripting/API_REFERENCE.md) across utilities, rendered macros, and Blogmark transforms. - SQLite and filesystem persistence with frontmatter, sidecars, rebuild, metadata diff/repair, and FTS5 search. - Markdown/Liquid rendering with native macros, multilingual routes, feeds, sitemap, Pagefind, and incremental site generation through cancellable section task groups. diff --git a/RUST_PLAN_CORE.md b/RUST_PLAN_CORE.md index 952aee2..86e47b3 100644 --- a/RUST_PLAN_CORE.md +++ b/RUST_PLAN_CORE.md @@ -99,7 +99,7 @@ Available: - Configurable online and airplane-mode OpenAI-compatible endpoints. - Secure API-key storage through the operating-system keychain. - Model catalog discovery and model selection. -- Post translation, media translation, image alt text, post analysis, taxonomy analysis, and language detection. +- Post translation, media translation, image alt text, post analysis, taxonomy analysis, WordPress-import taxonomy mapping, and language detection. - Explicit offline gating and user-visible errors. - SQLite fields for input, output, cache-read, and cache-write token usage. diff --git a/RUST_PLAN_EXTENSION.md b/RUST_PLAN_EXTENSION.md index 42e72e9..e65e767 100644 --- a/RUST_PLAN_EXTENSION.md +++ b/RUST_PLAN_EXTENSION.md @@ -22,12 +22,14 @@ Open: - Expose `bds.sync` through the scripting API using the shared Git workflow. -### WordPress import — Open +### WordPress import — Complete -Open: +Done: -- WXR parsing, import analysis, recoverable execution, saved import definitions, and import UI. -- The existing media importer is core functionality and does not implement this workflow. +- Streaming, untrusted-input-safe WXR parsing for site metadata, posts, pages, attachments, categories, and tags. +- HTML5-to-Markdown conversion, WordPress shortcode conversion, complete new/update/conflict/duplicate/missing classification, date and macro analysis, and saved project-scoped import definitions. +- Localized native import sidebar/editor with WXR and uploads pickers, cached analysis reopening, conflict resolution, manual and airplane-gated AI taxonomy mapping, item review, live progress/ETA, and execution results. +- Taxonomy/posts/media/pages execution through core persistence engines in recoverable 500-item batches, including filesystem rollback, source metadata/status/timestamps, unique-slug import, overwrite/ignore behavior, and media-parent links. ### Conversational AI and agent tools — Open @@ -39,7 +41,7 @@ Open: - Agent integrations such as Claude Code and Copilot where required by the specs. - Replace the current Chat placeholders with the working feature. -Core endpoint settings, offline gating, key storage, model discovery, and six one-shot operations are already implemented. +Core endpoint settings, offline gating, key storage, model discovery, and seven one-shot operations are already implemented. ### Embeddings, semantic search, and duplicates — Open @@ -123,12 +125,11 @@ Open: ## Suggested Order -1. WordPress import. -2. CLI, MCP, and domain events. -3. Conversational AI and agent tools. -4. Embeddings and duplicate detection. -5. Documentation and menu UX. -6. Headless server and TUI. -7. A2UI after conversational AI exists. +1. CLI, MCP, and domain events. +2. Conversational AI and agent tools. +3. Embeddings and duplicate detection. +4. Documentation and menu UX. +5. Headless server and TUI. +6. A2UI after conversational AI exists. The order may change when an extension directly unlocks a concrete user workflow; it must not create a parallel data model or bypass core engines. diff --git a/crates/bds-core/Cargo.toml b/crates/bds-core/Cargo.toml index feb4ecc..88835fc 100644 --- a/crates/bds-core/Cargo.toml +++ b/crates/bds-core/Cargo.toml @@ -23,6 +23,8 @@ pulldown-cmark = { workspace = true } liquid = { workspace = true } liquid-core = { workspace = true } quick-xml = { workspace = true } +htmd = { workspace = true } +regex = { workspace = true } rayon = { workspace = true } pagefind = { workspace = true } reqwest = { workspace = true } diff --git a/crates/bds-core/src/db/queries/import_definition.rs b/crates/bds-core/src/db/queries/import_definition.rs new file mode 100644 index 0000000..b952be8 --- /dev/null +++ b/crates/bds-core/src/db/queries/import_definition.rs @@ -0,0 +1,62 @@ +use diesel::prelude::*; + +use crate::db::DbConnection; +use crate::db::schema::import_definitions; +use crate::model::ImportDefinition; + +pub fn insert_import_definition( + conn: &DbConnection, + definition: &ImportDefinition, +) -> QueryResult<()> { + conn.with(|connection| { + diesel::insert_into(import_definitions::table) + .values(definition.clone()) + .execute(connection) + .map(|_| ()) + }) +} + +pub fn get_import_definition(conn: &DbConnection, id: &str) -> QueryResult { + conn.with(|connection| { + import_definitions::table + .filter(import_definitions::id.eq(id)) + .select(ImportDefinition::as_select()) + .first(connection) + }) +} + +pub fn list_import_definitions( + conn: &DbConnection, + project_id: &str, +) -> QueryResult> { + conn.with(|connection| { + import_definitions::table + .filter(import_definitions::project_id.eq(project_id)) + .order(( + import_definitions::updated_at.desc(), + import_definitions::created_at.desc(), + )) + .select(ImportDefinition::as_select()) + .load(connection) + }) +} + +pub fn update_import_definition( + conn: &DbConnection, + definition: &ImportDefinition, +) -> QueryResult<()> { + conn.with(|connection| { + diesel::update(import_definitions::table.filter(import_definitions::id.eq(&definition.id))) + .set(definition.clone()) + .execute(connection) + .map(|_| ()) + }) +} + +pub fn delete_import_definition(conn: &DbConnection, id: &str) -> QueryResult<()> { + conn.with(|connection| { + diesel::delete(import_definitions::table.filter(import_definitions::id.eq(id))) + .execute(connection) + .map(|_| ()) + }) +} diff --git a/crates/bds-core/src/db/queries/mod.rs b/crates/bds-core/src/db/queries/mod.rs index a0b5ed7..1fb3d1d 100644 --- a/crates/bds-core/src/db/queries/mod.rs +++ b/crates/bds-core/src/db/queries/mod.rs @@ -1,4 +1,5 @@ pub mod generated_file_hash; +pub mod import_definition; pub mod media; pub mod media_translation; pub mod post; diff --git a/crates/bds-core/src/engine/ai.rs b/crates/bds-core/src/engine/ai.rs index ca00dd0..8577c88 100644 --- a/crates/bds-core/src/engine/ai.rs +++ b/crates/bds-core/src/engine/ai.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::time::Duration; use crate::db::DbConnection as Connection; @@ -83,6 +84,7 @@ pub struct AiModelInfo { #[derive(Debug, Clone, PartialEq, Eq)] pub enum OneShotOperation { AnalyzeTaxonomy, + MapImportTaxonomy, AnalyzePost, DetectLanguage, TranslatePost { target_language: String }, @@ -102,6 +104,12 @@ pub struct TaxonomySuggestion { pub categories: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportTaxonomyMapping { + pub category_mappings: BTreeMap, + pub tag_mappings: BTreeMap, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PostAnalysisResult { pub title: String, @@ -146,6 +154,7 @@ pub struct TokenUsage { #[derive(Debug, Clone, PartialEq, Eq)] pub enum OneShotResponse { Taxonomy(TaxonomySuggestion), + ImportTaxonomyMapping(ImportTaxonomyMapping), PostAnalysis(PostAnalysisResult), LanguageDetection(LanguageDetectionResult), Translation(TranslationResult), @@ -451,6 +460,7 @@ fn select_model( let selected = match operation { OneShotOperation::AnalyzeImage => settings.image_model.as_ref(), OneShotOperation::AnalyzeTaxonomy + | OneShotOperation::MapImportTaxonomy | OneShotOperation::AnalyzePost | OneShotOperation::DetectLanguage | OneShotOperation::TranslatePost { .. } @@ -475,6 +485,9 @@ fn build_system_prompt(base_prompt: &str, operation: &OneShotOperation) -> Strin OneShotOperation::AnalyzeTaxonomy => { "Return only JSON with tags and categories for the post." } + OneShotOperation::MapImportTaxonomy => { + "Map each imported category and tag to an existing equivalent when one exists. Return JSON objects keyed by each imported term; omit terms without a sound match." + } OneShotOperation::AnalyzePost => { "Return only JSON with title, excerpt, and slug suggestions for the post." } @@ -511,6 +524,11 @@ fn build_one_shot_user_content(request: &OneShotRequest) -> EngineResult serde_json::to_string(&request.content)? ) .into()), + OneShotOperation::MapImportTaxonomy => Ok(format!( + "Map these imported taxonomy terms to existing project terms without inventing terms: {}", + serde_json::to_string(&request.content)? + ) + .into()), OneShotOperation::AnalyzePost => Ok(format!( "Analyze this post and suggest title, excerpt, and slug: {}", serde_json::to_string(&request.content)? @@ -580,6 +598,24 @@ fn response_schema(operation: &OneShotOperation) -> (&'static str, Value) { "required": ["tags", "categories"] }), ), + OneShotOperation::MapImportTaxonomy => ( + "import_taxonomy_mapping", + json!({ + "type": "object", + "additionalProperties": false, + "properties": { + "category_mappings": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "tag_mappings": { + "type": "object", + "additionalProperties": { "type": "string" } + } + }, + "required": ["category_mappings", "tag_mappings"] + }), + ), OneShotOperation::AnalyzePost => ( "post_analysis", json!({ @@ -654,6 +690,9 @@ fn parse_one_shot_response( OneShotOperation::AnalyzeTaxonomy => { OneShotResponse::Taxonomy(serde_json::from_str(content)?) } + OneShotOperation::MapImportTaxonomy => { + OneShotResponse::ImportTaxonomyMapping(serde_json::from_str(content)?) + } OneShotOperation::AnalyzePost => { OneShotResponse::PostAnalysis(serde_json::from_str(content)?) } @@ -939,6 +978,33 @@ mod tests { ); } + #[test] + fn run_one_shot_supports_import_taxonomy_mapping_via_airplane_endpoint() { + let response = run_airplane_one_shot( + OneShotRequest { + operation: OneShotOperation::MapImportTaxonomy, + content: json!({ + "imported_categories": ["Old Engineering"], + "imported_tags": ["Old Rust"], + "existing_categories": ["Engineering"], + "existing_tags": ["Rust"] + }), + }, + r#"{"choices":[{"message":{"content":"{\"category_mappings\":{\"Old Engineering\":\"Engineering\"},\"tag_mappings\":{\"Old Rust\":\"Rust\"}}"}}],"usage":{"prompt_tokens":10,"completion_tokens":5}}"#, + ); + + assert_eq!( + response, + OneShotResponse::ImportTaxonomyMapping(ImportTaxonomyMapping { + category_mappings: BTreeMap::from([( + "Old Engineering".to_string(), + "Engineering".to_string(), + )]), + tag_mappings: BTreeMap::from([("Old Rust".to_string(), "Rust".to_string(),)]), + }) + ); + } + #[test] fn run_one_shot_supports_post_analysis_via_airplane_endpoint() { let response = run_airplane_one_shot( diff --git a/crates/bds-core/src/engine/media.rs b/crates/bds-core/src/engine/media.rs index 2691e2a..cf41483 100644 --- a/crates/bds-core/src/engine/media.rs +++ b/crates/bds-core/src/engine/media.rs @@ -63,6 +63,43 @@ pub fn import_media( author: Option<&str>, language: Option<&str>, tags: Vec, +) -> EngineResult { + import_media_at( + conn, + data_dir, + project_id, + source_path, + original_name, + title, + alt, + caption, + author, + language, + tags, + now_unix_ms(), + ) +} + +/// Import media while preserving a trusted source timestamp. Importers use +/// this path so the canonical date-based location and sidecar metadata match +/// the source system; interactive imports continue to use the current time. +#[expect( + clippy::too_many_arguments, + reason = "arguments are the user-supplied media metadata fields plus source timestamp" +)] +pub(crate) fn import_media_at( + conn: &Connection, + data_dir: &Path, + project_id: &str, + source_path: &Path, + original_name: &str, + title: Option<&str>, + alt: Option<&str>, + caption: Option<&str>, + author: Option<&str>, + language: Option<&str>, + tags: Vec, + created_at: i64, ) -> EngineResult { // Validate file type per spec let ext = Path::new(original_name) @@ -77,12 +114,10 @@ pub fn import_media( } let id = Uuid::new_v4().to_string(); - let now = now_unix_ms(); - let filename = format!("{id}.{ext}"); // Compute target directory and copy file - let dir_path = media_dir_path(now); + let dir_path = media_dir_path(created_at); let rel_file_path = format!("{dir_path}{filename}"); let abs_file_path = data_dir.join(&rel_file_path); @@ -128,8 +163,8 @@ pub fn import_media( sidecar_path: sidecar_rel.clone(), checksum: Some(checksum), tags, - created_at: now, - updated_at: now, + created_at, + updated_at: created_at, }; // Write sidecar diff --git a/crates/bds-core/src/engine/mod.rs b/crates/bds-core/src/engine/mod.rs index e9260d0..c36b77b 100644 --- a/crates/bds-core/src/engine/mod.rs +++ b/crates/bds-core/src/engine/mod.rs @@ -27,5 +27,6 @@ pub mod template_rebuild; pub mod validate_media; pub mod validate_site; pub mod validate_translations; +pub mod wordpress_import; pub use error::{EngineError, EngineResult}; diff --git a/crates/bds-core/src/engine/wordpress_import.rs b/crates/bds-core/src/engine/wordpress_import.rs new file mode 100644 index 0000000..04cfc7a --- /dev/null +++ b/crates/bds-core/src/engine/wordpress_import.rs @@ -0,0 +1,1827 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fs; +use std::io::BufReader; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Component, Path, PathBuf}; +use std::time::Instant; + +use chrono::{DateTime, Datelike, NaiveDateTime, Utc}; +use quick_xml::Reader; +use quick_xml::events::{BytesStart, Event}; +use regex::Regex; +use serde_json::json; +use uuid::Uuid; + +use crate::db::DbConnection as Connection; +use crate::db::queries::{ + import_definition as qd, media as qm, media_translation as qmt, post as qp, + post_translation as qpt, tag as qt, +}; +use crate::engine::{EngineError, EngineResult, ai, media, meta, post, post_media, tag}; +use crate::model::{ + ImportCandidate, ImportCounts, ImportDateBucket, ImportDefinition, ImportExecutionCounts, + ImportExecutionResult, ImportItemKind, ImportItemStatus, ImportMacroUsage, ImportPhase, + ImportProgress, ImportReport, ImportResolution, ImportedSite, Media, Post, TaxonomyCandidate, + TaxonomyKind, +}; +use crate::util::{ + atomic_write_str, content_hash, file_hash, media_sidecar_path, media_translation_sidecar_path, + now_unix_ms, post_file_path, slugify, +}; + +const TRANSACTION_BATCH_SIZE: usize = 500; + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct WxrSite { + pub title: String, + pub link: String, + pub description: String, + pub language: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct WxrTaxonomy { + pub name: String, + pub slug: String, + pub parent: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct WxrPost { + pub source_id: Option, + pub title: String, + pub slug: String, + pub content: String, + pub excerpt: String, + pub published_at: Option, + pub created_at: Option, + pub updated_at: Option, + pub creator: String, + pub status: String, + pub post_type: String, + pub categories: Vec, + pub tags: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct WxrMedia { + pub source_id: Option, + pub title: String, + pub url: String, + pub filename: String, + pub relative_path: String, + pub published_at: Option, + pub parent_source_id: Option, + pub mime_type: String, + pub description: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct WxrExport { + pub site: WxrSite, + pub posts: Vec, + pub pages: Vec, + pub media: Vec, + pub categories: Vec, + pub tags: Vec, +} + +#[derive(Debug, Default)] +struct RawItem { + post_id: String, + title: String, + content: String, + excerpt: String, + pub_date: String, + post_date: String, + post_modified: String, + creator: String, + status: String, + post_type: String, + post_name: String, + post_parent: String, + attachment_url: String, + categories: Vec, + tags: Vec, +} + +#[derive(Debug, Default)] +struct ParserState { + stack: Vec, + channel_seen: bool, + site: WxrSite, + categories: Vec, + tags: Vec, + items: Vec, + current_category: Option, + current_tag: Option, + current_item: Option, + current_taxonomy_domain: Option, + text: String, +} + +pub fn parse_wxr_file(path: &Path) -> EngineResult { + let file = fs::File::open(path)?; + parse_wxr_reader(Reader::from_reader(BufReader::new(file))) +} + +pub fn parse_wxr_xml(xml: &str) -> EngineResult { + parse_wxr_reader(Reader::from_reader(xml.as_bytes())) +} + +fn parse_wxr_reader(mut reader: Reader) -> EngineResult { + reader.config_mut().expand_empty_elements = true; + reader.config_mut().check_end_names = true; + let mut state = ParserState::default(); + let mut buffer = Vec::new(); + + loop { + match reader.read_event_into(&mut buffer) { + Ok(Event::Start(start)) => parser_start(&mut state, &start, reader.decoder())?, + Ok(Event::End(end)) => { + let name = reader + .decoder() + .decode(end.name().as_ref()) + .map_err(|error| EngineError::Parse(error.to_string()))? + .into_owned(); + parser_end(&mut state, &name); + } + Ok(Event::Text(text)) => { + state.text.push_str( + &text + .decode() + .map_err(|error| EngineError::Parse(error.to_string()))?, + ); + } + Ok(Event::CData(text)) => { + state.text.push_str( + &text + .decode() + .map_err(|error| EngineError::Parse(error.to_string()))?, + ); + } + Ok(Event::GeneralRef(reference)) => { + let name = reference + .decode() + .map_err(|error| EngineError::Parse(error.to_string()))?; + state.text.push_str(match name.as_ref() { + "amp" => "&", + "lt" => "<", + "gt" => ">", + "quot" => "\"", + "apos" => "'", + _ => "", + }); + } + Ok(Event::Eof) => break, + Ok(_) => {} + Err(error) => { + return Err(EngineError::Parse(format!( + "invalid WXR XML at byte {}: {error}", + reader.error_position() + ))); + } + } + buffer.clear(); + } + + if !state.channel_seen { + return Err(EngineError::Validation( + "invalid WXR file: no RSS channel element found".to_string(), + )); + } + if !state.stack.is_empty() { + return Err(EngineError::Parse( + "invalid WXR XML: document ended before all elements were closed".to_string(), + )); + } + Ok(build_wxr_export(state)) +} + +fn parser_start( + state: &mut ParserState, + start: &BytesStart<'_>, + decoder: quick_xml::encoding::Decoder, +) -> EngineResult<()> { + let name = decoder + .decode(start.name().as_ref()) + .map_err(|error| EngineError::Parse(error.to_string()))? + .into_owned(); + let parent = state.stack.last().map(String::as_str); + + match (parent, name.as_str()) { + (Some("rss"), "channel") => state.channel_seen = true, + (Some("channel"), "wp:category") => state.current_category = Some(WxrTaxonomy::default()), + (Some("channel"), "wp:tag") => state.current_tag = Some(WxrTaxonomy::default()), + (Some("channel"), "item") => state.current_item = Some(RawItem::default()), + (Some("item"), "category") => { + state.current_taxonomy_domain = attribute(start, b"domain", decoder)?; + } + _ => {} + } + state.stack.push(name); + state.text.clear(); + Ok(()) +} + +fn attribute( + start: &BytesStart<'_>, + expected: &[u8], + decoder: quick_xml::encoding::Decoder, +) -> EngineResult> { + for attribute in start.attributes() { + let attribute = attribute.map_err(|error| EngineError::Parse(error.to_string()))?; + if attribute.key.as_ref() == expected { + return Ok(Some( + attribute + .decoded_and_normalized_value(quick_xml::XmlVersion::Implicit1_0, decoder) + .map_err(|error| EngineError::Parse(error.to_string()))? + .into_owned(), + )); + } + } + Ok(None) +} + +fn parser_end(state: &mut ParserState, name: &str) { + let parent = state + .stack + .iter() + .rev() + .nth(1) + .map(String::as_str) + .unwrap_or(""); + let text = state.text.trim().to_string(); + + if state.current_category.is_none() + && state.current_tag.is_none() + && state.current_item.is_none() + && parent == "channel" + { + match name { + "title" => state.site.title = text.clone(), + "link" => state.site.link = text.clone(), + "description" => state.site.description = text.clone(), + "language" => state.site.language = text.clone(), + _ => {} + } + } + + if let Some(category) = state.current_category.as_mut() + && parent == "wp:category" + { + match name { + "wp:cat_name" => category.name = text.clone(), + "wp:category_nicename" => category.slug = text.clone(), + "wp:category_parent" => category.parent = text.clone(), + _ => {} + } + } + if parent == "channel" + && name == "wp:category" + && let Some(category) = state.current_category.take() + { + state.categories.push(category); + } + + if let Some(tag) = state.current_tag.as_mut() + && parent == "wp:tag" + { + match name { + "wp:tag_name" => tag.name = text.clone(), + "wp:tag_slug" => tag.slug = text.clone(), + _ => {} + } + } + if parent == "channel" + && name == "wp:tag" + && let Some(tag) = state.current_tag.take() + { + state.tags.push(tag); + } + + if let Some(item) = state.current_item.as_mut() + && parent == "item" + { + match name { + "title" => item.title = text.clone(), + "pubDate" => item.pub_date = text.clone(), + "dc:creator" => item.creator = text.clone(), + "content:encoded" => item.content = text.clone(), + "excerpt:encoded" => item.excerpt = text.clone(), + "wp:post_id" => item.post_id = text.clone(), + "wp:post_date" => item.post_date = text.clone(), + "wp:post_modified" => item.post_modified = text.clone(), + "wp:post_name" => item.post_name = text.clone(), + "wp:status" => item.status = text.clone(), + "wp:post_type" => item.post_type = text.clone(), + "wp:post_parent" => item.post_parent = text.clone(), + "wp:attachment_url" => item.attachment_url = text.clone(), + "category" => match state.current_taxonomy_domain.as_deref() { + Some("category") if !text.is_empty() => item.categories.push(text.clone()), + Some("post_tag") if !text.is_empty() => item.tags.push(text.clone()), + _ => {} + }, + _ => {} + } + } + if parent == "item" && name == "category" { + state.current_taxonomy_domain = None; + } + if parent == "channel" + && name == "item" + && let Some(item) = state.current_item.take() + { + state.items.push(item); + } + + state.stack.pop(); + state.text.clear(); +} + +fn build_wxr_export(state: ParserState) -> WxrExport { + let mut export = WxrExport { + site: state.site, + categories: state.categories, + tags: state.tags, + ..WxrExport::default() + }; + for item in state.items { + if item.post_type == "attachment" { + let filename = Path::new(&item.attachment_url) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("") + .to_string(); + let relative_path = item + .attachment_url + .split_once("/wp-content/uploads/") + .map(|(_, suffix)| suffix.to_string()) + .unwrap_or_else(|| filename.clone()); + let extension = Path::new(&filename) + .extension() + .and_then(|extension| extension.to_str()) + .unwrap_or("") + .to_string(); + export.media.push(WxrMedia { + source_id: parse_id(&item.post_id), + title: item.title, + url: item.attachment_url, + filename, + relative_path, + published_at: parse_timestamp(&item.pub_date), + parent_source_id: parse_id(&item.post_parent), + mime_type: crate::util::thumbnail::mime_from_extension(&extension).to_string(), + description: item.content, + }); + } else { + let post = WxrPost { + source_id: parse_id(&item.post_id), + title: item.title, + slug: item.post_name, + content: item.content, + excerpt: item.excerpt, + published_at: parse_timestamp(&item.pub_date), + created_at: parse_timestamp(&item.post_date) + .or_else(|| parse_timestamp(&item.pub_date)), + updated_at: parse_timestamp(&item.post_modified) + .or_else(|| parse_timestamp(&item.post_date)) + .or_else(|| parse_timestamp(&item.pub_date)), + creator: item.creator, + status: item.status, + post_type: item.post_type.clone(), + categories: item.categories, + tags: item.tags, + }; + if item.post_type == "page" { + export.pages.push(post); + } else if item.post_type == "post" { + export.posts.push(post); + } + } + } + export +} + +fn parse_id(value: &str) -> Option { + value.parse().ok().filter(|value| *value != 0) +} + +fn parse_timestamp(value: &str) -> Option { + let value = value.trim(); + if value.is_empty() { + return None; + } + DateTime::parse_from_rfc2822(value) + .map(|date| date.timestamp_millis()) + .or_else(|_| DateTime::parse_from_rfc3339(value).map(|date| date.timestamp_millis())) + .or_else(|_| { + NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S") + .map(|date| date.and_utc().timestamp_millis()) + }) + .ok() +} + +pub fn analyze_wxr( + conn: &Connection, + data_dir: &Path, + project_id: &str, + wxr_file: &Path, + uploads_folder: Option<&Path>, + mut progress: Option<&mut dyn FnMut(ImportProgress)>, +) -> EngineResult { + let export = parse_wxr_file(wxr_file)?; + let existing_posts = qp::list_posts_by_project(conn, project_id)?; + let existing_media = qm::list_media_by_project(conn, project_id)?; + let existing_tags = qt::list_tags_by_project(conn, project_id)?; + let existing_categories = meta::read_categories_json(data_dir).unwrap_or_default(); + + let posts_by_slug: HashMap = existing_posts + .iter() + .map(|post| (post.slug.clone(), post)) + .collect(); + let posts_by_checksum: HashMap = existing_posts + .iter() + .filter_map(|post| { + existing_post_body(post, data_dir).map(|body| (content_hash(body.as_bytes()), post)) + }) + .collect(); + let post_checksums = existing_posts + .iter() + .filter_map(|post| { + existing_post_body(post, data_dir) + .map(|body| (post.id.clone(), content_hash(body.as_bytes()))) + }) + .collect::>(); + let media_by_name: HashMap = existing_media + .iter() + .map(|media| (media.original_name.to_lowercase(), media)) + .collect(); + let media_by_checksum: HashMap = existing_media + .iter() + .filter_map(|media| media.checksum.clone().map(|checksum| (checksum, media))) + .collect(); + + notify( + &mut progress, + ImportPhase::Posts, + 0, + export.posts.len(), + "analyzing_posts", + None, + ); + let posts = export + .posts + .iter() + .map(|item| { + analyze_post( + item, + ImportItemKind::Post, + &posts_by_slug, + &posts_by_checksum, + &post_checksums, + ) + }) + .collect::>>()?; + notify( + &mut progress, + ImportPhase::Posts, + posts.len(), + posts.len(), + "analyzed_posts", + None, + ); + notify( + &mut progress, + ImportPhase::Pages, + 0, + export.pages.len(), + "analyzing_pages", + None, + ); + let pages = export + .pages + .iter() + .map(|item| { + analyze_post( + item, + ImportItemKind::Page, + &posts_by_slug, + &posts_by_checksum, + &post_checksums, + ) + }) + .collect::>>()?; + notify( + &mut progress, + ImportPhase::Pages, + pages.len(), + pages.len(), + "analyzed_pages", + None, + ); + notify( + &mut progress, + ImportPhase::Media, + 0, + export.media.len(), + "analyzing_media", + None, + ); + let media = export + .media + .iter() + .map(|item| analyze_media(item, uploads_folder, &media_by_name, &media_by_checksum)) + .collect::>(); + notify( + &mut progress, + ImportPhase::Media, + media.len(), + media.len(), + "analyzed_media", + None, + ); + + let category_names = existing_categories + .iter() + .map(|name| name.to_lowercase()) + .collect::>(); + let tag_names = existing_tags + .iter() + .map(|tag| tag.name.to_lowercase()) + .collect::>(); + let taxonomies = export + .categories + .iter() + .map(|item| TaxonomyCandidate { + kind: TaxonomyKind::Category, + name: item.name.clone(), + slug: nonempty(&item.slug), + exists_in_project: category_names.contains(&item.name.to_lowercase()), + mapped_to: None, + }) + .chain(export.tags.iter().map(|item| TaxonomyCandidate { + kind: TaxonomyKind::Tag, + name: item.name.clone(), + slug: nonempty(&item.slug), + exists_in_project: tag_names.contains(&item.name.to_lowercase()), + mapped_to: None, + })) + .collect::>(); + notify( + &mut progress, + ImportPhase::Taxonomy, + taxonomies.len(), + taxonomies.len(), + "analyzed_taxonomy", + None, + ); + + Ok(ImportReport { + source_file: wxr_file.to_string_lossy().to_string(), + uploads_folder: uploads_folder.map(|path| path.to_string_lossy().to_string()), + site: ImportedSite { + title: export.site.title, + url: nonempty(&export.site.link), + language: nonempty(&export.site.language), + }, + post_counts: counts(&posts), + page_counts: counts(&pages), + media_counts: counts(&media), + date_distribution: date_distribution(&posts, &pages, &media), + macros: analyze_macros(export.posts.iter().chain(&export.pages)), + posts, + pages, + media, + taxonomies, + }) +} + +fn existing_post_body(post: &Post, data_dir: &Path) -> Option { + post.content + .clone() + .or_else(|| post.published_content.clone()) + .or_else(|| { + (!post.file_path.is_empty()) + .then(|| fs::read_to_string(data_dir.join(&post.file_path)).ok()) + .flatten() + .and_then(|raw| crate::util::frontmatter::read_post_file(&raw).ok()) + .map(|(_, body)| body) + }) +} + +fn analyze_post( + item: &WxrPost, + kind: ImportItemKind, + posts_by_slug: &HashMap, + posts_by_checksum: &HashMap, + post_checksums: &HashMap, +) -> EngineResult { + let content = html_to_markdown(&item.content)?; + let checksum = content_hash(content.as_bytes()); + let slug = if item.slug.trim().is_empty() { + slugify(&item.title) + } else { + item.slug.clone() + }; + let existing_by_slug = posts_by_slug.get(&slug).copied(); + let existing_by_checksum = posts_by_checksum.get(&checksum).copied(); + let (status, existing_id) = if let Some(existing) = existing_by_slug { + if post_checksums.get(&existing.id) == Some(&checksum) { + (ImportItemStatus::Update, Some(existing.id.clone())) + } else { + (ImportItemStatus::Conflict, Some(existing.id.clone())) + } + } else if let Some(existing) = existing_by_checksum { + ( + ImportItemStatus::ContentDuplicate, + Some(existing.id.clone()), + ) + } else { + (ImportItemStatus::New, None) + }; + Ok(ImportCandidate { + kind, + source_id: item.source_id, + title: item.title.clone(), + slug: Some(slug), + filename: None, + relative_path: None, + status, + resolution: (status == ImportItemStatus::Conflict).then_some(ImportResolution::Ignore), + existing_id, + author: nonempty(&item.creator), + excerpt: nonempty(&item.excerpt), + content: Some(content), + source_status: nonempty(&item.status), + categories: item.categories.clone(), + tags: item.tags.clone(), + source_path: None, + parent_source_id: None, + created_at: item.created_at, + updated_at: item.updated_at, + published_at: item.published_at, + checksum: Some(checksum), + mime_type: None, + description: None, + }) +} + +fn analyze_media( + item: &WxrMedia, + uploads_folder: Option<&Path>, + media_by_name: &HashMap, + media_by_checksum: &HashMap, +) -> ImportCandidate { + let source_path = + uploads_folder.and_then(|folder| safe_upload_path(folder, &item.relative_path)); + let checksum = source_path + .as_ref() + .filter(|path| path.is_file()) + .and_then(|path| file_hash(path).ok()); + let existing_by_name = media_by_name.get(&item.filename.to_lowercase()).copied(); + let existing_by_checksum = checksum + .as_ref() + .and_then(|checksum| media_by_checksum.get(checksum).copied()); + let (status, existing_id) = if checksum.is_none() { + (ImportItemStatus::Missing, None) + } else if let Some(existing) = existing_by_name { + if existing.checksum == checksum { + (ImportItemStatus::Update, Some(existing.id.clone())) + } else { + (ImportItemStatus::Conflict, Some(existing.id.clone())) + } + } else if let Some(existing) = existing_by_checksum { + ( + ImportItemStatus::ContentDuplicate, + Some(existing.id.clone()), + ) + } else { + (ImportItemStatus::New, None) + }; + ImportCandidate { + kind: ImportItemKind::Media, + source_id: item.source_id, + title: item.title.clone(), + slug: None, + filename: Some(item.filename.clone()), + relative_path: Some(item.relative_path.clone()), + status, + resolution: (status == ImportItemStatus::Conflict).then_some(ImportResolution::Ignore), + existing_id, + author: None, + excerpt: None, + content: None, + source_status: None, + categories: Vec::new(), + tags: Vec::new(), + source_path: source_path.map(|path| path.to_string_lossy().to_string()), + parent_source_id: item.parent_source_id, + created_at: item.published_at, + updated_at: item.published_at, + published_at: item.published_at, + checksum, + mime_type: nonempty(&item.mime_type), + description: nonempty(&item.description), + } +} + +fn safe_upload_path(folder: &Path, relative: &str) -> Option { + let relative = Path::new(relative); + if relative + .components() + .all(|part| matches!(part, Component::Normal(_))) + { + Some(folder.join(relative)) + } else { + None + } +} + +fn counts(items: &[ImportCandidate]) -> ImportCounts { + let mut result = ImportCounts::default(); + for item in items { + match item.status { + ImportItemStatus::New => result.new_count += 1, + ImportItemStatus::Update => result.update_count += 1, + ImportItemStatus::Conflict => result.conflict_count += 1, + ImportItemStatus::ContentDuplicate => result.duplicate_count += 1, + ImportItemStatus::Missing => result.missing_count += 1, + } + } + result +} + +fn date_distribution( + posts: &[ImportCandidate], + pages: &[ImportCandidate], + media: &[ImportCandidate], +) -> Vec { + let mut years = BTreeMap::::new(); + for item in posts.iter().chain(pages) { + if let Some(timestamp) = item.created_at.or(item.published_at) + && let Some(date) = DateTime::::from_timestamp_millis(timestamp) + { + years.entry(date.year()).or_default().0 += 1; + } + } + for item in media { + if let Some(timestamp) = item.created_at + && let Some(date) = DateTime::::from_timestamp_millis(timestamp) + { + years.entry(date.year()).or_default().1 += 1; + } + } + years + .into_iter() + .map(|(year, (post_count, media_count))| ImportDateBucket { + year, + post_count, + media_count, + }) + .collect() +} + +fn html_to_markdown(html: &str) -> EngineResult { + let converted = htmd::convert(&transform_shortcodes(html)).map_err(|error| { + EngineError::Parse(format!("HTML to Markdown conversion failed: {error}")) + })?; + let converted = converted + .replace('\u{e000}', "[[") + .replace('\u{e001}', "]]"); + Ok(Regex::new(r"\n{3,}") + .unwrap() + .replace_all(converted.trim(), "\n\n") + .into_owned()) +} + +fn transform_shortcodes(value: &str) -> String { + let bytes = value.as_bytes(); + let mut output = String::with_capacity(value.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'[' + && bytes.get(index.wrapping_sub(1)) != Some(&b'[') + && bytes + .get(index + 1) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + && let Some(end_offset) = value[index + 1..].find(']') + { + let end = index + 1 + end_offset; + if bytes.get(end + 1) != Some(&b']') { + let inner = value[index + 1..end].trim().trim_end_matches('/').trim(); + output.push('\u{e000}'); + output.push_str(inner); + output.push('\u{e001}'); + index = end + 1; + continue; + } + } + let character = value[index..].chars().next().unwrap(); + output.push(character); + index += character.len_utf8(); + } + output +} + +fn analyze_macros<'a>(items: impl Iterator) -> Vec { + let shortcode = Regex::new(r#"\[(\w+)([^\]]*?)(?:\s*/)?\]"#).unwrap(); + let parameter = Regex::new(r#"(\w+)=(?:"([^"]*)"|'([^']*)'|([^\s\]"']+))"#).unwrap(); + let mut macros = BTreeMap::::new(); + for item in items { + for capture in shortcode.captures_iter(&item.content) { + let name = capture[1].to_lowercase(); + let parameters = parameter + .captures_iter(capture.get(2).map(|value| value.as_str()).unwrap_or("")) + .map(|capture| { + let value = capture + .get(2) + .or_else(|| capture.get(3)) + .or_else(|| capture.get(4)) + .map(|value| value.as_str()) + .unwrap_or(""); + (capture[1].to_string(), value.to_string()) + }) + .collect::>(); + let entry = macros + .entry(name.clone()) + .or_insert_with(|| ImportMacroUsage { + name, + total_count: 0, + post_slugs: Vec::new(), + parameters: Vec::new(), + }); + entry.total_count += 1; + if !item.slug.is_empty() && !entry.post_slugs.contains(&item.slug) { + entry.post_slugs.push(item.slug.clone()); + } + if !entry.parameters.contains(¶meters) { + entry.parameters.push(parameters); + } + } + } + macros.into_values().collect() +} + +fn nonempty(value: &str) -> Option { + (!value.trim().is_empty()).then(|| value.to_string()) +} + +pub fn create_definition( + conn: &Connection, + project_id: &str, + name: &str, +) -> EngineResult { + let now = now_unix_ms(); + let definition = ImportDefinition { + id: Uuid::new_v4().to_string(), + project_id: project_id.to_string(), + name: name.to_string(), + wxr_file_path: None, + uploads_folder_path: None, + last_analysis_result: None, + created_at: now, + updated_at: now, + }; + qd::insert_import_definition(conn, &definition)?; + Ok(definition) +} + +pub fn get_definition(conn: &Connection, id: &str) -> EngineResult { + qd::get_import_definition(conn, id).map_err(Into::into) +} + +pub fn list_definitions( + conn: &Connection, + project_id: &str, +) -> EngineResult> { + qd::list_import_definitions(conn, project_id).map_err(Into::into) +} + +pub fn update_definition( + conn: &Connection, + id: &str, + name: Option<&str>, + wxr_file_path: Option>, + uploads_folder_path: Option>, + analysis: Option>, +) -> EngineResult { + let mut definition = get_definition(conn, id)?; + if let Some(name) = name { + definition.name = name.to_string(); + } + if let Some(path) = wxr_file_path { + definition.wxr_file_path = path.map(|path| path.to_string_lossy().to_string()); + } + if let Some(path) = uploads_folder_path { + definition.uploads_folder_path = path.map(|path| path.to_string_lossy().to_string()); + } + if let Some(report) = analysis { + definition.last_analysis_result = report.map(serde_json::to_string).transpose()?; + } + definition.updated_at = now_unix_ms(); + qd::update_import_definition(conn, &definition)?; + Ok(definition) +} + +pub fn delete_definition(conn: &Connection, id: &str) -> EngineResult<()> { + qd::delete_import_definition(conn, id).map_err(Into::into) +} + +pub fn set_conflict_resolution( + report: &mut ImportReport, + kind: ImportItemKind, + identity: &str, + resolution: ImportResolution, +) -> EngineResult<()> { + let candidates = match kind { + ImportItemKind::Post => &mut report.posts, + ImportItemKind::Page => &mut report.pages, + ImportItemKind::Media => &mut report.media, + }; + let candidate = candidates + .iter_mut() + .find(|candidate| { + candidate.status == ImportItemStatus::Conflict + && candidate.slug.as_deref().or(candidate.filename.as_deref()) == Some(identity) + }) + .ok_or_else(|| EngineError::NotFound(format!("import conflict {identity}")))?; + candidate.resolution = Some(resolution); + Ok(()) +} + +pub fn set_taxonomy_mapping( + report: &mut ImportReport, + kind: TaxonomyKind, + source_name: &str, + target_name: Option<&str>, +) -> EngineResult<()> { + let candidate = report + .taxonomies + .iter_mut() + .find(|candidate| candidate.kind == kind && candidate.name == source_name) + .ok_or_else(|| EngineError::NotFound(format!("import taxonomy {source_name}")))?; + candidate.mapped_to = target_name + .map(str::trim) + .filter(|target| !target.is_empty()) + .map(str::to_string); + Ok(()) +} + +pub fn auto_map_taxonomy( + conn: &Connection, + data_dir: &Path, + project_id: &str, + offline_mode: bool, + report: &mut ImportReport, +) -> EngineResult { + let existing_categories = meta::read_categories_json(data_dir).unwrap_or_default(); + let existing_tags = qt::list_tags_by_project(conn, project_id)? + .into_iter() + .map(|tag| tag.name) + .collect::>(); + let imported_categories = report + .taxonomies + .iter() + .filter(|item| item.kind == TaxonomyKind::Category && !item.exists_in_project) + .map(|item| item.name.clone()) + .collect::>(); + let imported_tags = report + .taxonomies + .iter() + .filter(|item| item.kind == TaxonomyKind::Tag && !item.exists_in_project) + .map(|item| item.name.clone()) + .collect::>(); + let request = ai::OneShotRequest { + operation: ai::OneShotOperation::MapImportTaxonomy, + content: json!({ + "imported_categories": imported_categories, + "imported_tags": imported_tags, + "existing_categories": existing_categories, + "existing_tags": existing_tags, + }), + }; + let (response, _) = ai::run_one_shot(conn, offline_mode, &request)?; + let ai::OneShotResponse::ImportTaxonomyMapping(mapping) = response else { + return Err(EngineError::Parse( + "AI returned the wrong response for import taxonomy mapping".to_string(), + )); + }; + Ok(apply_ai_taxonomy_mapping( + report, + &mapping, + &existing_categories, + &existing_tags, + )) +} + +fn apply_ai_taxonomy_mapping( + report: &mut ImportReport, + mapping: &ai::ImportTaxonomyMapping, + existing_categories: &[String], + existing_tags: &[String], +) -> usize { + let mut count = 0; + for item in &mut report.taxonomies { + let (mapped, existing) = match item.kind { + TaxonomyKind::Category => ( + mapping.category_mappings.get(&item.name), + existing_categories, + ), + TaxonomyKind::Tag => (mapping.tag_mappings.get(&item.name), existing_tags), + }; + let canonical = mapped.and_then(|mapped| { + existing + .iter() + .find(|candidate| candidate.eq_ignore_ascii_case(mapped.trim())) + }); + if let Some(canonical) = canonical { + item.mapped_to = Some(canonical.clone()); + count += 1; + } + } + count +} + +pub fn empty_report() -> ImportReport { + ImportReport::default() +} + +pub fn taxonomy_candidate( + kind: TaxonomyKind, + name: &str, + exists_in_project: bool, +) -> TaxonomyCandidate { + TaxonomyCandidate { + kind, + name: name.to_string(), + slug: None, + exists_in_project, + mapped_to: None, + } +} + +pub fn execute_import( + conn: &Connection, + data_dir: &Path, + project_id: &str, + report: &ImportReport, + default_author: Option<&str>, + mut progress: Option<&mut dyn FnMut(ImportProgress)>, +) -> EngineResult { + let started = Instant::now(); + let taxonomy_map = taxonomy_mapping(conn, data_dir, project_id, report)?; + let mut result = ImportExecutionResult::default(); + let mut post_ids = HashMap::new(); + + notify( + &mut progress, + ImportPhase::Taxonomy, + 0, + report.taxonomies.len(), + "creating_taxonomy", + Some(started), + ); + run_batches( + conn, + data_dir, + &report.taxonomies, + |_| { + Ok(vec![ + PathBuf::from("meta/categories.json"), + PathBuf::from("meta/category-meta.json"), + PathBuf::from("meta/tags.json"), + ]) + }, + |item| { + let imported = if item.exists_in_project || item.mapped_to.is_some() { + false + } else { + match item.kind { + TaxonomyKind::Category => meta::add_category(data_dir, &item.name)?, + TaxonomyKind::Tag => { + tag::create_tag(conn, data_dir, project_id, &item.name, None)?; + } + } + true + }; + if imported { + result.taxonomy.imported += 1; + } else { + result.taxonomy.skipped += 1; + } + let current = result.taxonomy.imported + result.taxonomy.skipped; + notify( + &mut progress, + ImportPhase::Taxonomy, + current, + report.taxonomies.len(), + &item.name, + Some(started), + ); + Ok(()) + }, + )?; + + execute_post_phase( + conn, + data_dir, + project_id, + &report.posts, + default_author, + &taxonomy_map, + &mut result.posts, + &mut post_ids, + ImportPhase::Posts, + &mut progress, + started, + )?; + execute_media_phase( + conn, + data_dir, + project_id, + &report.media, + default_author, + &post_ids, + &mut result.media, + &mut progress, + started, + )?; + execute_post_phase( + conn, + data_dir, + project_id, + &report.pages, + default_author, + &taxonomy_map, + &mut result.pages, + &mut post_ids, + ImportPhase::Pages, + &mut progress, + started, + )?; + notify( + &mut progress, + ImportPhase::Complete, + 1, + 1, + "import_complete", + Some(started), + ); + Ok(result) +} + +fn taxonomy_mapping( + conn: &Connection, + data_dir: &Path, + project_id: &str, + report: &ImportReport, +) -> EngineResult> { + let categories = meta::read_categories_json(data_dir).unwrap_or_default(); + let tags = qt::list_tags_by_project(conn, project_id)?; + let mut mapping = HashMap::new(); + for item in &report.taxonomies { + let requested = item.mapped_to.as_deref().unwrap_or(&item.name); + let canonical = match item.kind { + TaxonomyKind::Category => categories + .iter() + .find(|name| name.eq_ignore_ascii_case(requested)) + .cloned(), + TaxonomyKind::Tag => tags + .iter() + .find(|tag| tag.name.eq_ignore_ascii_case(requested)) + .map(|tag| tag.name.clone()), + }; + if item.mapped_to.is_some() && canonical.is_none() { + return Err(EngineError::Validation(format!( + "import taxonomy mapping target does not exist: {requested}" + ))); + } + mapping.insert( + (item.kind, item.name.to_lowercase()), + canonical.unwrap_or_else(|| item.name.clone()), + ); + } + Ok(mapping) +} + +#[expect(clippy::too_many_arguments, reason = "phase orchestration inputs")] +fn execute_post_phase( + conn: &Connection, + data_dir: &Path, + project_id: &str, + items: &[ImportCandidate], + default_author: Option<&str>, + taxonomy: &HashMap<(TaxonomyKind, String), String>, + counts: &mut ImportExecutionCounts, + post_ids: &mut HashMap, + phase: ImportPhase, + progress: &mut Option<&mut dyn FnMut(ImportProgress)>, + started: Instant, +) -> EngineResult<()> { + notify( + progress, + phase, + 0, + items.len(), + match phase { + ImportPhase::Posts => "importing_posts", + ImportPhase::Pages => "importing_pages", + _ => "importing_items", + }, + Some(started), + ); + run_batches( + conn, + data_dir, + items, + |item| touched_post_paths(conn, item), + |item| { + if !should_import(item) { + counts.skipped += 1; + } else { + let post = import_post_item( + conn, + data_dir, + project_id, + item, + default_author, + taxonomy, + phase == ImportPhase::Pages, + )?; + counts.imported += 1; + if let Some(source_id) = item.source_id { + post_ids.insert(source_id, post.id); + } + } + notify( + progress, + phase, + counts.imported + counts.skipped, + items.len(), + &item.title, + Some(started), + ); + Ok(()) + }, + ) +} + +fn import_post_item( + conn: &Connection, + data_dir: &Path, + project_id: &str, + item: &ImportCandidate, + default_author: Option<&str>, + taxonomy: &HashMap<(TaxonomyKind, String), String>, + page: bool, +) -> EngineResult { + let author = item.author.as_deref().or(default_author); + let tags = resolve_terms(&item.tags, TaxonomyKind::Tag, taxonomy); + let mut categories = resolve_terms(&item.categories, TaxonomyKind::Category, taxonomy); + if page + && !categories + .iter() + .any(|category| category.eq_ignore_ascii_case("page")) + { + categories.push("page".to_string()); + } + let content = item.content.as_deref().unwrap_or(""); + let mut imported = if item.status == ImportItemStatus::Conflict + && item.resolution == Some(ImportResolution::Overwrite) + { + let id = item + .existing_id + .as_deref() + .ok_or_else(|| EngineError::NotFound("conflict target".to_string()))?; + post::update_post( + conn, + data_dir, + id, + Some(&item.title), + None, + Some(item.excerpt.as_deref()), + Some(content), + Some(tags), + Some(categories), + Some(author), + None, + None, + None, + )? + } else { + let created = post::create_post( + conn, + data_dir, + project_id, + &item.title, + Some(content), + tags, + categories, + author, + None, + None, + )?; + let desired_slug = item.slug.as_deref().unwrap_or(&created.slug); + if item.status != ImportItemStatus::Conflict + || item.resolution != Some(ImportResolution::Import) + { + post::update_post( + conn, + data_dir, + &created.id, + None, + Some(desired_slug), + None, + None, + None, + None, + None, + None, + None, + None, + )? + } else { + created + } + }; + + let previous_file_path = imported.file_path.clone(); + let previous_translation_paths = qpt::list_post_translations_by_post(conn, &imported.id)? + .into_iter() + .filter(|translation| !translation.file_path.is_empty()) + .map(|translation| translation.file_path) + .collect::>(); + imported.created_at = item.created_at.unwrap_or(imported.created_at); + imported.updated_at = item.updated_at.unwrap_or(imported.created_at); + imported.published_at = if item.source_status.as_deref() == Some("publish") { + item.published_at.or(Some(imported.created_at)) + } else { + None + }; + imported.checksum = item.checksum.clone(); + qp::update_post(conn, &imported)?; + if item.source_status.as_deref() == Some("publish") { + imported = post::publish_post(conn, data_dir, &imported.id)?; + let body = imported.published_content.clone().unwrap_or_default(); + imported.created_at = item.created_at.unwrap_or(imported.created_at); + imported.updated_at = item.updated_at.unwrap_or(imported.created_at); + imported.published_at = item.published_at.or(Some(imported.created_at)); + imported.file_path = post_file_path(imported.created_at, &imported.slug); + let serialized = crate::util::frontmatter::write_post_file(&imported, &body); + imported.checksum = Some(content_hash(serialized.as_bytes())); + atomic_write_str(&data_dir.join(&imported.file_path), &serialized)?; + if !previous_file_path.is_empty() && previous_file_path != imported.file_path { + remove_file_if_present(&data_dir.join(previous_file_path))?; + } + let current_translation_paths = qpt::list_post_translations_by_post(conn, &imported.id)? + .into_iter() + .map(|translation| translation.file_path) + .collect::>(); + for previous_path in previous_translation_paths { + if !current_translation_paths.contains(&previous_path) { + remove_file_if_present(&data_dir.join(previous_path))?; + } + } + qp::update_post(conn, &imported)?; + } + Ok(imported) +} + +fn resolve_terms( + terms: &[String], + kind: TaxonomyKind, + taxonomy: &HashMap<(TaxonomyKind, String), String>, +) -> Vec { + let mut result = Vec::new(); + for term in terms { + let resolved = taxonomy + .get(&(kind, term.to_lowercase())) + .cloned() + .unwrap_or_else(|| term.clone()); + if !result + .iter() + .any(|item: &String| item.eq_ignore_ascii_case(&resolved)) + { + result.push(resolved); + } + } + result +} + +#[expect(clippy::too_many_arguments, reason = "phase orchestration inputs")] +fn execute_media_phase( + conn: &Connection, + data_dir: &Path, + project_id: &str, + items: &[ImportCandidate], + default_author: Option<&str>, + post_ids: &HashMap, + counts: &mut ImportExecutionCounts, + progress: &mut Option<&mut dyn FnMut(ImportProgress)>, + started: Instant, +) -> EngineResult<()> { + notify( + progress, + ImportPhase::Media, + 0, + items.len(), + "importing_media", + Some(started), + ); + run_batches( + conn, + data_dir, + items, + |item| touched_media_paths(conn, data_dir, item), + |item| { + if !should_import(item) { + counts.skipped += 1; + } else { + let source = item + .source_path + .as_deref() + .map(Path::new) + .filter(|path| path.is_file()) + .ok_or_else(|| EngineError::NotFound(format!("media source {}", item.title)))?; + let filename = item.filename.as_deref().unwrap_or("imported-media"); + let imported = if item.status == ImportItemStatus::Conflict + && item.resolution == Some(ImportResolution::Overwrite) + { + let id = item.existing_id.as_deref().ok_or_else(|| { + EngineError::NotFound("media conflict target".to_string()) + })?; + media::replace_media_file(conn, data_dir, id, source)?; + media::update_media( + conn, + data_dir, + id, + Some(Some(&item.title)), + Some(item.description.as_deref()), + None, + Some(default_author), + None, + None, + )? + } else { + media::import_media_at( + conn, + data_dir, + project_id, + source, + filename, + nonempty(&item.title).as_deref(), + item.description.as_deref(), + None, + default_author, + None, + Vec::new(), + item.created_at.unwrap_or_else(now_unix_ms), + )? + }; + if let Some(parent_id) = item.parent_source_id.and_then(|id| post_ids.get(&id)) { + post_media::link_media_to_post( + conn, + data_dir, + project_id, + parent_id, + &imported.id, + 0, + )?; + } + counts.imported += 1; + } + notify( + progress, + ImportPhase::Media, + counts.imported + counts.skipped, + items.len(), + item.filename.as_deref().unwrap_or(&item.title), + Some(started), + ); + Ok(()) + }, + ) +} + +fn should_import(item: &ImportCandidate) -> bool { + item.status == ImportItemStatus::New + || (item.status == ImportItemStatus::Conflict + && matches!( + item.resolution, + Some(ImportResolution::Overwrite | ImportResolution::Import) + )) +} + +fn touched_post_paths(conn: &Connection, item: &ImportCandidate) -> EngineResult> { + if item.status != ImportItemStatus::Conflict + || item.resolution != Some(ImportResolution::Overwrite) + { + return Ok(Vec::new()); + } + let id = item + .existing_id + .as_deref() + .ok_or_else(|| EngineError::NotFound("post conflict target".to_string()))?; + let post = qp::get_post_by_id(conn, id)?; + let mut paths = Vec::new(); + if !post.file_path.is_empty() { + paths.push(PathBuf::from(post.file_path)); + } + paths.extend( + qpt::list_post_translations_by_post(conn, id)? + .into_iter() + .filter(|translation| !translation.file_path.is_empty()) + .map(|translation| PathBuf::from(translation.file_path)), + ); + Ok(paths) +} + +fn touched_media_paths( + conn: &Connection, + data_dir: &Path, + item: &ImportCandidate, +) -> EngineResult> { + if item.status != ImportItemStatus::Conflict + || item.resolution != Some(ImportResolution::Overwrite) + { + return Ok(Vec::new()); + } + let id = item + .existing_id + .as_deref() + .ok_or_else(|| EngineError::NotFound("media conflict target".to_string()))?; + let media = qm::get_media_by_id(conn, id)?; + let mut paths = vec![PathBuf::from(&media.file_path)]; + paths.push(PathBuf::from(if media.sidecar_path.is_empty() { + media_sidecar_path(&media.file_path) + } else { + media.sidecar_path.clone() + })); + paths.extend( + qmt::list_media_translations_by_media(conn, id)? + .into_iter() + .map(|translation| { + PathBuf::from(media_translation_sidecar_path( + &media.file_path, + &translation.language, + )) + }), + ); + + let thumbnail_dir = data_dir.join("thumbnails").join(&id[..2.min(id.len())]); + if thumbnail_dir.is_dir() { + for entry in fs::read_dir(&thumbnail_dir)? { + let entry = entry?; + let filename = entry.file_name(); + if entry.file_type()?.is_file() + && filename + .to_str() + .is_some_and(|filename| filename.starts_with(&format!("{id}-"))) + { + paths.push( + entry + .path() + .strip_prefix(data_dir) + .map_err(|error| EngineError::Validation(error.to_string()))? + .to_path_buf(), + ); + } + } + } + Ok(paths) +} + +fn run_batches( + conn: &Connection, + data_dir: &Path, + items: &[T], + mut touched_paths: impl FnMut(&T) -> EngineResult>, + mut operation: impl FnMut(&T) -> EngineResult<()>, +) -> EngineResult<()> { + for batch in items.chunks(TRANSACTION_BATCH_SIZE) { + let touched = batch + .iter() + .map(&mut touched_paths) + .collect::>>()? + .into_iter() + .flatten() + .collect::>(); + let filesystem = FilesystemSnapshot::capture(data_dir, &touched)?; + conn.begin_savepoint()?; + for item in batch { + if let Err(error) = operation(item) { + let _ = conn.rollback_savepoint(); + filesystem.restore(data_dir)?; + return Err(error); + } + } + if let Err(error) = conn.release_savepoint() { + let _ = conn.rollback_savepoint(); + filesystem.restore(data_dir)?; + return Err(error.into()); + } + } + Ok(()) +} + +#[derive(Debug)] +struct FilesystemSnapshot { + existing_files: HashSet, + directories: HashSet, + backups: HashMap>, +} + +impl FilesystemSnapshot { + fn capture(data_dir: &Path, touched_paths: &HashSet) -> EngineResult { + let mut existing_files = HashSet::new(); + let mut directories = HashSet::new(); + if data_dir.exists() { + for entry in walkdir::WalkDir::new(data_dir) { + let entry = entry.map_err(|error| EngineError::Io(error.into()))?; + let relative = entry + .path() + .strip_prefix(data_dir) + .map_err(|error| EngineError::Parse(error.to_string()))? + .to_path_buf(); + if relative.as_os_str().is_empty() { + continue; + } + if entry.file_type().is_dir() { + directories.insert(relative); + } else if entry.file_type().is_file() { + existing_files.insert(relative); + } + } + } + let backups = touched_paths + .iter() + .map(|path| validated_relative_path(data_dir, path)) + .collect::>>()? + .into_iter() + .filter(|path| existing_files.contains(path)) + .map(|path| fs::read(data_dir.join(&path)).map(|content| (path, content))) + .collect::, _>>()?; + Ok(Self { + existing_files, + directories, + backups, + }) + } + + fn restore(self, data_dir: &Path) -> EngineResult<()> { + if data_dir.exists() { + let mut current = walkdir::WalkDir::new(data_dir) + .min_depth(1) + .into_iter() + .collect::, _>>() + .map_err(|error| EngineError::Io(error.into()))?; + current.sort_by_key(|entry| std::cmp::Reverse(entry.depth())); + for entry in current { + let relative = entry + .path() + .strip_prefix(data_dir) + .map_err(|error| EngineError::Parse(error.to_string()))?; + if entry.file_type().is_file() && !self.existing_files.contains(relative) { + fs::remove_file(entry.path())?; + } else if entry.file_type().is_dir() && !self.directories.contains(relative) { + fs::remove_dir(entry.path())?; + } + } + } + for (relative, content) in self.backups { + let path = data_dir.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, content)?; + } + Ok(()) + } +} + +fn validated_relative_path(data_dir: &Path, path: &Path) -> EngineResult { + let relative = if path.is_absolute() { + path.strip_prefix(data_dir).map_err(|_| { + EngineError::Validation(format!("path escapes data folder: {}", path.display())) + })? + } else { + path + }; + if relative.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) { + return Err(EngineError::Validation(format!( + "path escapes data folder: {}", + path.display() + ))); + } + Ok(relative.to_path_buf()) +} + +fn remove_file_if_present(path: &Path) -> EngineResult<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn notify( + callback: &mut Option<&mut dyn FnMut(ImportProgress)>, + phase: ImportPhase, + current: usize, + total: usize, + detail: &str, + started: Option, +) { + let Some(callback) = callback.as_deref_mut() else { + return; + }; + let eta_ms = started.and_then(|started| { + (current > 0 && current < total).then(|| { + (started.elapsed().as_millis() as u64 / current as u64) + .saturating_mul((total - current) as u64) + }) + }); + let _ = catch_unwind(AssertUnwindSafe(|| { + callback(ImportProgress { + phase, + current, + total, + detail: detail.to_string(), + eta_ms, + }) + })); +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + #[test] + fn ai_mapping_only_accepts_canonical_existing_terms() { + let mut report = ImportReport { + taxonomies: vec![ + taxonomy_candidate(TaxonomyKind::Category, "Old Engineering", false), + taxonomy_candidate(TaxonomyKind::Tag, "Old Rust", false), + taxonomy_candidate(TaxonomyKind::Tag, "Unknown", false), + ], + ..ImportReport::default() + }; + let mapping = ai::ImportTaxonomyMapping { + category_mappings: BTreeMap::from([( + "Old Engineering".to_string(), + "engineering".to_string(), + )]), + tag_mappings: BTreeMap::from([ + ("Old Rust".to_string(), "RUST".to_string()), + ("Unknown".to_string(), "Invented".to_string()), + ]), + }; + + let count = apply_ai_taxonomy_mapping( + &mut report, + &mapping, + &["Engineering".to_string()], + &["Rust".to_string()], + ); + + assert_eq!(count, 2); + assert_eq!( + report.taxonomies[0].mapped_to.as_deref(), + Some("Engineering") + ); + assert_eq!(report.taxonomies[1].mapped_to.as_deref(), Some("Rust")); + assert!(report.taxonomies[2].mapped_to.is_none()); + } + + #[test] + fn progress_reports_eta_without_propagating_callback_panics() { + let started = Instant::now(); + let mut observed = None; + notify( + &mut Some(&mut |progress| observed = Some(progress)), + ImportPhase::Posts, + 1, + 2, + "first", + Some(started), + ); + assert!(observed.unwrap().eta_ms.is_some()); + + notify( + &mut Some(&mut |_| panic!("observer failed")), + ImportPhase::Posts, + 1, + 2, + "first", + Some(started), + ); + } +} diff --git a/crates/bds-core/src/model/import.rs b/crates/bds-core/src/model/import.rs new file mode 100644 index 0000000..7b297ac --- /dev/null +++ b/crates/bds-core/src/model/import.rs @@ -0,0 +1,224 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportItemKind { + Post, + Page, + Media, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportItemStatus { + New, + Update, + Conflict, + ContentDuplicate, + Missing, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportResolution { + Ignore, + Overwrite, + Import, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaxonomyKind { + Category, + Tag, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct ImportedSite { + pub title: String, + pub url: Option, + pub language: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportCandidate { + pub kind: ImportItemKind, + pub source_id: Option, + pub title: String, + pub slug: Option, + pub filename: Option, + pub relative_path: Option, + pub status: ImportItemStatus, + pub resolution: Option, + pub existing_id: Option, + pub author: Option, + pub excerpt: Option, + pub content: Option, + pub source_status: Option, + #[serde(default)] + pub categories: Vec, + #[serde(default)] + pub tags: Vec, + pub source_path: Option, + pub parent_source_id: Option, + pub created_at: Option, + pub updated_at: Option, + pub published_at: Option, + pub checksum: Option, + pub mime_type: Option, + pub description: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaxonomyCandidate { + pub kind: TaxonomyKind, + pub name: String, + pub slug: Option, + pub exists_in_project: bool, + pub mapped_to: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct ImportCounts { + pub new_count: usize, + pub update_count: usize, + pub conflict_count: usize, + pub duplicate_count: usize, + pub missing_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportDateBucket { + pub year: i32, + pub post_count: usize, + pub media_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportMacroUsage { + pub name: String, + pub total_count: usize, + #[serde(default)] + pub post_slugs: Vec, + #[serde(default)] + pub parameters: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct ImportReport { + pub source_file: String, + pub uploads_folder: Option, + pub site: ImportedSite, + #[serde(default)] + pub posts: Vec, + #[serde(default)] + pub pages: Vec, + #[serde(default)] + pub media: Vec, + #[serde(default)] + pub taxonomies: Vec, + pub post_counts: ImportCounts, + pub page_counts: ImportCounts, + pub media_counts: ImportCounts, + #[serde(default)] + pub date_distribution: Vec, + #[serde(default)] + pub macros: Vec, +} + +impl ImportReport { + pub fn importable_count(&self) -> usize { + let taxonomy = self + .taxonomies + .iter() + .filter(|item| !item.exists_in_project && item.mapped_to.is_none()) + .count(); + taxonomy + + self + .posts + .iter() + .chain(&self.media) + .chain(&self.pages) + .filter(|item| match item.status { + ImportItemStatus::New => true, + ImportItemStatus::Conflict => matches!( + item.resolution, + Some(ImportResolution::Overwrite | ImportResolution::Import) + ), + _ => false, + }) + .count() + } +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + diesel::Queryable, + diesel::Selectable, + diesel::Insertable, + diesel::AsChangeset, +)] +#[diesel( + table_name = crate::db::schema::import_definitions, + check_for_backend(diesel::sqlite::Sqlite) +)] +#[diesel(treat_none_as_default_value = false, treat_none_as_null = true)] +pub struct ImportDefinition { + pub id: String, + pub project_id: String, + pub name: String, + pub wxr_file_path: Option, + pub uploads_folder_path: Option, + pub last_analysis_result: Option, + pub created_at: i64, + pub updated_at: i64, +} + +impl ImportDefinition { + pub fn analysis(&self) -> Result, serde_json::Error> { + self.last_analysis_result + .as_deref() + .map(serde_json::from_str) + .transpose() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportPhase { + Taxonomy, + Posts, + Media, + Pages, + Complete, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportProgress { + pub phase: ImportPhase, + pub current: usize, + pub total: usize, + pub detail: String, + pub eta_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct ImportExecutionCounts { + pub imported: usize, + pub skipped: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct ImportExecutionResult { + pub taxonomy: ImportExecutionCounts, + pub posts: ImportExecutionCounts, + pub media: ImportExecutionCounts, + pub pages: ImportExecutionCounts, +} diff --git a/crates/bds-core/src/model/mod.rs b/crates/bds-core/src/model/mod.rs index 26b259a..c86a610 100644 --- a/crates/bds-core/src/model/mod.rs +++ b/crates/bds-core/src/model/mod.rs @@ -1,4 +1,5 @@ mod generation; +mod import; mod media; pub mod metadata; mod post; @@ -11,6 +12,11 @@ pub use generation::{ DbNotification, GeneratedFileHash, NotificationAction, NotificationEntity, PublishingPreferences, SshMode, }; +pub use import::{ + ImportCandidate, ImportCounts, ImportDateBucket, ImportDefinition, ImportExecutionCounts, + ImportExecutionResult, ImportItemKind, ImportItemStatus, ImportMacroUsage, ImportPhase, + ImportProgress, ImportReport, ImportResolution, ImportedSite, TaxonomyCandidate, TaxonomyKind, +}; pub use media::{Media, MediaTranslation}; pub use metadata::{CategorySettings, ProjectMetadata, TagEntry}; pub use post::{Post, PostLink, PostMedia, PostStatus, PostTranslation}; diff --git a/crates/bds-core/src/scripting/core_host.rs b/crates/bds-core/src/scripting/core_host.rs index ecff343..f2ebcf7 100644 --- a/crates/bds-core/src/scripting/core_host.rs +++ b/crates/bds-core/src/scripting/core_host.rs @@ -1738,6 +1738,7 @@ fn post_ai_content(data_dir: &Path, post: &Post) -> HostResult { fn one_shot_json(value: engine::ai::OneShotResponse) -> HostResult { let value = match value { engine::ai::OneShotResponse::Taxonomy(value) => serde_json::to_value(value), + engine::ai::OneShotResponse::ImportTaxonomyMapping(value) => serde_json::to_value(value), engine::ai::OneShotResponse::PostAnalysis(value) => serde_json::to_value(value), engine::ai::OneShotResponse::LanguageDetection(value) => serde_json::to_value(value), engine::ai::OneShotResponse::Translation(value) => serde_json::to_value(value), diff --git a/crates/bds-core/tests/wordpress_import.rs b/crates/bds-core/tests/wordpress_import.rs new file mode 100644 index 0000000..9457b1d --- /dev/null +++ b/crates/bds-core/tests/wordpress_import.rs @@ -0,0 +1,807 @@ +use std::fs; +use std::path::Path; + +use bds_core::db::Database; +use bds_core::db::fts::ensure_fts_tables; +use bds_core::engine::{meta, post, project, tag, wordpress_import as import}; +use bds_core::model::{ + ImportItemKind, ImportItemStatus, ImportResolution, PostStatus, TaxonomyKind, +}; +use bds_core::util::frontmatter::read_post_file; +use bds_core::util::sidecar::read_sidecar; +use image::DynamicImage; +use tempfile::TempDir; + +fn setup() -> (Database, TempDir, bds_core::model::Project) { + let db = Database::open_in_memory().unwrap(); + db.migrate().unwrap(); + ensure_fts_tables(db.conn()).unwrap(); + let dir = TempDir::new().unwrap(); + let project = project::create_project( + db.conn(), + "WordPress Import", + Some(dir.path().to_str().unwrap()), + ) + .unwrap(); + (db, dir, project) +} + +fn write_image(path: &Path) { + DynamicImage::new_rgb8(4, 3).save(path).unwrap(); +} + +#[test] +fn parser_rejects_malformed_or_channel_less_xml_and_ignores_unknown_elements() { + assert!(import::parse_wxr_xml("").is_err()); + assert!(import::parse_wxr_xml("").is_err()); + + let parsed = import::parse_wxr_xml(&sample_wxr( + "ignoredMenunav_menu_item", + )) + .unwrap(); + assert_eq!(parsed.site.title, "Legacy & Blog"); + assert_eq!(parsed.posts.len(), 1); + assert_eq!(parsed.pages.len(), 1); + assert_eq!(parsed.media.len(), 1); + assert_eq!(parsed.categories[0].name, "General"); + assert_eq!(parsed.tags[0].name, "News"); +} + +#[test] +fn analysis_converts_html_and_shortcodes_and_classifies_every_status() { + let (db, dir, project) = setup(); + meta::add_category(dir.path(), "GENERAL").unwrap(); + tag::create_tag(db.conn(), dir.path(), &project.id, "nEWs", None).unwrap(); + + let update = post::create_post( + db.conn(), + dir.path(), + &project.id, + "Update", + Some("Update body"), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + post::update_post( + db.conn(), + dir.path(), + &update.id, + None, + Some("hello-world"), + None, + None, + None, + None, + None, + None, + None, + None, + ) + .unwrap(); + let duplicate = post::create_post( + db.conn(), + dir.path(), + &project.id, + "Duplicate", + Some("Duplicate body"), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + assert_ne!(update.id, duplicate.id); + + let uploads = dir.path().join("uploads/2024/05"); + fs::create_dir_all(&uploads).unwrap(); + write_image(&uploads.join("photo.png")); + let wxr = dir.path().join("legacy.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + + let mut progress = Vec::new(); + let report = import::analyze_wxr( + db.conn(), + dir.path(), + &project.id, + &wxr, + Some(dir.path().join("uploads").as_path()), + Some(&mut |value| progress.push(value)), + ) + .unwrap(); + + assert!( + report.posts[0] + .content + .as_deref() + .unwrap() + .contains("**world**") + ); + assert!( + report.posts[0] + .content + .as_deref() + .unwrap() + .contains("[[gallery ids=\"1,2\"]]"), + "{}", + report.posts[0].content.as_deref().unwrap() + ); + assert_eq!(report.posts[0].status, ImportItemStatus::Conflict); + assert_eq!(report.posts[0].resolution, Some(ImportResolution::Ignore)); + assert_eq!(report.pages[0].status, ImportItemStatus::New); + assert_eq!(report.media[0].status, ImportItemStatus::New); + assert!( + report + .taxonomies + .iter() + .any(|item| { item.kind == TaxonomyKind::Category && item.exists_in_project }) + ); + assert!( + report + .taxonomies + .iter() + .any(|item| { item.kind == TaxonomyKind::Tag && item.exists_in_project }) + ); + assert_eq!(report.date_distribution[0].year, 2024); + assert_eq!(report.macros[0].name, "gallery"); + assert!(!progress.is_empty()); +} + +#[test] +fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() { + let (db, dir, project) = setup(); + let uploads = dir.path().join("uploads/2024/05"); + fs::create_dir_all(&uploads).unwrap(); + write_image(&uploads.join("same.png")); + fs::copy(uploads.join("same.png"), uploads.join("copy.png")).unwrap(); + DynamicImage::new_rgb8(5, 3) + .save(uploads.join("different.png")) + .unwrap(); + let old_different = dir.path().join("old-different.png"); + DynamicImage::new_rgb8(2, 2).save(&old_different).unwrap(); + bds_core::engine::media::import_media( + db.conn(), + dir.path(), + &project.id, + &uploads.join("same.png"), + "same.png", + None, + None, + None, + None, + None, + Vec::new(), + ) + .unwrap(); + bds_core::engine::media::import_media( + db.conn(), + dir.path(), + &project.id, + &old_different, + "different.png", + None, + None, + None, + None, + None, + Vec::new(), + ) + .unwrap(); + + let same_body = "Hello **world**.\n\n[[gallery ids=\"1,2\"]]"; + let update = post::create_post( + db.conn(), + dir.path(), + &project.id, + "Hello World", + Some(same_body), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + assert_eq!(update.slug, "hello-world"); + let collision = post::create_post( + db.conn(), + dir.path(), + &project.id, + "Collision", + Some("Local body"), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + assert_eq!(collision.slug, "collision"); + post::create_post( + db.conn(), + dir.path(), + &project.id, + "Local Duplicate", + Some("Shared duplicate body"), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + + let extras = format!( + r#" +Collision102collisiondraftpost +Duplicate ContentShared duplicate body

]]>103duplicate-contentdraftpost +{} +{} +{} +{} +{}"#, + attachment_item(302, "Same", "same.png"), + attachment_item(303, "Copy", "copy.png"), + attachment_item(304, "Different", "different.png"), + attachment_item(305, "Missing", "missing.png"), + attachment_item(306, "Escape", "../escape.png"), + ); + let wxr = dir.path().join("classifications.xml"); + fs::write(&wxr, sample_wxr(&extras)).unwrap(); + let posts_before = serde_json::to_value( + bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap(), + ) + .unwrap(); + let media_before = serde_json::to_value( + bds_core::db::queries::media::list_media_by_project(db.conn(), &project.id).unwrap(), + ) + .unwrap(); + let tags_before = serde_json::to_value( + bds_core::db::queries::tag::list_tags_by_project(db.conn(), &project.id).unwrap(), + ) + .unwrap(); + let categories_before = fs::read(dir.path().join("meta/categories.json")).unwrap(); + let report = import::analyze_wxr( + db.conn(), + dir.path(), + &project.id, + &wxr, + Some(dir.path().join("uploads").as_path()), + None, + ) + .unwrap(); + assert_eq!( + serde_json::to_value( + bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap() + ) + .unwrap(), + posts_before + ); + assert_eq!( + serde_json::to_value( + bds_core::db::queries::media::list_media_by_project(db.conn(), &project.id).unwrap() + ) + .unwrap(), + media_before + ); + assert_eq!( + serde_json::to_value( + bds_core::db::queries::tag::list_tags_by_project(db.conn(), &project.id).unwrap() + ) + .unwrap(), + tags_before + ); + assert_eq!( + fs::read(dir.path().join("meta/categories.json")).unwrap(), + categories_before + ); + + let post_statuses = report + .posts + .iter() + .map(|item| item.status) + .collect::>(); + assert!(post_statuses.contains(&ImportItemStatus::Update)); + assert!(post_statuses.contains(&ImportItemStatus::Conflict)); + assert!(post_statuses.contains(&ImportItemStatus::ContentDuplicate)); + let media_status = |name: &str| { + report + .media + .iter() + .find(|item| item.filename.as_deref() == Some(name)) + .unwrap() + .status + }; + assert_eq!(media_status("same.png"), ImportItemStatus::Update); + assert_eq!(media_status("copy.png"), ImportItemStatus::ContentDuplicate); + assert_eq!(media_status("different.png"), ImportItemStatus::Conflict); + assert_eq!(media_status("missing.png"), ImportItemStatus::Missing); + assert_eq!(media_status("escape.png"), ImportItemStatus::Missing); + assert!( + report + .media + .iter() + .find(|item| item.filename.as_deref() == Some("escape.png")) + .unwrap() + .source_path + .is_none() + ); +} + +#[test] +fn definitions_round_trip_saved_analysis_and_are_project_scoped() { + let (db, dir, project) = setup(); + let definition = import::create_definition(db.conn(), &project.id, "Legacy").unwrap(); + let wxr = dir.path().join("legacy.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + let report = import::analyze_wxr(db.conn(), dir.path(), &project.id, &wxr, None, None).unwrap(); + + let updated = import::update_definition( + db.conn(), + &definition.id, + Some("Renamed"), + Some(Some(wxr.as_path())), + Some(Some(dir.path())), + Some(Some(&report)), + ) + .unwrap(); + assert_eq!(updated.name, "Renamed"); + assert_eq!(updated.analysis().unwrap().unwrap(), report); + assert_eq!( + import::list_definitions(db.conn(), &project.id) + .unwrap() + .len(), + 1 + ); + import::delete_definition(db.conn(), &definition.id).unwrap(); + assert!(import::get_definition(db.conn(), &definition.id).is_err()); +} + +#[test] +fn execution_uses_core_engines_preserves_metadata_and_links_media_parent() { + let (db, dir, project) = setup(); + let uploads = dir.path().join("uploads/2024/05"); + fs::create_dir_all(&uploads).unwrap(); + write_image(&uploads.join("photo.png")); + let wxr = dir.path().join("legacy.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + let report = import::analyze_wxr( + db.conn(), + dir.path(), + &project.id, + &wxr, + Some(dir.path().join("uploads").as_path()), + None, + ) + .unwrap(); + + let mut progress = Vec::new(); + let result = import::execute_import( + db.conn(), + dir.path(), + &project.id, + &report, + Some("Fallback Author"), + Some(&mut |value| progress.push(value)), + ) + .unwrap(); + assert_eq!(result.posts.imported, 1); + assert_eq!(result.pages.imported, 1); + assert_eq!(result.media.imported, 1); + + let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap(); + let imported = posts + .iter() + .find(|post| post.slug == "hello-world") + .unwrap(); + assert_eq!(imported.status, PostStatus::Published); + assert_eq!(imported.author.as_deref(), Some("Importer")); + assert_eq!(imported.created_at, 1_714_564_800_000); + assert_eq!(imported.updated_at, 1_714_653_000_000); + assert_eq!(imported.published_at, Some(1_714_564_800_000)); + assert_eq!(imported.file_path, "posts/2024/05/hello-world.md"); + assert!(dir.path().join(&imported.file_path).is_file()); + let raw = fs::read_to_string(dir.path().join(&imported.file_path)).unwrap(); + let (frontmatter, _) = read_post_file(&raw).unwrap(); + assert_eq!(frontmatter.created_at, imported.created_at); + assert_eq!(frontmatter.updated_at, imported.updated_at); + assert_eq!(frontmatter.published_at, imported.published_at); + let page = posts.iter().find(|post| post.slug == "about").unwrap(); + assert!(page.categories.iter().any(|category| category == "page")); + + let media = + bds_core::db::queries::media::list_media_by_project(db.conn(), &project.id).unwrap(); + assert_eq!(media[0].alt.as_deref(), Some("Photo description")); + assert_eq!(media[0].author.as_deref(), Some("Fallback Author")); + assert_eq!(media[0].created_at, 1_714_737_600_000); + assert!(media[0].file_path.starts_with("media/2024/05/")); + let sidecar = + read_sidecar(&fs::read_to_string(dir.path().join(&media[0].sidecar_path)).unwrap()) + .unwrap(); + assert_eq!(sidecar.created_at, media[0].created_at); + let links = + bds_core::db::queries::post_media::list_post_media_by_media(db.conn(), &media[0].id) + .unwrap(); + assert_eq!(links[0].post_id, imported.id); + let phases = progress.iter().map(|item| item.phase).collect::>(); + assert!(phases.windows(2).all(|window| window[0] <= window[1])); + assert_eq!(phases.last(), Some(&bds_core::model::ImportPhase::Complete)); + for phase in [ + bds_core::model::ImportPhase::Taxonomy, + bds_core::model::ImportPhase::Posts, + bds_core::model::ImportPhase::Media, + bds_core::model::ImportPhase::Pages, + ] { + assert!( + progress + .iter() + .any(|item| item.phase == phase && item.current == 0), + "missing start event for {phase:?}" + ); + } + + let diff = + bds_core::engine::metadata_diff::compute_metadata_diff(db.conn(), dir.path(), &project.id) + .unwrap(); + assert!(diff.errors.is_empty(), "{:?}", diff.errors); + assert!( + diff.diffs + .iter() + .all(|item| item.entity_type != "post" && item.entity_type != "media"), + "{:?}", + diff.diffs + ); + assert!( + bds_core::engine::post::rebuild_posts_from_filesystem(db.conn(), dir.path(), &project.id,) + .unwrap() + .errors + .is_empty() + ); + assert!( + bds_core::engine::media::rebuild_media_from_filesystem(db.conn(), dir.path(), &project.id,) + .unwrap() + .errors + .is_empty() + ); +} + +#[test] +fn conflict_import_uses_unique_slug_and_progress_panics_do_not_abort() { + let (db, dir, project) = setup(); + let existing = post::create_post( + db.conn(), + dir.path(), + &project.id, + "Hello World", + Some("local"), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + assert_eq!(existing.slug, "hello-world"); + let wxr = dir.path().join("legacy.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + let mut report = import::analyze_wxr( + db.conn(), + dir.path(), + &project.id, + &wxr, + None, + Some(&mut |_| panic!("observer failure")), + ) + .unwrap(); + import::set_conflict_resolution( + &mut report, + ImportItemKind::Post, + "hello-world", + ImportResolution::Import, + ) + .unwrap(); + import::execute_import( + db.conn(), + dir.path(), + &project.id, + &report, + None, + Some(&mut |_| panic!("observer failure")), + ) + .unwrap(); + let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap(); + assert_eq!(posts.len(), 3); + assert!( + posts + .iter() + .any(|post| post.slug.starts_with("hello-world-") && post.slug != "hello-world") + ); +} + +#[test] +fn default_ignore_resolution_skips_conflict_without_mutating_existing_post() { + let (db, dir, project) = setup(); + let existing = post::create_post( + db.conn(), + dir.path(), + &project.id, + "Hello World", + Some("Local body"), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + let wxr = dir.path().join("ignore.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + let mut report = + import::analyze_wxr(db.conn(), dir.path(), &project.id, &wxr, None, None).unwrap(); + report.taxonomies.clear(); + report.pages.clear(); + report.media.clear(); + assert_eq!(report.posts[0].resolution, Some(ImportResolution::Ignore)); + + let result = + import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).unwrap(); + assert_eq!(result.posts.imported, 0); + assert_eq!(result.posts.skipped, 1); + let unchanged = bds_core::db::queries::post::get_post_by_id(db.conn(), &existing.id).unwrap(); + assert_eq!(unchanged.content.as_deref(), Some("Local body")); +} + +#[test] +fn overwrite_resolution_preserves_existing_post_and_media_identity() { + let (db, dir, project) = setup(); + let existing_post = post::create_post( + db.conn(), + dir.path(), + &project.id, + "Hello World", + Some("Local post body"), + vec![], + vec![], + None, + None, + None, + ) + .unwrap(); + let old_image = dir.path().join("old-photo.png"); + DynamicImage::new_rgb8(2, 2).save(&old_image).unwrap(); + let existing_media = bds_core::engine::media::import_media( + db.conn(), + dir.path(), + &project.id, + &old_image, + "photo.png", + Some("Old photo"), + Some("Old alt"), + None, + None, + None, + Vec::new(), + ) + .unwrap(); + let uploads = dir.path().join("uploads/2024/05"); + fs::create_dir_all(&uploads).unwrap(); + write_image(&uploads.join("photo.png")); + let wxr = dir.path().join("overwrite.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + let mut report = import::analyze_wxr( + db.conn(), + dir.path(), + &project.id, + &wxr, + Some(dir.path().join("uploads").as_path()), + None, + ) + .unwrap(); + import::set_conflict_resolution( + &mut report, + ImportItemKind::Post, + "hello-world", + ImportResolution::Overwrite, + ) + .unwrap(); + import::set_conflict_resolution( + &mut report, + ImportItemKind::Media, + "photo.png", + ImportResolution::Overwrite, + ) + .unwrap(); + + import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).unwrap(); + let overwritten_post = + bds_core::db::queries::post::get_post_by_id(db.conn(), &existing_post.id).unwrap(); + assert_eq!(overwritten_post.id, existing_post.id); + assert_eq!(overwritten_post.status, PostStatus::Published); + assert_eq!( + overwritten_post.published_content.as_deref(), + report.posts[0].content.as_deref() + ); + let overwritten_media = + bds_core::db::queries::media::get_media_by_id(db.conn(), &existing_media.id).unwrap(); + assert_eq!(overwritten_media.id, existing_media.id); + assert_eq!(overwritten_media.alt.as_deref(), Some("Photo description")); + assert_ne!(overwritten_media.checksum, existing_media.checksum); +} + +#[test] +fn airplane_ai_mapping_requires_a_local_endpoint() { + let (db, _dir, _project) = setup(); + let mut report = import::empty_report(); + report.taxonomies.push(import::taxonomy_candidate( + TaxonomyKind::Tag, + "Legacy News", + false, + )); + let error = import::auto_map_taxonomy(db.conn(), _dir.path(), &_project.id, true, &mut report) + .unwrap_err(); + assert!(error.to_string().contains("airplane")); + assert!(report.taxonomies[0].mapped_to.is_none()); +} + +#[test] +fn execution_rejects_a_saved_taxonomy_mapping_to_a_nonexistent_term() { + let (db, dir, project) = setup(); + let mut report = import::empty_report(); + let mut candidate = import::taxonomy_candidate(TaxonomyKind::Category, "Old", false); + candidate.mapped_to = Some("Does Not Exist".to_string()); + report.taxonomies.push(candidate); + + let error = import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None) + .unwrap_err(); + assert!(error.to_string().contains("mapping target does not exist")); + assert!( + !meta::read_categories_json(dir.path()) + .unwrap() + .iter() + .any(|category| category == "Old") + ); +} + +#[test] +fn execution_commits_complete_500_item_batches_and_rolls_back_database_and_files_for_failure() { + let (db, dir, project) = setup(); + let wxr = dir.path().join("legacy.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + let mut report = + import::analyze_wxr(db.conn(), dir.path(), &project.id, &wxr, None, None).unwrap(); + let template = report.posts[0].clone(); + report.taxonomies.clear(); + report.media.clear(); + report.pages.clear(); + report.posts = (0..=500) + .map(|index| { + let mut item = template.clone(); + item.source_id = Some(10_000 + index); + item.title = format!("Batch {index}"); + item.slug = Some(format!("batch-{index}")); + item.status = ImportItemStatus::New; + item.resolution = None; + item.existing_id = None; + item + }) + .collect(); + let mut failing = template; + failing.title = "Fail after file write".to_string(); + failing.slug = Some("batch-500".to_string()); + failing.status = ImportItemStatus::New; + failing.resolution = None; + failing.existing_id = None; + report.posts.push(failing); + + assert!( + import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).is_err() + ); + let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap(); + assert_eq!(posts.len(), 500); + assert!(posts.iter().any(|post| post.slug == "batch-499")); + assert!(!posts.iter().any(|post| post.slug == "batch-500")); + assert!(!dir.path().join("posts/2024/05/batch-500.md").exists()); + assert!(dir.path().join("posts/2024/05/batch-499.md").is_file()); +} + +#[test] +fn failed_media_batch_restores_an_overwritten_binary_sidecar_and_database_row() { + let (db, dir, project) = setup(); + let old_image = dir.path().join("old.png"); + DynamicImage::new_rgb8(2, 2).save(&old_image).unwrap(); + let existing = bds_core::engine::media::import_media( + db.conn(), + dir.path(), + &project.id, + &old_image, + "photo.png", + Some("Old title"), + Some("Old alt"), + None, + None, + None, + Vec::new(), + ) + .unwrap(); + let original_binary = fs::read(dir.path().join(&existing.file_path)).unwrap(); + let original_sidecar = fs::read(dir.path().join(&existing.sidecar_path)).unwrap(); + + let uploads = dir.path().join("uploads/2024/05"); + fs::create_dir_all(&uploads).unwrap(); + write_image(&uploads.join("photo.png")); + let wxr = dir.path().join("rollback-overwrite.xml"); + fs::write(&wxr, sample_wxr("")).unwrap(); + let mut report = import::analyze_wxr( + db.conn(), + dir.path(), + &project.id, + &wxr, + Some(dir.path().join("uploads").as_path()), + None, + ) + .unwrap(); + report.taxonomies.clear(); + report.posts.clear(); + report.pages.clear(); + import::set_conflict_resolution( + &mut report, + ImportItemKind::Media, + "photo.png", + ImportResolution::Overwrite, + ) + .unwrap(); + let mut missing = report.media[0].clone(); + missing.title = "Missing".to_string(); + missing.filename = Some("missing.png".to_string()); + missing.source_path = Some( + dir.path() + .join("missing.png") + .to_string_lossy() + .into_owned(), + ); + missing.status = ImportItemStatus::New; + missing.resolution = None; + missing.existing_id = None; + report.media.push(missing); + + assert!( + import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).is_err() + ); + let restored = bds_core::db::queries::media::get_media_by_id(db.conn(), &existing.id).unwrap(); + assert_eq!( + serde_json::to_value(&restored).unwrap(), + serde_json::to_value(&existing).unwrap() + ); + assert_eq!( + fs::read(dir.path().join(&existing.file_path)).unwrap(), + original_binary + ); + assert_eq!( + fs::read(dir.path().join(&existing.sidecar_path)).unwrap(), + original_sidecar + ); +} + +fn sample_wxr(extra: &str) -> String { + format!( + r#" + +Legacy & Bloghttps://legacy.exampleen{extra} +general +news +Hello WorldWed, 01 May 2024 12:00:00 +0000Hello world.

[gallery ids="1,2"]

]]>Legacy hello1012024-05-01 12:00:002024-05-02 12:30:00hello-worldpublishpost +AboutThu, 02 May 2024 12:00:00 +0000ImporterAbout page

]]>2012024-05-02 12:00:002024-05-02 12:30:00aboutdraftpageGeneral +PhotoFri, 03 May 2024 12:00:00 +0000301101photoinheritattachmenthttps://legacy.example/wp-content/uploads/2024/05/photo.png +"# + ) +} + +fn attachment_item(id: i64, title: &str, filename: &str) -> String { + format!( + r#"{title}Fri, 03 May 2024 12:00:00 +0000{title}{id}101{title}inheritattachmenthttps://legacy.example/wp-content/uploads/2024/05/{filename}"# + ) +} diff --git a/crates/bds-ui/Cargo.toml b/crates/bds-ui/Cargo.toml index c202549..fea7d31 100644 --- a/crates/bds-ui/Cargo.toml +++ b/crates/bds-ui/Cargo.toml @@ -17,6 +17,7 @@ open = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } uuid = { workspace = true } +futures = { workspace = true } wry = "0.55" [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/bds-ui/src/app.rs b/crates/bds-ui/src/app.rs index 9d7ece6..b184b80 100644 --- a/crates/bds-ui/src/app.rs +++ b/crates/bds-ui/src/app.rs @@ -13,8 +13,8 @@ use bds_core::engine::ai::{self, AiEndpointConfig, AiEndpointKind}; use bds_core::engine::task::{TaskId, TaskManager, TaskStatus}; use bds_core::i18n::{UiLocale, detect_os_locale}; use bds_core::model::{ - Media, Post, PostStatus, PostTranslation, Project, PublishingPreferences, Script, SshMode, - Template, + ImportDefinition, ImportReport, Media, Post, PostStatus, PostTranslation, Project, + PublishingPreferences, Script, SshMode, Template, }; use crate::components::webview::{self, WebViewConfig, WebViewController}; @@ -32,6 +32,9 @@ use crate::views::{ DashboardTimelineMonth, }, git::{GitDiffLoad, GitDiffState, GitNetworkCompletion, GitSnapshot, GitUiState}, + import_editor::{ + ImportAnalysisEvent, ImportEditorMsg, ImportEditorState, ImportExecutionEvent, + }, media_editor::{LinkedPostItem, MediaEditorMsg, MediaEditorState}, metadata_diff::MetadataDiffState, modal, @@ -118,6 +121,26 @@ pub enum Message { media_id: String, result: Result, String>, }, + ImportUploadsPicked { + definition_id: String, + path: Option, + }, + ImportWxrPicked { + definition_id: String, + path: Option, + }, + ImportAnalysisEvent { + definition_id: String, + event: ImportAnalysisEvent, + }, + ImportExecutionEvent { + definition_id: String, + event: ImportExecutionEvent, + }, + ImportAutoMapFinished { + definition_id: String, + result: Result<(ImportDefinition, ImportReport, usize), String>, + }, // Tasks TaskTick, @@ -269,6 +292,7 @@ pub enum Message { ScriptEditor(ScriptEditorMsg), Tags(TagsMsg), Settings(SettingsMsg), + ImportEditor(ImportEditorMsg), // Editor data loading PostLoaded(Result), @@ -295,6 +319,7 @@ pub enum Message { CreateMedia, CreateScript, CreateTemplate, + CreateImport, Noop, InitMenuBar, @@ -798,6 +823,7 @@ pub struct BdsApp { sidebar_media: Vec, sidebar_scripts: Vec