Implement complete WordPress WXR import workflow.

This commit is contained in:
2026-07-19 13:17:45 +02:00
parent 9dab0ca57e
commit 429f471e4e
29 changed files with 4927 additions and 26 deletions

View File

@@ -23,6 +23,8 @@ pulldown-cmark = { workspace = true }
liquid = { workspace = true }
liquid-core = { workspace = true }
quick-xml = { workspace = true }
htmd = { workspace = true }
regex = { workspace = true }
rayon = { workspace = true }
pagefind = { workspace = true }
reqwest = { workspace = true }

View File

@@ -0,0 +1,62 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::import_definitions;
use crate::model::ImportDefinition;
pub fn insert_import_definition(
conn: &DbConnection,
definition: &ImportDefinition,
) -> QueryResult<()> {
conn.with(|connection| {
diesel::insert_into(import_definitions::table)
.values(definition.clone())
.execute(connection)
.map(|_| ())
})
}
pub fn get_import_definition(conn: &DbConnection, id: &str) -> QueryResult<ImportDefinition> {
conn.with(|connection| {
import_definitions::table
.filter(import_definitions::id.eq(id))
.select(ImportDefinition::as_select())
.first(connection)
})
}
pub fn list_import_definitions(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<ImportDefinition>> {
conn.with(|connection| {
import_definitions::table
.filter(import_definitions::project_id.eq(project_id))
.order((
import_definitions::updated_at.desc(),
import_definitions::created_at.desc(),
))
.select(ImportDefinition::as_select())
.load(connection)
})
}
pub fn update_import_definition(
conn: &DbConnection,
definition: &ImportDefinition,
) -> QueryResult<()> {
conn.with(|connection| {
diesel::update(import_definitions::table.filter(import_definitions::id.eq(&definition.id)))
.set(definition.clone())
.execute(connection)
.map(|_| ())
})
}
pub fn delete_import_definition(conn: &DbConnection, id: &str) -> QueryResult<()> {
conn.with(|connection| {
diesel::delete(import_definitions::table.filter(import_definitions::id.eq(id)))
.execute(connection)
.map(|_| ())
})
}

View File

@@ -1,4 +1,5 @@
pub mod generated_file_hash;
pub mod import_definition;
pub mod media;
pub mod media_translation;
pub mod post;

View File

@@ -1,3 +1,4 @@
use std::collections::BTreeMap;
use std::time::Duration;
use crate::db::DbConnection as Connection;
@@ -83,6 +84,7 @@ pub struct AiModelInfo {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OneShotOperation {
AnalyzeTaxonomy,
MapImportTaxonomy,
AnalyzePost,
DetectLanguage,
TranslatePost { target_language: String },
@@ -102,6 +104,12 @@ pub struct TaxonomySuggestion {
pub categories: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportTaxonomyMapping {
pub category_mappings: BTreeMap<String, String>,
pub tag_mappings: BTreeMap<String, String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PostAnalysisResult {
pub title: String,
@@ -146,6 +154,7 @@ pub struct TokenUsage {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OneShotResponse {
Taxonomy(TaxonomySuggestion),
ImportTaxonomyMapping(ImportTaxonomyMapping),
PostAnalysis(PostAnalysisResult),
LanguageDetection(LanguageDetectionResult),
Translation(TranslationResult),
@@ -451,6 +460,7 @@ fn select_model(
let selected = match operation {
OneShotOperation::AnalyzeImage => settings.image_model.as_ref(),
OneShotOperation::AnalyzeTaxonomy
| OneShotOperation::MapImportTaxonomy
| OneShotOperation::AnalyzePost
| OneShotOperation::DetectLanguage
| OneShotOperation::TranslatePost { .. }
@@ -475,6 +485,9 @@ fn build_system_prompt(base_prompt: &str, operation: &OneShotOperation) -> Strin
OneShotOperation::AnalyzeTaxonomy => {
"Return only JSON with tags and categories for the post."
}
OneShotOperation::MapImportTaxonomy => {
"Map each imported category and tag to an existing equivalent when one exists. Return JSON objects keyed by each imported term; omit terms without a sound match."
}
OneShotOperation::AnalyzePost => {
"Return only JSON with title, excerpt, and slug suggestions for the post."
}
@@ -511,6 +524,11 @@ fn build_one_shot_user_content(request: &OneShotRequest) -> EngineResult<Value>
serde_json::to_string(&request.content)?
)
.into()),
OneShotOperation::MapImportTaxonomy => Ok(format!(
"Map these imported taxonomy terms to existing project terms without inventing terms: {}",
serde_json::to_string(&request.content)?
)
.into()),
OneShotOperation::AnalyzePost => Ok(format!(
"Analyze this post and suggest title, excerpt, and slug: {}",
serde_json::to_string(&request.content)?
@@ -580,6 +598,24 @@ fn response_schema(operation: &OneShotOperation) -> (&'static str, Value) {
"required": ["tags", "categories"]
}),
),
OneShotOperation::MapImportTaxonomy => (
"import_taxonomy_mapping",
json!({
"type": "object",
"additionalProperties": false,
"properties": {
"category_mappings": {
"type": "object",
"additionalProperties": { "type": "string" }
},
"tag_mappings": {
"type": "object",
"additionalProperties": { "type": "string" }
}
},
"required": ["category_mappings", "tag_mappings"]
}),
),
OneShotOperation::AnalyzePost => (
"post_analysis",
json!({
@@ -654,6 +690,9 @@ fn parse_one_shot_response(
OneShotOperation::AnalyzeTaxonomy => {
OneShotResponse::Taxonomy(serde_json::from_str(content)?)
}
OneShotOperation::MapImportTaxonomy => {
OneShotResponse::ImportTaxonomyMapping(serde_json::from_str(content)?)
}
OneShotOperation::AnalyzePost => {
OneShotResponse::PostAnalysis(serde_json::from_str(content)?)
}
@@ -939,6 +978,33 @@ mod tests {
);
}
#[test]
fn run_one_shot_supports_import_taxonomy_mapping_via_airplane_endpoint() {
let response = run_airplane_one_shot(
OneShotRequest {
operation: OneShotOperation::MapImportTaxonomy,
content: json!({
"imported_categories": ["Old Engineering"],
"imported_tags": ["Old Rust"],
"existing_categories": ["Engineering"],
"existing_tags": ["Rust"]
}),
},
r#"{"choices":[{"message":{"content":"{\"category_mappings\":{\"Old Engineering\":\"Engineering\"},\"tag_mappings\":{\"Old Rust\":\"Rust\"}}"}}],"usage":{"prompt_tokens":10,"completion_tokens":5}}"#,
);
assert_eq!(
response,
OneShotResponse::ImportTaxonomyMapping(ImportTaxonomyMapping {
category_mappings: BTreeMap::from([(
"Old Engineering".to_string(),
"Engineering".to_string(),
)]),
tag_mappings: BTreeMap::from([("Old Rust".to_string(), "Rust".to_string(),)]),
})
);
}
#[test]
fn run_one_shot_supports_post_analysis_via_airplane_endpoint() {
let response = run_airplane_one_shot(

View File

@@ -63,6 +63,43 @@ pub fn import_media(
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
) -> EngineResult<Media> {
import_media_at(
conn,
data_dir,
project_id,
source_path,
original_name,
title,
alt,
caption,
author,
language,
tags,
now_unix_ms(),
)
}
/// Import media while preserving a trusted source timestamp. Importers use
/// this path so the canonical date-based location and sidecar metadata match
/// the source system; interactive imports continue to use the current time.
#[expect(
clippy::too_many_arguments,
reason = "arguments are the user-supplied media metadata fields plus source timestamp"
)]
pub(crate) fn import_media_at(
conn: &Connection,
data_dir: &Path,
project_id: &str,
source_path: &Path,
original_name: &str,
title: Option<&str>,
alt: Option<&str>,
caption: Option<&str>,
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
created_at: i64,
) -> EngineResult<Media> {
// Validate file type per spec
let ext = Path::new(original_name)
@@ -77,12 +114,10 @@ pub fn import_media(
}
let id = Uuid::new_v4().to_string();
let now = now_unix_ms();
let filename = format!("{id}.{ext}");
// Compute target directory and copy file
let dir_path = media_dir_path(now);
let dir_path = media_dir_path(created_at);
let rel_file_path = format!("{dir_path}{filename}");
let abs_file_path = data_dir.join(&rel_file_path);
@@ -128,8 +163,8 @@ pub fn import_media(
sidecar_path: sidecar_rel.clone(),
checksum: Some(checksum),
tags,
created_at: now,
updated_at: now,
created_at,
updated_at: created_at,
};
// Write sidecar

View File

@@ -27,5 +27,6 @@ pub mod template_rebuild;
pub mod validate_media;
pub mod validate_site;
pub mod validate_translations;
pub mod wordpress_import;
pub use error::{EngineError, EngineResult};

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,224 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportItemKind {
Post,
Page,
Media,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportItemStatus {
New,
Update,
Conflict,
ContentDuplicate,
Missing,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportResolution {
Ignore,
Overwrite,
Import,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaxonomyKind {
Category,
Tag,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportedSite {
pub title: String,
pub url: Option<String>,
pub language: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportCandidate {
pub kind: ImportItemKind,
pub source_id: Option<i64>,
pub title: String,
pub slug: Option<String>,
pub filename: Option<String>,
pub relative_path: Option<String>,
pub status: ImportItemStatus,
pub resolution: Option<ImportResolution>,
pub existing_id: Option<String>,
pub author: Option<String>,
pub excerpt: Option<String>,
pub content: Option<String>,
pub source_status: Option<String>,
#[serde(default)]
pub categories: Vec<String>,
#[serde(default)]
pub tags: Vec<String>,
pub source_path: Option<String>,
pub parent_source_id: Option<i64>,
pub created_at: Option<i64>,
pub updated_at: Option<i64>,
pub published_at: Option<i64>,
pub checksum: Option<String>,
pub mime_type: Option<String>,
pub description: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaxonomyCandidate {
pub kind: TaxonomyKind,
pub name: String,
pub slug: Option<String>,
pub exists_in_project: bool,
pub mapped_to: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportCounts {
pub new_count: usize,
pub update_count: usize,
pub conflict_count: usize,
pub duplicate_count: usize,
pub missing_count: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportDateBucket {
pub year: i32,
pub post_count: usize,
pub media_count: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportMacroUsage {
pub name: String,
pub total_count: usize,
#[serde(default)]
pub post_slugs: Vec<String>,
#[serde(default)]
pub parameters: Vec<BTreeMap<String, String>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportReport {
pub source_file: String,
pub uploads_folder: Option<String>,
pub site: ImportedSite,
#[serde(default)]
pub posts: Vec<ImportCandidate>,
#[serde(default)]
pub pages: Vec<ImportCandidate>,
#[serde(default)]
pub media: Vec<ImportCandidate>,
#[serde(default)]
pub taxonomies: Vec<TaxonomyCandidate>,
pub post_counts: ImportCounts,
pub page_counts: ImportCounts,
pub media_counts: ImportCounts,
#[serde(default)]
pub date_distribution: Vec<ImportDateBucket>,
#[serde(default)]
pub macros: Vec<ImportMacroUsage>,
}
impl ImportReport {
pub fn importable_count(&self) -> usize {
let taxonomy = self
.taxonomies
.iter()
.filter(|item| !item.exists_in_project && item.mapped_to.is_none())
.count();
taxonomy
+ self
.posts
.iter()
.chain(&self.media)
.chain(&self.pages)
.filter(|item| match item.status {
ImportItemStatus::New => true,
ImportItemStatus::Conflict => matches!(
item.resolution,
Some(ImportResolution::Overwrite | ImportResolution::Import)
),
_ => false,
})
.count()
}
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
diesel::Queryable,
diesel::Selectable,
diesel::Insertable,
diesel::AsChangeset,
)]
#[diesel(
table_name = crate::db::schema::import_definitions,
check_for_backend(diesel::sqlite::Sqlite)
)]
#[diesel(treat_none_as_default_value = false, treat_none_as_null = true)]
pub struct ImportDefinition {
pub id: String,
pub project_id: String,
pub name: String,
pub wxr_file_path: Option<String>,
pub uploads_folder_path: Option<String>,
pub last_analysis_result: Option<String>,
pub created_at: i64,
pub updated_at: i64,
}
impl ImportDefinition {
pub fn analysis(&self) -> Result<Option<ImportReport>, serde_json::Error> {
self.last_analysis_result
.as_deref()
.map(serde_json::from_str)
.transpose()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportPhase {
Taxonomy,
Posts,
Media,
Pages,
Complete,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportProgress {
pub phase: ImportPhase,
pub current: usize,
pub total: usize,
pub detail: String,
pub eta_ms: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportExecutionCounts {
pub imported: usize,
pub skipped: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportExecutionResult {
pub taxonomy: ImportExecutionCounts,
pub posts: ImportExecutionCounts,
pub media: ImportExecutionCounts,
pub pages: ImportExecutionCounts,
}

View File

@@ -1,4 +1,5 @@
mod generation;
mod import;
mod media;
pub mod metadata;
mod post;
@@ -11,6 +12,11 @@ pub use generation::{
DbNotification, GeneratedFileHash, NotificationAction, NotificationEntity,
PublishingPreferences, SshMode,
};
pub use import::{
ImportCandidate, ImportCounts, ImportDateBucket, ImportDefinition, ImportExecutionCounts,
ImportExecutionResult, ImportItemKind, ImportItemStatus, ImportMacroUsage, ImportPhase,
ImportProgress, ImportReport, ImportResolution, ImportedSite, TaxonomyCandidate, TaxonomyKind,
};
pub use media::{Media, MediaTranslation};
pub use metadata::{CategorySettings, ProjectMetadata, TagEntry};
pub use post::{Post, PostLink, PostMedia, PostStatus, PostTranslation};

View File

@@ -1738,6 +1738,7 @@ fn post_ai_content(data_dir: &Path, post: &Post) -> HostResult<Value> {
fn one_shot_json(value: engine::ai::OneShotResponse) -> HostResult<Value> {
let value = match value {
engine::ai::OneShotResponse::Taxonomy(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::ImportTaxonomyMapping(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::PostAnalysis(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::LanguageDetection(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::Translation(value) => serde_json::to_value(value),

View File

@@ -0,0 +1,807 @@
use std::fs;
use std::path::Path;
use bds_core::db::Database;
use bds_core::db::fts::ensure_fts_tables;
use bds_core::engine::{meta, post, project, tag, wordpress_import as import};
use bds_core::model::{
ImportItemKind, ImportItemStatus, ImportResolution, PostStatus, TaxonomyKind,
};
use bds_core::util::frontmatter::read_post_file;
use bds_core::util::sidecar::read_sidecar;
use image::DynamicImage;
use tempfile::TempDir;
fn setup() -> (Database, TempDir, bds_core::model::Project) {
let db = Database::open_in_memory().unwrap();
db.migrate().unwrap();
ensure_fts_tables(db.conn()).unwrap();
let dir = TempDir::new().unwrap();
let project = project::create_project(
db.conn(),
"WordPress Import",
Some(dir.path().to_str().unwrap()),
)
.unwrap();
(db, dir, project)
}
fn write_image(path: &Path) {
DynamicImage::new_rgb8(4, 3).save(path).unwrap();
}
#[test]
fn parser_rejects_malformed_or_channel_less_xml_and_ignores_unknown_elements() {
assert!(import::parse_wxr_xml("<rss><channel>").is_err());
assert!(import::parse_wxr_xml("<rss></rss>").is_err());
let parsed = import::parse_wxr_xml(&sample_wxr(
"<unknown:payload>ignored</unknown:payload><item><title>Menu</title><wp:post_type>nav_menu_item</wp:post_type></item>",
))
.unwrap();
assert_eq!(parsed.site.title, "Legacy & Blog");
assert_eq!(parsed.posts.len(), 1);
assert_eq!(parsed.pages.len(), 1);
assert_eq!(parsed.media.len(), 1);
assert_eq!(parsed.categories[0].name, "General");
assert_eq!(parsed.tags[0].name, "News");
}
#[test]
fn analysis_converts_html_and_shortcodes_and_classifies_every_status() {
let (db, dir, project) = setup();
meta::add_category(dir.path(), "GENERAL").unwrap();
tag::create_tag(db.conn(), dir.path(), &project.id, "nEWs", None).unwrap();
let update = post::create_post(
db.conn(),
dir.path(),
&project.id,
"Update",
Some("Update body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
post::update_post(
db.conn(),
dir.path(),
&update.id,
None,
Some("hello-world"),
None,
None,
None,
None,
None,
None,
None,
None,
)
.unwrap();
let duplicate = post::create_post(
db.conn(),
dir.path(),
&project.id,
"Duplicate",
Some("Duplicate body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
assert_ne!(update.id, duplicate.id);
let uploads = dir.path().join("uploads/2024/05");
fs::create_dir_all(&uploads).unwrap();
write_image(&uploads.join("photo.png"));
let wxr = dir.path().join("legacy.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let mut progress = Vec::new();
let report = import::analyze_wxr(
db.conn(),
dir.path(),
&project.id,
&wxr,
Some(dir.path().join("uploads").as_path()),
Some(&mut |value| progress.push(value)),
)
.unwrap();
assert!(
report.posts[0]
.content
.as_deref()
.unwrap()
.contains("**world**")
);
assert!(
report.posts[0]
.content
.as_deref()
.unwrap()
.contains("[[gallery ids=\"1,2\"]]"),
"{}",
report.posts[0].content.as_deref().unwrap()
);
assert_eq!(report.posts[0].status, ImportItemStatus::Conflict);
assert_eq!(report.posts[0].resolution, Some(ImportResolution::Ignore));
assert_eq!(report.pages[0].status, ImportItemStatus::New);
assert_eq!(report.media[0].status, ImportItemStatus::New);
assert!(
report
.taxonomies
.iter()
.any(|item| { item.kind == TaxonomyKind::Category && item.exists_in_project })
);
assert!(
report
.taxonomies
.iter()
.any(|item| { item.kind == TaxonomyKind::Tag && item.exists_in_project })
);
assert_eq!(report.date_distribution[0].year, 2024);
assert_eq!(report.macros[0].name, "gallery");
assert!(!progress.is_empty());
}
#[test]
fn analysis_distinguishes_new_updates_conflicts_duplicates_and_missing_media() {
let (db, dir, project) = setup();
let uploads = dir.path().join("uploads/2024/05");
fs::create_dir_all(&uploads).unwrap();
write_image(&uploads.join("same.png"));
fs::copy(uploads.join("same.png"), uploads.join("copy.png")).unwrap();
DynamicImage::new_rgb8(5, 3)
.save(uploads.join("different.png"))
.unwrap();
let old_different = dir.path().join("old-different.png");
DynamicImage::new_rgb8(2, 2).save(&old_different).unwrap();
bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
&uploads.join("same.png"),
"same.png",
None,
None,
None,
None,
None,
Vec::new(),
)
.unwrap();
bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
&old_different,
"different.png",
None,
None,
None,
None,
None,
Vec::new(),
)
.unwrap();
let same_body = "Hello **world**.\n\n[[gallery ids=\"1,2\"]]";
let update = post::create_post(
db.conn(),
dir.path(),
&project.id,
"Hello World",
Some(same_body),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
assert_eq!(update.slug, "hello-world");
let collision = post::create_post(
db.conn(),
dir.path(),
&project.id,
"Collision",
Some("Local body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
assert_eq!(collision.slug, "collision");
post::create_post(
db.conn(),
dir.path(),
&project.id,
"Local Duplicate",
Some("Shared duplicate body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
let extras = format!(
r#"
<item><title>Collision</title><content:encoded><![CDATA[Incoming body]]></content:encoded><wp:post_id>102</wp:post_id><wp:post_name>collision</wp:post_name><wp:status>draft</wp:status><wp:post_type>post</wp:post_type></item>
<item><title>Duplicate Content</title><content:encoded><![CDATA[<p>Shared duplicate body</p>]]></content:encoded><wp:post_id>103</wp:post_id><wp:post_name>duplicate-content</wp:post_name><wp:status>draft</wp:status><wp:post_type>post</wp:post_type></item>
{}
{}
{}
{}
{}"#,
attachment_item(302, "Same", "same.png"),
attachment_item(303, "Copy", "copy.png"),
attachment_item(304, "Different", "different.png"),
attachment_item(305, "Missing", "missing.png"),
attachment_item(306, "Escape", "../escape.png"),
);
let wxr = dir.path().join("classifications.xml");
fs::write(&wxr, sample_wxr(&extras)).unwrap();
let posts_before = serde_json::to_value(
bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap(),
)
.unwrap();
let media_before = serde_json::to_value(
bds_core::db::queries::media::list_media_by_project(db.conn(), &project.id).unwrap(),
)
.unwrap();
let tags_before = serde_json::to_value(
bds_core::db::queries::tag::list_tags_by_project(db.conn(), &project.id).unwrap(),
)
.unwrap();
let categories_before = fs::read(dir.path().join("meta/categories.json")).unwrap();
let report = import::analyze_wxr(
db.conn(),
dir.path(),
&project.id,
&wxr,
Some(dir.path().join("uploads").as_path()),
None,
)
.unwrap();
assert_eq!(
serde_json::to_value(
bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap()
)
.unwrap(),
posts_before
);
assert_eq!(
serde_json::to_value(
bds_core::db::queries::media::list_media_by_project(db.conn(), &project.id).unwrap()
)
.unwrap(),
media_before
);
assert_eq!(
serde_json::to_value(
bds_core::db::queries::tag::list_tags_by_project(db.conn(), &project.id).unwrap()
)
.unwrap(),
tags_before
);
assert_eq!(
fs::read(dir.path().join("meta/categories.json")).unwrap(),
categories_before
);
let post_statuses = report
.posts
.iter()
.map(|item| item.status)
.collect::<Vec<_>>();
assert!(post_statuses.contains(&ImportItemStatus::Update));
assert!(post_statuses.contains(&ImportItemStatus::Conflict));
assert!(post_statuses.contains(&ImportItemStatus::ContentDuplicate));
let media_status = |name: &str| {
report
.media
.iter()
.find(|item| item.filename.as_deref() == Some(name))
.unwrap()
.status
};
assert_eq!(media_status("same.png"), ImportItemStatus::Update);
assert_eq!(media_status("copy.png"), ImportItemStatus::ContentDuplicate);
assert_eq!(media_status("different.png"), ImportItemStatus::Conflict);
assert_eq!(media_status("missing.png"), ImportItemStatus::Missing);
assert_eq!(media_status("escape.png"), ImportItemStatus::Missing);
assert!(
report
.media
.iter()
.find(|item| item.filename.as_deref() == Some("escape.png"))
.unwrap()
.source_path
.is_none()
);
}
#[test]
fn definitions_round_trip_saved_analysis_and_are_project_scoped() {
let (db, dir, project) = setup();
let definition = import::create_definition(db.conn(), &project.id, "Legacy").unwrap();
let wxr = dir.path().join("legacy.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let report = import::analyze_wxr(db.conn(), dir.path(), &project.id, &wxr, None, None).unwrap();
let updated = import::update_definition(
db.conn(),
&definition.id,
Some("Renamed"),
Some(Some(wxr.as_path())),
Some(Some(dir.path())),
Some(Some(&report)),
)
.unwrap();
assert_eq!(updated.name, "Renamed");
assert_eq!(updated.analysis().unwrap().unwrap(), report);
assert_eq!(
import::list_definitions(db.conn(), &project.id)
.unwrap()
.len(),
1
);
import::delete_definition(db.conn(), &definition.id).unwrap();
assert!(import::get_definition(db.conn(), &definition.id).is_err());
}
#[test]
fn execution_uses_core_engines_preserves_metadata_and_links_media_parent() {
let (db, dir, project) = setup();
let uploads = dir.path().join("uploads/2024/05");
fs::create_dir_all(&uploads).unwrap();
write_image(&uploads.join("photo.png"));
let wxr = dir.path().join("legacy.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let report = import::analyze_wxr(
db.conn(),
dir.path(),
&project.id,
&wxr,
Some(dir.path().join("uploads").as_path()),
None,
)
.unwrap();
let mut progress = Vec::new();
let result = import::execute_import(
db.conn(),
dir.path(),
&project.id,
&report,
Some("Fallback Author"),
Some(&mut |value| progress.push(value)),
)
.unwrap();
assert_eq!(result.posts.imported, 1);
assert_eq!(result.pages.imported, 1);
assert_eq!(result.media.imported, 1);
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap();
let imported = posts
.iter()
.find(|post| post.slug == "hello-world")
.unwrap();
assert_eq!(imported.status, PostStatus::Published);
assert_eq!(imported.author.as_deref(), Some("Importer"));
assert_eq!(imported.created_at, 1_714_564_800_000);
assert_eq!(imported.updated_at, 1_714_653_000_000);
assert_eq!(imported.published_at, Some(1_714_564_800_000));
assert_eq!(imported.file_path, "posts/2024/05/hello-world.md");
assert!(dir.path().join(&imported.file_path).is_file());
let raw = fs::read_to_string(dir.path().join(&imported.file_path)).unwrap();
let (frontmatter, _) = read_post_file(&raw).unwrap();
assert_eq!(frontmatter.created_at, imported.created_at);
assert_eq!(frontmatter.updated_at, imported.updated_at);
assert_eq!(frontmatter.published_at, imported.published_at);
let page = posts.iter().find(|post| post.slug == "about").unwrap();
assert!(page.categories.iter().any(|category| category == "page"));
let media =
bds_core::db::queries::media::list_media_by_project(db.conn(), &project.id).unwrap();
assert_eq!(media[0].alt.as_deref(), Some("Photo description"));
assert_eq!(media[0].author.as_deref(), Some("Fallback Author"));
assert_eq!(media[0].created_at, 1_714_737_600_000);
assert!(media[0].file_path.starts_with("media/2024/05/"));
let sidecar =
read_sidecar(&fs::read_to_string(dir.path().join(&media[0].sidecar_path)).unwrap())
.unwrap();
assert_eq!(sidecar.created_at, media[0].created_at);
let links =
bds_core::db::queries::post_media::list_post_media_by_media(db.conn(), &media[0].id)
.unwrap();
assert_eq!(links[0].post_id, imported.id);
let phases = progress.iter().map(|item| item.phase).collect::<Vec<_>>();
assert!(phases.windows(2).all(|window| window[0] <= window[1]));
assert_eq!(phases.last(), Some(&bds_core::model::ImportPhase::Complete));
for phase in [
bds_core::model::ImportPhase::Taxonomy,
bds_core::model::ImportPhase::Posts,
bds_core::model::ImportPhase::Media,
bds_core::model::ImportPhase::Pages,
] {
assert!(
progress
.iter()
.any(|item| item.phase == phase && item.current == 0),
"missing start event for {phase:?}"
);
}
let diff =
bds_core::engine::metadata_diff::compute_metadata_diff(db.conn(), dir.path(), &project.id)
.unwrap();
assert!(diff.errors.is_empty(), "{:?}", diff.errors);
assert!(
diff.diffs
.iter()
.all(|item| item.entity_type != "post" && item.entity_type != "media"),
"{:?}",
diff.diffs
);
assert!(
bds_core::engine::post::rebuild_posts_from_filesystem(db.conn(), dir.path(), &project.id,)
.unwrap()
.errors
.is_empty()
);
assert!(
bds_core::engine::media::rebuild_media_from_filesystem(db.conn(), dir.path(), &project.id,)
.unwrap()
.errors
.is_empty()
);
}
#[test]
fn conflict_import_uses_unique_slug_and_progress_panics_do_not_abort() {
let (db, dir, project) = setup();
let existing = post::create_post(
db.conn(),
dir.path(),
&project.id,
"Hello World",
Some("local"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
assert_eq!(existing.slug, "hello-world");
let wxr = dir.path().join("legacy.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let mut report = import::analyze_wxr(
db.conn(),
dir.path(),
&project.id,
&wxr,
None,
Some(&mut |_| panic!("observer failure")),
)
.unwrap();
import::set_conflict_resolution(
&mut report,
ImportItemKind::Post,
"hello-world",
ImportResolution::Import,
)
.unwrap();
import::execute_import(
db.conn(),
dir.path(),
&project.id,
&report,
None,
Some(&mut |_| panic!("observer failure")),
)
.unwrap();
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap();
assert_eq!(posts.len(), 3);
assert!(
posts
.iter()
.any(|post| post.slug.starts_with("hello-world-") && post.slug != "hello-world")
);
}
#[test]
fn default_ignore_resolution_skips_conflict_without_mutating_existing_post() {
let (db, dir, project) = setup();
let existing = post::create_post(
db.conn(),
dir.path(),
&project.id,
"Hello World",
Some("Local body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
let wxr = dir.path().join("ignore.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let mut report =
import::analyze_wxr(db.conn(), dir.path(), &project.id, &wxr, None, None).unwrap();
report.taxonomies.clear();
report.pages.clear();
report.media.clear();
assert_eq!(report.posts[0].resolution, Some(ImportResolution::Ignore));
let result =
import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).unwrap();
assert_eq!(result.posts.imported, 0);
assert_eq!(result.posts.skipped, 1);
let unchanged = bds_core::db::queries::post::get_post_by_id(db.conn(), &existing.id).unwrap();
assert_eq!(unchanged.content.as_deref(), Some("Local body"));
}
#[test]
fn overwrite_resolution_preserves_existing_post_and_media_identity() {
let (db, dir, project) = setup();
let existing_post = post::create_post(
db.conn(),
dir.path(),
&project.id,
"Hello World",
Some("Local post body"),
vec![],
vec![],
None,
None,
None,
)
.unwrap();
let old_image = dir.path().join("old-photo.png");
DynamicImage::new_rgb8(2, 2).save(&old_image).unwrap();
let existing_media = bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
&old_image,
"photo.png",
Some("Old photo"),
Some("Old alt"),
None,
None,
None,
Vec::new(),
)
.unwrap();
let uploads = dir.path().join("uploads/2024/05");
fs::create_dir_all(&uploads).unwrap();
write_image(&uploads.join("photo.png"));
let wxr = dir.path().join("overwrite.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let mut report = import::analyze_wxr(
db.conn(),
dir.path(),
&project.id,
&wxr,
Some(dir.path().join("uploads").as_path()),
None,
)
.unwrap();
import::set_conflict_resolution(
&mut report,
ImportItemKind::Post,
"hello-world",
ImportResolution::Overwrite,
)
.unwrap();
import::set_conflict_resolution(
&mut report,
ImportItemKind::Media,
"photo.png",
ImportResolution::Overwrite,
)
.unwrap();
import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).unwrap();
let overwritten_post =
bds_core::db::queries::post::get_post_by_id(db.conn(), &existing_post.id).unwrap();
assert_eq!(overwritten_post.id, existing_post.id);
assert_eq!(overwritten_post.status, PostStatus::Published);
assert_eq!(
overwritten_post.published_content.as_deref(),
report.posts[0].content.as_deref()
);
let overwritten_media =
bds_core::db::queries::media::get_media_by_id(db.conn(), &existing_media.id).unwrap();
assert_eq!(overwritten_media.id, existing_media.id);
assert_eq!(overwritten_media.alt.as_deref(), Some("Photo description"));
assert_ne!(overwritten_media.checksum, existing_media.checksum);
}
#[test]
fn airplane_ai_mapping_requires_a_local_endpoint() {
let (db, _dir, _project) = setup();
let mut report = import::empty_report();
report.taxonomies.push(import::taxonomy_candidate(
TaxonomyKind::Tag,
"Legacy News",
false,
));
let error = import::auto_map_taxonomy(db.conn(), _dir.path(), &_project.id, true, &mut report)
.unwrap_err();
assert!(error.to_string().contains("airplane"));
assert!(report.taxonomies[0].mapped_to.is_none());
}
#[test]
fn execution_rejects_a_saved_taxonomy_mapping_to_a_nonexistent_term() {
let (db, dir, project) = setup();
let mut report = import::empty_report();
let mut candidate = import::taxonomy_candidate(TaxonomyKind::Category, "Old", false);
candidate.mapped_to = Some("Does Not Exist".to_string());
report.taxonomies.push(candidate);
let error = import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None)
.unwrap_err();
assert!(error.to_string().contains("mapping target does not exist"));
assert!(
!meta::read_categories_json(dir.path())
.unwrap()
.iter()
.any(|category| category == "Old")
);
}
#[test]
fn execution_commits_complete_500_item_batches_and_rolls_back_database_and_files_for_failure() {
let (db, dir, project) = setup();
let wxr = dir.path().join("legacy.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let mut report =
import::analyze_wxr(db.conn(), dir.path(), &project.id, &wxr, None, None).unwrap();
let template = report.posts[0].clone();
report.taxonomies.clear();
report.media.clear();
report.pages.clear();
report.posts = (0..=500)
.map(|index| {
let mut item = template.clone();
item.source_id = Some(10_000 + index);
item.title = format!("Batch {index}");
item.slug = Some(format!("batch-{index}"));
item.status = ImportItemStatus::New;
item.resolution = None;
item.existing_id = None;
item
})
.collect();
let mut failing = template;
failing.title = "Fail after file write".to_string();
failing.slug = Some("batch-500".to_string());
failing.status = ImportItemStatus::New;
failing.resolution = None;
failing.existing_id = None;
report.posts.push(failing);
assert!(
import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).is_err()
);
let posts = bds_core::db::queries::post::list_posts_by_project(db.conn(), &project.id).unwrap();
assert_eq!(posts.len(), 500);
assert!(posts.iter().any(|post| post.slug == "batch-499"));
assert!(!posts.iter().any(|post| post.slug == "batch-500"));
assert!(!dir.path().join("posts/2024/05/batch-500.md").exists());
assert!(dir.path().join("posts/2024/05/batch-499.md").is_file());
}
#[test]
fn failed_media_batch_restores_an_overwritten_binary_sidecar_and_database_row() {
let (db, dir, project) = setup();
let old_image = dir.path().join("old.png");
DynamicImage::new_rgb8(2, 2).save(&old_image).unwrap();
let existing = bds_core::engine::media::import_media(
db.conn(),
dir.path(),
&project.id,
&old_image,
"photo.png",
Some("Old title"),
Some("Old alt"),
None,
None,
None,
Vec::new(),
)
.unwrap();
let original_binary = fs::read(dir.path().join(&existing.file_path)).unwrap();
let original_sidecar = fs::read(dir.path().join(&existing.sidecar_path)).unwrap();
let uploads = dir.path().join("uploads/2024/05");
fs::create_dir_all(&uploads).unwrap();
write_image(&uploads.join("photo.png"));
let wxr = dir.path().join("rollback-overwrite.xml");
fs::write(&wxr, sample_wxr("")).unwrap();
let mut report = import::analyze_wxr(
db.conn(),
dir.path(),
&project.id,
&wxr,
Some(dir.path().join("uploads").as_path()),
None,
)
.unwrap();
report.taxonomies.clear();
report.posts.clear();
report.pages.clear();
import::set_conflict_resolution(
&mut report,
ImportItemKind::Media,
"photo.png",
ImportResolution::Overwrite,
)
.unwrap();
let mut missing = report.media[0].clone();
missing.title = "Missing".to_string();
missing.filename = Some("missing.png".to_string());
missing.source_path = Some(
dir.path()
.join("missing.png")
.to_string_lossy()
.into_owned(),
);
missing.status = ImportItemStatus::New;
missing.resolution = None;
missing.existing_id = None;
report.media.push(missing);
assert!(
import::execute_import(db.conn(), dir.path(), &project.id, &report, None, None).is_err()
);
let restored = bds_core::db::queries::media::get_media_by_id(db.conn(), &existing.id).unwrap();
assert_eq!(
serde_json::to_value(&restored).unwrap(),
serde_json::to_value(&existing).unwrap()
);
assert_eq!(
fs::read(dir.path().join(&existing.file_path)).unwrap(),
original_binary
);
assert_eq!(
fs::read(dir.path().join(&existing.sidecar_path)).unwrap(),
original_sidecar
);
}
fn sample_wxr(extra: &str) -> String {
format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:excerpt="http://wordpress.org/export/1.2/excerpt/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:wp="http://wordpress.org/export/1.2/" xmlns:unknown="urn:unknown">
<channel><title>Legacy &amp; Blog</title><link>https://legacy.example</link><language>en</language>{extra}
<wp:category><wp:cat_name><![CDATA[General]]></wp:cat_name><wp:category_nicename>general</wp:category_nicename><wp:category_parent></wp:category_parent></wp:category>
<wp:tag><wp:tag_slug>news</wp:tag_slug><wp:tag_name><![CDATA[News]]></wp:tag_name></wp:tag>
<item><title>Hello World</title><pubDate>Wed, 01 May 2024 12:00:00 +0000</pubDate><dc:creator><![CDATA[Importer]]></dc:creator><content:encoded><![CDATA[<p>Hello <strong>world</strong>.</p><p>[gallery ids="1,2"]</p>]]></content:encoded><excerpt:encoded>Legacy hello</excerpt:encoded><wp:post_id>101</wp:post_id><wp:post_date>2024-05-01 12:00:00</wp:post_date><wp:post_modified>2024-05-02 12:30:00</wp:post_modified><wp:post_name>hello-world</wp:post_name><wp:status>publish</wp:status><wp:post_type>post</wp:post_type><category domain="category"><![CDATA[General]]></category><category domain="post_tag"><![CDATA[News]]></category></item>
<item><title>About</title><pubDate>Thu, 02 May 2024 12:00:00 +0000</pubDate><dc:creator>Importer</dc:creator><content:encoded><![CDATA[<p>About page</p>]]></content:encoded><wp:post_id>201</wp:post_id><wp:post_date>2024-05-02 12:00:00</wp:post_date><wp:post_modified>2024-05-02 12:30:00</wp:post_modified><wp:post_name>about</wp:post_name><wp:status>draft</wp:status><wp:post_type>page</wp:post_type><category domain="category">General</category></item>
<item><title>Photo</title><pubDate>Fri, 03 May 2024 12:00:00 +0000</pubDate><content:encoded><![CDATA[Photo description]]></content:encoded><wp:post_id>301</wp:post_id><wp:post_parent>101</wp:post_parent><wp:post_name>photo</wp:post_name><wp:status>inherit</wp:status><wp:post_type>attachment</wp:post_type><wp:attachment_url>https://legacy.example/wp-content/uploads/2024/05/photo.png</wp:attachment_url></item>
</channel></rss>"#
)
}
fn attachment_item(id: i64, title: &str, filename: &str) -> String {
format!(
r#"<item><title>{title}</title><pubDate>Fri, 03 May 2024 12:00:00 +0000</pubDate><content:encoded>{title}</content:encoded><wp:post_id>{id}</wp:post_id><wp:post_parent>101</wp:post_parent><wp:post_name>{title}</wp:post_name><wp:status>inherit</wp:status><wp:post_type>attachment</wp:post_type><wp:attachment_url>https://legacy.example/wp-content/uploads/2024/05/{filename}</wp:attachment_url></item>"#
)
}