Implement complete WordPress WXR import workflow.

This commit is contained in:
2026-07-19 13:17:45 +02:00
parent 9dab0ca57e
commit 429f471e4e
29 changed files with 4927 additions and 26 deletions

View File

@@ -0,0 +1,62 @@
use diesel::prelude::*;
use crate::db::DbConnection;
use crate::db::schema::import_definitions;
use crate::model::ImportDefinition;
pub fn insert_import_definition(
conn: &DbConnection,
definition: &ImportDefinition,
) -> QueryResult<()> {
conn.with(|connection| {
diesel::insert_into(import_definitions::table)
.values(definition.clone())
.execute(connection)
.map(|_| ())
})
}
pub fn get_import_definition(conn: &DbConnection, id: &str) -> QueryResult<ImportDefinition> {
conn.with(|connection| {
import_definitions::table
.filter(import_definitions::id.eq(id))
.select(ImportDefinition::as_select())
.first(connection)
})
}
pub fn list_import_definitions(
conn: &DbConnection,
project_id: &str,
) -> QueryResult<Vec<ImportDefinition>> {
conn.with(|connection| {
import_definitions::table
.filter(import_definitions::project_id.eq(project_id))
.order((
import_definitions::updated_at.desc(),
import_definitions::created_at.desc(),
))
.select(ImportDefinition::as_select())
.load(connection)
})
}
pub fn update_import_definition(
conn: &DbConnection,
definition: &ImportDefinition,
) -> QueryResult<()> {
conn.with(|connection| {
diesel::update(import_definitions::table.filter(import_definitions::id.eq(&definition.id)))
.set(definition.clone())
.execute(connection)
.map(|_| ())
})
}
pub fn delete_import_definition(conn: &DbConnection, id: &str) -> QueryResult<()> {
conn.with(|connection| {
diesel::delete(import_definitions::table.filter(import_definitions::id.eq(id)))
.execute(connection)
.map(|_| ())
})
}

View File

@@ -1,4 +1,5 @@
pub mod generated_file_hash;
pub mod import_definition;
pub mod media;
pub mod media_translation;
pub mod post;

View File

@@ -1,3 +1,4 @@
use std::collections::BTreeMap;
use std::time::Duration;
use crate::db::DbConnection as Connection;
@@ -83,6 +84,7 @@ pub struct AiModelInfo {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OneShotOperation {
AnalyzeTaxonomy,
MapImportTaxonomy,
AnalyzePost,
DetectLanguage,
TranslatePost { target_language: String },
@@ -102,6 +104,12 @@ pub struct TaxonomySuggestion {
pub categories: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportTaxonomyMapping {
pub category_mappings: BTreeMap<String, String>,
pub tag_mappings: BTreeMap<String, String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PostAnalysisResult {
pub title: String,
@@ -146,6 +154,7 @@ pub struct TokenUsage {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OneShotResponse {
Taxonomy(TaxonomySuggestion),
ImportTaxonomyMapping(ImportTaxonomyMapping),
PostAnalysis(PostAnalysisResult),
LanguageDetection(LanguageDetectionResult),
Translation(TranslationResult),
@@ -451,6 +460,7 @@ fn select_model(
let selected = match operation {
OneShotOperation::AnalyzeImage => settings.image_model.as_ref(),
OneShotOperation::AnalyzeTaxonomy
| OneShotOperation::MapImportTaxonomy
| OneShotOperation::AnalyzePost
| OneShotOperation::DetectLanguage
| OneShotOperation::TranslatePost { .. }
@@ -475,6 +485,9 @@ fn build_system_prompt(base_prompt: &str, operation: &OneShotOperation) -> Strin
OneShotOperation::AnalyzeTaxonomy => {
"Return only JSON with tags and categories for the post."
}
OneShotOperation::MapImportTaxonomy => {
"Map each imported category and tag to an existing equivalent when one exists. Return JSON objects keyed by each imported term; omit terms without a sound match."
}
OneShotOperation::AnalyzePost => {
"Return only JSON with title, excerpt, and slug suggestions for the post."
}
@@ -511,6 +524,11 @@ fn build_one_shot_user_content(request: &OneShotRequest) -> EngineResult<Value>
serde_json::to_string(&request.content)?
)
.into()),
OneShotOperation::MapImportTaxonomy => Ok(format!(
"Map these imported taxonomy terms to existing project terms without inventing terms: {}",
serde_json::to_string(&request.content)?
)
.into()),
OneShotOperation::AnalyzePost => Ok(format!(
"Analyze this post and suggest title, excerpt, and slug: {}",
serde_json::to_string(&request.content)?
@@ -580,6 +598,24 @@ fn response_schema(operation: &OneShotOperation) -> (&'static str, Value) {
"required": ["tags", "categories"]
}),
),
OneShotOperation::MapImportTaxonomy => (
"import_taxonomy_mapping",
json!({
"type": "object",
"additionalProperties": false,
"properties": {
"category_mappings": {
"type": "object",
"additionalProperties": { "type": "string" }
},
"tag_mappings": {
"type": "object",
"additionalProperties": { "type": "string" }
}
},
"required": ["category_mappings", "tag_mappings"]
}),
),
OneShotOperation::AnalyzePost => (
"post_analysis",
json!({
@@ -654,6 +690,9 @@ fn parse_one_shot_response(
OneShotOperation::AnalyzeTaxonomy => {
OneShotResponse::Taxonomy(serde_json::from_str(content)?)
}
OneShotOperation::MapImportTaxonomy => {
OneShotResponse::ImportTaxonomyMapping(serde_json::from_str(content)?)
}
OneShotOperation::AnalyzePost => {
OneShotResponse::PostAnalysis(serde_json::from_str(content)?)
}
@@ -939,6 +978,33 @@ mod tests {
);
}
#[test]
fn run_one_shot_supports_import_taxonomy_mapping_via_airplane_endpoint() {
let response = run_airplane_one_shot(
OneShotRequest {
operation: OneShotOperation::MapImportTaxonomy,
content: json!({
"imported_categories": ["Old Engineering"],
"imported_tags": ["Old Rust"],
"existing_categories": ["Engineering"],
"existing_tags": ["Rust"]
}),
},
r#"{"choices":[{"message":{"content":"{\"category_mappings\":{\"Old Engineering\":\"Engineering\"},\"tag_mappings\":{\"Old Rust\":\"Rust\"}}"}}],"usage":{"prompt_tokens":10,"completion_tokens":5}}"#,
);
assert_eq!(
response,
OneShotResponse::ImportTaxonomyMapping(ImportTaxonomyMapping {
category_mappings: BTreeMap::from([(
"Old Engineering".to_string(),
"Engineering".to_string(),
)]),
tag_mappings: BTreeMap::from([("Old Rust".to_string(), "Rust".to_string(),)]),
})
);
}
#[test]
fn run_one_shot_supports_post_analysis_via_airplane_endpoint() {
let response = run_airplane_one_shot(

View File

@@ -63,6 +63,43 @@ pub fn import_media(
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
) -> EngineResult<Media> {
import_media_at(
conn,
data_dir,
project_id,
source_path,
original_name,
title,
alt,
caption,
author,
language,
tags,
now_unix_ms(),
)
}
/// Import media while preserving a trusted source timestamp. Importers use
/// this path so the canonical date-based location and sidecar metadata match
/// the source system; interactive imports continue to use the current time.
#[expect(
clippy::too_many_arguments,
reason = "arguments are the user-supplied media metadata fields plus source timestamp"
)]
pub(crate) fn import_media_at(
conn: &Connection,
data_dir: &Path,
project_id: &str,
source_path: &Path,
original_name: &str,
title: Option<&str>,
alt: Option<&str>,
caption: Option<&str>,
author: Option<&str>,
language: Option<&str>,
tags: Vec<String>,
created_at: i64,
) -> EngineResult<Media> {
// Validate file type per spec
let ext = Path::new(original_name)
@@ -77,12 +114,10 @@ pub fn import_media(
}
let id = Uuid::new_v4().to_string();
let now = now_unix_ms();
let filename = format!("{id}.{ext}");
// Compute target directory and copy file
let dir_path = media_dir_path(now);
let dir_path = media_dir_path(created_at);
let rel_file_path = format!("{dir_path}{filename}");
let abs_file_path = data_dir.join(&rel_file_path);
@@ -128,8 +163,8 @@ pub fn import_media(
sidecar_path: sidecar_rel.clone(),
checksum: Some(checksum),
tags,
created_at: now,
updated_at: now,
created_at,
updated_at: created_at,
};
// Write sidecar

View File

@@ -27,5 +27,6 @@ pub mod template_rebuild;
pub mod validate_media;
pub mod validate_site;
pub mod validate_translations;
pub mod wordpress_import;
pub use error::{EngineError, EngineResult};

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,224 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportItemKind {
Post,
Page,
Media,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportItemStatus {
New,
Update,
Conflict,
ContentDuplicate,
Missing,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportResolution {
Ignore,
Overwrite,
Import,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaxonomyKind {
Category,
Tag,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportedSite {
pub title: String,
pub url: Option<String>,
pub language: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportCandidate {
pub kind: ImportItemKind,
pub source_id: Option<i64>,
pub title: String,
pub slug: Option<String>,
pub filename: Option<String>,
pub relative_path: Option<String>,
pub status: ImportItemStatus,
pub resolution: Option<ImportResolution>,
pub existing_id: Option<String>,
pub author: Option<String>,
pub excerpt: Option<String>,
pub content: Option<String>,
pub source_status: Option<String>,
#[serde(default)]
pub categories: Vec<String>,
#[serde(default)]
pub tags: Vec<String>,
pub source_path: Option<String>,
pub parent_source_id: Option<i64>,
pub created_at: Option<i64>,
pub updated_at: Option<i64>,
pub published_at: Option<i64>,
pub checksum: Option<String>,
pub mime_type: Option<String>,
pub description: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaxonomyCandidate {
pub kind: TaxonomyKind,
pub name: String,
pub slug: Option<String>,
pub exists_in_project: bool,
pub mapped_to: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportCounts {
pub new_count: usize,
pub update_count: usize,
pub conflict_count: usize,
pub duplicate_count: usize,
pub missing_count: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportDateBucket {
pub year: i32,
pub post_count: usize,
pub media_count: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportMacroUsage {
pub name: String,
pub total_count: usize,
#[serde(default)]
pub post_slugs: Vec<String>,
#[serde(default)]
pub parameters: Vec<BTreeMap<String, String>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportReport {
pub source_file: String,
pub uploads_folder: Option<String>,
pub site: ImportedSite,
#[serde(default)]
pub posts: Vec<ImportCandidate>,
#[serde(default)]
pub pages: Vec<ImportCandidate>,
#[serde(default)]
pub media: Vec<ImportCandidate>,
#[serde(default)]
pub taxonomies: Vec<TaxonomyCandidate>,
pub post_counts: ImportCounts,
pub page_counts: ImportCounts,
pub media_counts: ImportCounts,
#[serde(default)]
pub date_distribution: Vec<ImportDateBucket>,
#[serde(default)]
pub macros: Vec<ImportMacroUsage>,
}
impl ImportReport {
pub fn importable_count(&self) -> usize {
let taxonomy = self
.taxonomies
.iter()
.filter(|item| !item.exists_in_project && item.mapped_to.is_none())
.count();
taxonomy
+ self
.posts
.iter()
.chain(&self.media)
.chain(&self.pages)
.filter(|item| match item.status {
ImportItemStatus::New => true,
ImportItemStatus::Conflict => matches!(
item.resolution,
Some(ImportResolution::Overwrite | ImportResolution::Import)
),
_ => false,
})
.count()
}
}
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Serialize,
Deserialize,
diesel::Queryable,
diesel::Selectable,
diesel::Insertable,
diesel::AsChangeset,
)]
#[diesel(
table_name = crate::db::schema::import_definitions,
check_for_backend(diesel::sqlite::Sqlite)
)]
#[diesel(treat_none_as_default_value = false, treat_none_as_null = true)]
pub struct ImportDefinition {
pub id: String,
pub project_id: String,
pub name: String,
pub wxr_file_path: Option<String>,
pub uploads_folder_path: Option<String>,
pub last_analysis_result: Option<String>,
pub created_at: i64,
pub updated_at: i64,
}
impl ImportDefinition {
pub fn analysis(&self) -> Result<Option<ImportReport>, serde_json::Error> {
self.last_analysis_result
.as_deref()
.map(serde_json::from_str)
.transpose()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ImportPhase {
Taxonomy,
Posts,
Media,
Pages,
Complete,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImportProgress {
pub phase: ImportPhase,
pub current: usize,
pub total: usize,
pub detail: String,
pub eta_ms: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportExecutionCounts {
pub imported: usize,
pub skipped: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct ImportExecutionResult {
pub taxonomy: ImportExecutionCounts,
pub posts: ImportExecutionCounts,
pub media: ImportExecutionCounts,
pub pages: ImportExecutionCounts,
}

View File

@@ -1,4 +1,5 @@
mod generation;
mod import;
mod media;
pub mod metadata;
mod post;
@@ -11,6 +12,11 @@ pub use generation::{
DbNotification, GeneratedFileHash, NotificationAction, NotificationEntity,
PublishingPreferences, SshMode,
};
pub use import::{
ImportCandidate, ImportCounts, ImportDateBucket, ImportDefinition, ImportExecutionCounts,
ImportExecutionResult, ImportItemKind, ImportItemStatus, ImportMacroUsage, ImportPhase,
ImportProgress, ImportReport, ImportResolution, ImportedSite, TaxonomyCandidate, TaxonomyKind,
};
pub use media::{Media, MediaTranslation};
pub use metadata::{CategorySettings, ProjectMetadata, TagEntry};
pub use post::{Post, PostLink, PostMedia, PostStatus, PostTranslation};

View File

@@ -1738,6 +1738,7 @@ fn post_ai_content(data_dir: &Path, post: &Post) -> HostResult<Value> {
fn one_shot_json(value: engine::ai::OneShotResponse) -> HostResult<Value> {
let value = match value {
engine::ai::OneShotResponse::Taxonomy(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::ImportTaxonomyMapping(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::PostAnalysis(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::LanguageDetection(value) => serde_json::to_value(value),
engine::ai::OneShotResponse::Translation(value) => serde_json::to_value(value),