Implement complete WordPress WXR import workflow.
This commit is contained in:
62
crates/bds-core/src/db/queries/import_definition.rs
Normal file
62
crates/bds-core/src/db/queries/import_definition.rs
Normal file
@@ -0,0 +1,62 @@
|
||||
use diesel::prelude::*;
|
||||
|
||||
use crate::db::DbConnection;
|
||||
use crate::db::schema::import_definitions;
|
||||
use crate::model::ImportDefinition;
|
||||
|
||||
pub fn insert_import_definition(
|
||||
conn: &DbConnection,
|
||||
definition: &ImportDefinition,
|
||||
) -> QueryResult<()> {
|
||||
conn.with(|connection| {
|
||||
diesel::insert_into(import_definitions::table)
|
||||
.values(definition.clone())
|
||||
.execute(connection)
|
||||
.map(|_| ())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_import_definition(conn: &DbConnection, id: &str) -> QueryResult<ImportDefinition> {
|
||||
conn.with(|connection| {
|
||||
import_definitions::table
|
||||
.filter(import_definitions::id.eq(id))
|
||||
.select(ImportDefinition::as_select())
|
||||
.first(connection)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn list_import_definitions(
|
||||
conn: &DbConnection,
|
||||
project_id: &str,
|
||||
) -> QueryResult<Vec<ImportDefinition>> {
|
||||
conn.with(|connection| {
|
||||
import_definitions::table
|
||||
.filter(import_definitions::project_id.eq(project_id))
|
||||
.order((
|
||||
import_definitions::updated_at.desc(),
|
||||
import_definitions::created_at.desc(),
|
||||
))
|
||||
.select(ImportDefinition::as_select())
|
||||
.load(connection)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn update_import_definition(
|
||||
conn: &DbConnection,
|
||||
definition: &ImportDefinition,
|
||||
) -> QueryResult<()> {
|
||||
conn.with(|connection| {
|
||||
diesel::update(import_definitions::table.filter(import_definitions::id.eq(&definition.id)))
|
||||
.set(definition.clone())
|
||||
.execute(connection)
|
||||
.map(|_| ())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn delete_import_definition(conn: &DbConnection, id: &str) -> QueryResult<()> {
|
||||
conn.with(|connection| {
|
||||
diesel::delete(import_definitions::table.filter(import_definitions::id.eq(id)))
|
||||
.execute(connection)
|
||||
.map(|_| ())
|
||||
})
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod generated_file_hash;
|
||||
pub mod import_definition;
|
||||
pub mod media;
|
||||
pub mod media_translation;
|
||||
pub mod post;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::db::DbConnection as Connection;
|
||||
@@ -83,6 +84,7 @@ pub struct AiModelInfo {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum OneShotOperation {
|
||||
AnalyzeTaxonomy,
|
||||
MapImportTaxonomy,
|
||||
AnalyzePost,
|
||||
DetectLanguage,
|
||||
TranslatePost { target_language: String },
|
||||
@@ -102,6 +104,12 @@ pub struct TaxonomySuggestion {
|
||||
pub categories: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ImportTaxonomyMapping {
|
||||
pub category_mappings: BTreeMap<String, String>,
|
||||
pub tag_mappings: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PostAnalysisResult {
|
||||
pub title: String,
|
||||
@@ -146,6 +154,7 @@ pub struct TokenUsage {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum OneShotResponse {
|
||||
Taxonomy(TaxonomySuggestion),
|
||||
ImportTaxonomyMapping(ImportTaxonomyMapping),
|
||||
PostAnalysis(PostAnalysisResult),
|
||||
LanguageDetection(LanguageDetectionResult),
|
||||
Translation(TranslationResult),
|
||||
@@ -451,6 +460,7 @@ fn select_model(
|
||||
let selected = match operation {
|
||||
OneShotOperation::AnalyzeImage => settings.image_model.as_ref(),
|
||||
OneShotOperation::AnalyzeTaxonomy
|
||||
| OneShotOperation::MapImportTaxonomy
|
||||
| OneShotOperation::AnalyzePost
|
||||
| OneShotOperation::DetectLanguage
|
||||
| OneShotOperation::TranslatePost { .. }
|
||||
@@ -475,6 +485,9 @@ fn build_system_prompt(base_prompt: &str, operation: &OneShotOperation) -> Strin
|
||||
OneShotOperation::AnalyzeTaxonomy => {
|
||||
"Return only JSON with tags and categories for the post."
|
||||
}
|
||||
OneShotOperation::MapImportTaxonomy => {
|
||||
"Map each imported category and tag to an existing equivalent when one exists. Return JSON objects keyed by each imported term; omit terms without a sound match."
|
||||
}
|
||||
OneShotOperation::AnalyzePost => {
|
||||
"Return only JSON with title, excerpt, and slug suggestions for the post."
|
||||
}
|
||||
@@ -511,6 +524,11 @@ fn build_one_shot_user_content(request: &OneShotRequest) -> EngineResult<Value>
|
||||
serde_json::to_string(&request.content)?
|
||||
)
|
||||
.into()),
|
||||
OneShotOperation::MapImportTaxonomy => Ok(format!(
|
||||
"Map these imported taxonomy terms to existing project terms without inventing terms: {}",
|
||||
serde_json::to_string(&request.content)?
|
||||
)
|
||||
.into()),
|
||||
OneShotOperation::AnalyzePost => Ok(format!(
|
||||
"Analyze this post and suggest title, excerpt, and slug: {}",
|
||||
serde_json::to_string(&request.content)?
|
||||
@@ -580,6 +598,24 @@ fn response_schema(operation: &OneShotOperation) -> (&'static str, Value) {
|
||||
"required": ["tags", "categories"]
|
||||
}),
|
||||
),
|
||||
OneShotOperation::MapImportTaxonomy => (
|
||||
"import_taxonomy_mapping",
|
||||
json!({
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"category_mappings": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "type": "string" }
|
||||
},
|
||||
"tag_mappings": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"required": ["category_mappings", "tag_mappings"]
|
||||
}),
|
||||
),
|
||||
OneShotOperation::AnalyzePost => (
|
||||
"post_analysis",
|
||||
json!({
|
||||
@@ -654,6 +690,9 @@ fn parse_one_shot_response(
|
||||
OneShotOperation::AnalyzeTaxonomy => {
|
||||
OneShotResponse::Taxonomy(serde_json::from_str(content)?)
|
||||
}
|
||||
OneShotOperation::MapImportTaxonomy => {
|
||||
OneShotResponse::ImportTaxonomyMapping(serde_json::from_str(content)?)
|
||||
}
|
||||
OneShotOperation::AnalyzePost => {
|
||||
OneShotResponse::PostAnalysis(serde_json::from_str(content)?)
|
||||
}
|
||||
@@ -939,6 +978,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_one_shot_supports_import_taxonomy_mapping_via_airplane_endpoint() {
|
||||
let response = run_airplane_one_shot(
|
||||
OneShotRequest {
|
||||
operation: OneShotOperation::MapImportTaxonomy,
|
||||
content: json!({
|
||||
"imported_categories": ["Old Engineering"],
|
||||
"imported_tags": ["Old Rust"],
|
||||
"existing_categories": ["Engineering"],
|
||||
"existing_tags": ["Rust"]
|
||||
}),
|
||||
},
|
||||
r#"{"choices":[{"message":{"content":"{\"category_mappings\":{\"Old Engineering\":\"Engineering\"},\"tag_mappings\":{\"Old Rust\":\"Rust\"}}"}}],"usage":{"prompt_tokens":10,"completion_tokens":5}}"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
response,
|
||||
OneShotResponse::ImportTaxonomyMapping(ImportTaxonomyMapping {
|
||||
category_mappings: BTreeMap::from([(
|
||||
"Old Engineering".to_string(),
|
||||
"Engineering".to_string(),
|
||||
)]),
|
||||
tag_mappings: BTreeMap::from([("Old Rust".to_string(), "Rust".to_string(),)]),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_one_shot_supports_post_analysis_via_airplane_endpoint() {
|
||||
let response = run_airplane_one_shot(
|
||||
|
||||
@@ -63,6 +63,43 @@ pub fn import_media(
|
||||
author: Option<&str>,
|
||||
language: Option<&str>,
|
||||
tags: Vec<String>,
|
||||
) -> EngineResult<Media> {
|
||||
import_media_at(
|
||||
conn,
|
||||
data_dir,
|
||||
project_id,
|
||||
source_path,
|
||||
original_name,
|
||||
title,
|
||||
alt,
|
||||
caption,
|
||||
author,
|
||||
language,
|
||||
tags,
|
||||
now_unix_ms(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Import media while preserving a trusted source timestamp. Importers use
|
||||
/// this path so the canonical date-based location and sidecar metadata match
|
||||
/// the source system; interactive imports continue to use the current time.
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "arguments are the user-supplied media metadata fields plus source timestamp"
|
||||
)]
|
||||
pub(crate) fn import_media_at(
|
||||
conn: &Connection,
|
||||
data_dir: &Path,
|
||||
project_id: &str,
|
||||
source_path: &Path,
|
||||
original_name: &str,
|
||||
title: Option<&str>,
|
||||
alt: Option<&str>,
|
||||
caption: Option<&str>,
|
||||
author: Option<&str>,
|
||||
language: Option<&str>,
|
||||
tags: Vec<String>,
|
||||
created_at: i64,
|
||||
) -> EngineResult<Media> {
|
||||
// Validate file type per spec
|
||||
let ext = Path::new(original_name)
|
||||
@@ -77,12 +114,10 @@ pub fn import_media(
|
||||
}
|
||||
|
||||
let id = Uuid::new_v4().to_string();
|
||||
let now = now_unix_ms();
|
||||
|
||||
let filename = format!("{id}.{ext}");
|
||||
|
||||
// Compute target directory and copy file
|
||||
let dir_path = media_dir_path(now);
|
||||
let dir_path = media_dir_path(created_at);
|
||||
let rel_file_path = format!("{dir_path}{filename}");
|
||||
let abs_file_path = data_dir.join(&rel_file_path);
|
||||
|
||||
@@ -128,8 +163,8 @@ pub fn import_media(
|
||||
sidecar_path: sidecar_rel.clone(),
|
||||
checksum: Some(checksum),
|
||||
tags,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
created_at,
|
||||
updated_at: created_at,
|
||||
};
|
||||
|
||||
// Write sidecar
|
||||
|
||||
@@ -27,5 +27,6 @@ pub mod template_rebuild;
|
||||
pub mod validate_media;
|
||||
pub mod validate_site;
|
||||
pub mod validate_translations;
|
||||
pub mod wordpress_import;
|
||||
|
||||
pub use error::{EngineError, EngineResult};
|
||||
|
||||
1827
crates/bds-core/src/engine/wordpress_import.rs
Normal file
1827
crates/bds-core/src/engine/wordpress_import.rs
Normal file
File diff suppressed because it is too large
Load Diff
224
crates/bds-core/src/model/import.rs
Normal file
224
crates/bds-core/src/model/import.rs
Normal file
@@ -0,0 +1,224 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImportItemKind {
|
||||
Post,
|
||||
Page,
|
||||
Media,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImportItemStatus {
|
||||
New,
|
||||
Update,
|
||||
Conflict,
|
||||
ContentDuplicate,
|
||||
Missing,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImportResolution {
|
||||
Ignore,
|
||||
Overwrite,
|
||||
Import,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TaxonomyKind {
|
||||
Category,
|
||||
Tag,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct ImportedSite {
|
||||
pub title: String,
|
||||
pub url: Option<String>,
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ImportCandidate {
|
||||
pub kind: ImportItemKind,
|
||||
pub source_id: Option<i64>,
|
||||
pub title: String,
|
||||
pub slug: Option<String>,
|
||||
pub filename: Option<String>,
|
||||
pub relative_path: Option<String>,
|
||||
pub status: ImportItemStatus,
|
||||
pub resolution: Option<ImportResolution>,
|
||||
pub existing_id: Option<String>,
|
||||
pub author: Option<String>,
|
||||
pub excerpt: Option<String>,
|
||||
pub content: Option<String>,
|
||||
pub source_status: Option<String>,
|
||||
#[serde(default)]
|
||||
pub categories: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
pub source_path: Option<String>,
|
||||
pub parent_source_id: Option<i64>,
|
||||
pub created_at: Option<i64>,
|
||||
pub updated_at: Option<i64>,
|
||||
pub published_at: Option<i64>,
|
||||
pub checksum: Option<String>,
|
||||
pub mime_type: Option<String>,
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TaxonomyCandidate {
|
||||
pub kind: TaxonomyKind,
|
||||
pub name: String,
|
||||
pub slug: Option<String>,
|
||||
pub exists_in_project: bool,
|
||||
pub mapped_to: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct ImportCounts {
|
||||
pub new_count: usize,
|
||||
pub update_count: usize,
|
||||
pub conflict_count: usize,
|
||||
pub duplicate_count: usize,
|
||||
pub missing_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ImportDateBucket {
|
||||
pub year: i32,
|
||||
pub post_count: usize,
|
||||
pub media_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ImportMacroUsage {
|
||||
pub name: String,
|
||||
pub total_count: usize,
|
||||
#[serde(default)]
|
||||
pub post_slugs: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub parameters: Vec<BTreeMap<String, String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct ImportReport {
|
||||
pub source_file: String,
|
||||
pub uploads_folder: Option<String>,
|
||||
pub site: ImportedSite,
|
||||
#[serde(default)]
|
||||
pub posts: Vec<ImportCandidate>,
|
||||
#[serde(default)]
|
||||
pub pages: Vec<ImportCandidate>,
|
||||
#[serde(default)]
|
||||
pub media: Vec<ImportCandidate>,
|
||||
#[serde(default)]
|
||||
pub taxonomies: Vec<TaxonomyCandidate>,
|
||||
pub post_counts: ImportCounts,
|
||||
pub page_counts: ImportCounts,
|
||||
pub media_counts: ImportCounts,
|
||||
#[serde(default)]
|
||||
pub date_distribution: Vec<ImportDateBucket>,
|
||||
#[serde(default)]
|
||||
pub macros: Vec<ImportMacroUsage>,
|
||||
}
|
||||
|
||||
impl ImportReport {
|
||||
pub fn importable_count(&self) -> usize {
|
||||
let taxonomy = self
|
||||
.taxonomies
|
||||
.iter()
|
||||
.filter(|item| !item.exists_in_project && item.mapped_to.is_none())
|
||||
.count();
|
||||
taxonomy
|
||||
+ self
|
||||
.posts
|
||||
.iter()
|
||||
.chain(&self.media)
|
||||
.chain(&self.pages)
|
||||
.filter(|item| match item.status {
|
||||
ImportItemStatus::New => true,
|
||||
ImportItemStatus::Conflict => matches!(
|
||||
item.resolution,
|
||||
Some(ImportResolution::Overwrite | ImportResolution::Import)
|
||||
),
|
||||
_ => false,
|
||||
})
|
||||
.count()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Clone,
|
||||
PartialEq,
|
||||
Eq,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
diesel::Queryable,
|
||||
diesel::Selectable,
|
||||
diesel::Insertable,
|
||||
diesel::AsChangeset,
|
||||
)]
|
||||
#[diesel(
|
||||
table_name = crate::db::schema::import_definitions,
|
||||
check_for_backend(diesel::sqlite::Sqlite)
|
||||
)]
|
||||
#[diesel(treat_none_as_default_value = false, treat_none_as_null = true)]
|
||||
pub struct ImportDefinition {
|
||||
pub id: String,
|
||||
pub project_id: String,
|
||||
pub name: String,
|
||||
pub wxr_file_path: Option<String>,
|
||||
pub uploads_folder_path: Option<String>,
|
||||
pub last_analysis_result: Option<String>,
|
||||
pub created_at: i64,
|
||||
pub updated_at: i64,
|
||||
}
|
||||
|
||||
impl ImportDefinition {
|
||||
pub fn analysis(&self) -> Result<Option<ImportReport>, serde_json::Error> {
|
||||
self.last_analysis_result
|
||||
.as_deref()
|
||||
.map(serde_json::from_str)
|
||||
.transpose()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImportPhase {
|
||||
Taxonomy,
|
||||
Posts,
|
||||
Media,
|
||||
Pages,
|
||||
Complete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ImportProgress {
|
||||
pub phase: ImportPhase,
|
||||
pub current: usize,
|
||||
pub total: usize,
|
||||
pub detail: String,
|
||||
pub eta_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct ImportExecutionCounts {
|
||||
pub imported: usize,
|
||||
pub skipped: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct ImportExecutionResult {
|
||||
pub taxonomy: ImportExecutionCounts,
|
||||
pub posts: ImportExecutionCounts,
|
||||
pub media: ImportExecutionCounts,
|
||||
pub pages: ImportExecutionCounts,
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
mod generation;
|
||||
mod import;
|
||||
mod media;
|
||||
pub mod metadata;
|
||||
mod post;
|
||||
@@ -11,6 +12,11 @@ pub use generation::{
|
||||
DbNotification, GeneratedFileHash, NotificationAction, NotificationEntity,
|
||||
PublishingPreferences, SshMode,
|
||||
};
|
||||
pub use import::{
|
||||
ImportCandidate, ImportCounts, ImportDateBucket, ImportDefinition, ImportExecutionCounts,
|
||||
ImportExecutionResult, ImportItemKind, ImportItemStatus, ImportMacroUsage, ImportPhase,
|
||||
ImportProgress, ImportReport, ImportResolution, ImportedSite, TaxonomyCandidate, TaxonomyKind,
|
||||
};
|
||||
pub use media::{Media, MediaTranslation};
|
||||
pub use metadata::{CategorySettings, ProjectMetadata, TagEntry};
|
||||
pub use post::{Post, PostLink, PostMedia, PostStatus, PostTranslation};
|
||||
|
||||
@@ -1738,6 +1738,7 @@ fn post_ai_content(data_dir: &Path, post: &Post) -> HostResult<Value> {
|
||||
fn one_shot_json(value: engine::ai::OneShotResponse) -> HostResult<Value> {
|
||||
let value = match value {
|
||||
engine::ai::OneShotResponse::Taxonomy(value) => serde_json::to_value(value),
|
||||
engine::ai::OneShotResponse::ImportTaxonomyMapping(value) => serde_json::to_value(value),
|
||||
engine::ai::OneShotResponse::PostAnalysis(value) => serde_json::to_value(value),
|
||||
engine::ai::OneShotResponse::LanguageDetection(value) => serde_json::to_value(value),
|
||||
engine::ai::OneShotResponse::Translation(value) => serde_json::to_value(value),
|
||||
|
||||
Reference in New Issue
Block a user