From 4f9be93c6d892c75e7c59f857dce1c659340bd6c Mon Sep 17 00:00:00 2001 From: Georg Bauer Date: Mon, 9 Mar 2026 22:49:25 +0100 Subject: [PATCH] Feature/worker threads generation (#43) * Add worker threads architecture plan for blog generation * fix: tries to optimize rendering, still slow * feat: moved site rendering into web worker * fix: calendar grabs from central data source for calendar * fix: feeds now use blog language content and not canonical content --------- Co-authored-by: hugo --- .vscode/settings.json | 4 +- WORKER_PLAN.md | 175 +++ src/main/database/generatedFileHashStore.ts | 24 + src/main/engine/BlogGenerationEngine.ts | 1097 +++++++++++++---- src/main/engine/DataBackedEngines.ts | 250 ++++ .../engine/GenerationPostSnapshotService.ts | 84 +- .../engine/GenerationRouteRendererFactory.ts | 66 +- src/main/engine/GenerationWorkerData.ts | 327 +++++ src/main/engine/GenerationWorkerPool.ts | 139 +++ src/main/engine/PageRenderer.ts | 12 +- src/main/engine/PostEngine.ts | 114 ++ src/main/engine/PostMediaEngine.ts | 26 + src/main/engine/RoutePageGenerationService.ts | 24 +- src/main/engine/assets/calendarRuntime.ts | 2 +- src/main/engine/generation.worker.ts | 287 +++++ src/main/ipc/blogHandlers.ts | 6 + src/main/preload.ts | 2 +- src/main/shared/electronApi.ts | 2 +- src/renderer/App.tsx | 1 - src/renderer/a2ui/components/A2UIText.tsx | 4 +- .../components/ChatSurface/ChatTranscript.tsx | 6 +- .../DocumentationView/DocumentationView.tsx | 2 +- .../components/Editor/MediaEditor.tsx | 13 +- src/renderer/components/Editor/PostEditor.tsx | 3 +- .../components/GitSidebar/GitSidebar.tsx | 2 +- .../ImportAnalysisView/ImportAnalysisView.tsx | 14 +- .../MilkdownEditor/MilkdownEditor.tsx | 2 +- .../components/ScriptsView/ScriptsView.tsx | 3 +- .../components/Sidebar/SidebarEntityList.tsx | 2 +- .../WindowTitleBar/WindowTitleBar.tsx | 2 +- src/renderer/macros/pythonMacroPreview.ts | 2 +- src/renderer/navigation/assistantPanelSpec.ts | 2 +- src/renderer/navigation/blogmarkHandling.ts | 4 +- .../navigation/blogmarkTransformOutput.ts | 4 +- src/renderer/python/PythonRuntimeManager.ts | 2 +- .../types/highlightjs-cdn-assets.d.ts | 4 + tests/engine/BlogGenerationEngine.test.ts | 153 ++- tests/engine/DataBackedEngines.test.ts | 471 +++++++ .../GenerationPostSnapshotService.test.ts | 61 + .../GenerationRouteRendererFactory.test.ts | 55 + tests/engine/GenerationWorkerData.test.ts | 236 ++++ tests/engine/GenerationWorkerPool.test.ts | 274 ++++ 42 files changed, 3617 insertions(+), 346 deletions(-) create mode 100644 WORKER_PLAN.md create mode 100644 src/main/engine/DataBackedEngines.ts create mode 100644 src/main/engine/GenerationWorkerData.ts create mode 100644 src/main/engine/GenerationWorkerPool.ts create mode 100644 src/main/engine/generation.worker.ts create mode 100644 src/renderer/types/highlightjs-cdn-assets.d.ts create mode 100644 tests/engine/DataBackedEngines.test.ts create mode 100644 tests/engine/GenerationWorkerData.test.ts create mode 100644 tests/engine/GenerationWorkerPool.test.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index ddd3786..75d27ca 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,6 +8,8 @@ "gh": true, "git add": true, "git commit": true, - "git push": true + "git push": true, + "uniq": true, + "diff": true } } diff --git a/WORKER_PLAN.md b/WORKER_PLAN.md new file mode 100644 index 0000000..3ae3dce --- /dev/null +++ b/WORKER_PLAN.md @@ -0,0 +1,175 @@ +# Worker Threads Architecture for Blog Generation + +## Problem + +Cmd-R (Render Site) is slow with 10k+ posts / ~20k pages. The rendering pipeline is **CPU-bound** (Liquid templates + Markdown parsing). All current `Promise.all` parallelism just interleaves I/O on a single core. Actual multi-core parallelism via `worker_threads` is needed. + +## Current Architecture + +``` +blogHandlers.ts (IPC entry) + → preloadGenerationData() — loads all posts, translations, menu + → Promise.all([5 section tasks]) ← runs on ONE CPU core + ├── core (root pages, sitemap, feeds, assets) + ├── single (one page per post — THE bottleneck, ~20k pages) + ├── category (paginated category index pages) + ├── tag (paginated tag index pages) + └── date (year/month/day archive pages) + +Each section calls BlogGenerationEngine.generate() which: + 1. Builds GenerationPostIndex (tags, categories, date buckets) + 2. Bulk-loads file hashes from DB + 3. Creates route renderer (Liquid + PreviewServer + cached engines) + 4. Renders pages sequentially/batched → writes files if hash changed +``` + +**Shared mutable state across sections:** +- SQLite database (libsql, WAL mode) — singleton connection +- File system output directory (but sections write to disjoint paths) +- Template caches (Liquid `cache: true`) — populated once, read-only after +- PreloadedGenerationData — read-only after creation + +**Existing worker_threads usage:** Pyodide macro workers (`pythonMacro.worker.ts`, `BlogmarkPythonWorkerRuntime.ts`) already use `worker_threads` successfully. + +## Target Architecture + +``` +MAIN THREAD WORKER THREADS (N = cpu_count - 1) +─────────── ────────────────────────────────── +blogHandlers.ts + preloadGenerationData() + serialize + partition posts + │ + ├──► Worker 1: posts[0..chunk] → own DB conn, own Liquid, render + write + ├──► Worker 2: posts[chunk..2chunk] → own DB conn, own Liquid, render + write + ├──► Worker 3: posts[2chunk..3chunk] → own DB conn, own Liquid, render + write + └──► Worker N: posts[...rest] → own DB conn, own Liquid, render + write + │ + ├── receive progress messages → TaskManager.emit() + ├── receive results → merge stats + └── return merged BlogGenerationResult +``` + +## Phased Implementation + +### Phase 1 — Single Post Worker Pool (highest impact) + +Move `generateSinglePostPages` to a worker pool. Single posts are the bottleneck (~20k of ~20k pages). Other sections stay in main thread. + +#### 1.1 Spike: Verify dependencies work in worker_threads +- [ ] Test `@libsql/client` opens a second connection in a worker thread (WAL mode) +- [ ] Test `liquidjs` renders a template in a worker thread +- [ ] Measure memory overhead per worker with 10k post metadata + +#### 1.2 Create `generation.worker.ts` +New worker entry point that: +- Receives via `workerData`: serialized options, post chunk, template roots, DB path, hash cache +- Opens its own `@libsql/client` connection (WAL mode allows concurrent readers/writers) +- Creates its own `Liquid` instance with `cache: true` + registers custom filters +- Creates its own `PageRenderer`, `PreviewServer`, route renderer +- Renders assigned posts → writes HTML files + updates file hashes in DB +- Sends progress via `parentPort.postMessage({ type: 'progress', ... })` +- Sends result via `parentPort.postMessage({ type: 'result', stats })` + +#### 1.3 Serialize `PreloadedGenerationData` +- `PostData[]` contains `Date` objects → serialize to ISO strings, parse back in worker +- Post content is lazy-loaded from filesystem → workers read post files directly +- `HtmlRewriteContext` maps → pass as plain `Record` (already partially converted) +- Each worker bulk-loads its own `generatedHashCache` from DB + +#### 1.4 Extract `PageRenderer` factory for workers +- Extract filter registration (markdown, i18n) into a shared `createPageRenderer(config)` function +- Workers call this with their own DB-backed engines +- Keep `macroTemplateCache` and `macroLiquid` as worker-local singletons (they self-populate) + +#### 1.5 Create `GenerationWorkerPool` +New class that: +- Spawns N workers (`os.cpus().length - 1`, configurable, min 1) +- Distributes post chunks to workers (round-robin or equal split) +- Collects progress messages → relays to `onProgress` callback +- Collects results → merges stats (pagesWritten, pagesSkipped) +- Handles worker errors/crashes gracefully +- Tears down workers when generation completes + +#### 1.6 Refactor `BlogGenerationEngine.generate()` coordinator +- Split into coordinator (main thread) + worker payload +- Coordinator: loads data, partitions posts, manages worker pool, merges results +- Multi-language subtree loop: each language pass creates a new set of worker tasks +- Non-single sections (core, category, tag, date) remain in main thread + +#### 1.7 Progress reporting +- Workers: `parentPort.postMessage({ type: 'progress', value, message })` +- Main thread: listen on each worker, relay to `TaskManager.emit()` → IPC → renderer +- Aggregate progress across all workers for accurate progress bar + +#### 1.8 Testing +- Unit tests: mock worker pool, test coordinator logic +- Integration test: spawn real workers with in-memory SQLite + template files +- Verify existing `BlogGenerationEngine.test.ts` tests still pass (they mock at engine boundary) + +### Phase 2 — All Sections in Workers + +Move category/tag/date sections to workers too. Each section gets one worker. + +- [ ] Category pages → one worker +- [ ] Tag pages → one worker +- [ ] Date archive pages → one worker +- [ ] Core pages stay in main thread (sitemap/feeds/assets are one-time + small) + +### Phase 3 — Python Macro Handling + +Handle posts with Python macros across worker boundaries. + +**Recommended approach: Pre-expansion pass** +1. Before distributing posts to workers, scan for Python macro markers +2. Expand macros in main thread (Pyodide is already in a worker — reuse existing `PythonMacroWorkerRuntime`) +3. Cache expanded content +4. Pass pre-expanded content to generation workers + +**Alternative approaches (if pre-expansion is too slow):** +- Workers send macro calls back to main thread via messages (RPC pattern) +- Workers skip macro posts; main thread renders them in a second pass + +## Key Files to Modify + +| File | Change | +|---|---| +| `src/main/engine/generation.worker.ts` | **NEW** — worker entry point | +| `src/main/engine/GenerationWorkerPool.ts` | **NEW** — worker pool manager | +| `src/main/engine/BlogGenerationEngine.ts` | Refactor `generate()` into coordinator | +| `src/main/engine/PageRenderer.ts` | Extract filter registration into factory function | +| `src/main/engine/GenerationRouteRendererFactory.ts` | Make usable from worker context | +| `src/main/ipc/blogHandlers.ts` | Pass DB path + template roots to worker pool | +| `src/main/engine/RoutePageGenerationService.ts` | `generateSinglePostPages` moves to worker | +| `vite.config.ts` / `tsconfig.main.json` | Worker entry point build config | + +## Data Serialization Requirements + +| Data | Size (10k posts) | Strategy | +|---|---|---| +| `BlogGenerationOptions` | ~1KB | Pass as `workerData` (plain object) | +| `PreloadedGenerationData` | ~2-5MB | Serialize Date→ISO string, pass via `workerData` | +| Post content (body) | N/A | Workers read from filesystem (lazy) | +| `HtmlRewriteContext` | ~500KB | Pass as `Record` in `workerData` | +| `generatedHashCache` | ~1MB | Each worker bulk-loads from DB independently | +| Template files | ~50KB | Workers read from filesystem | +| Progress updates | tiny | `parentPort.postMessage()` | + +## Risks & Mitigations + +| Risk | Mitigation | +|---|---| +| `@libsql/client` native bindings may not work in workers | Spike first (1.1). Fallback: use `better-sqlite3` directly in workers. | +| Memory pressure (N copies of post metadata) | Measure in spike. Could use `SharedArrayBuffer` or reduce per-worker data. | +| Pyodide macros can't run in generation workers | Phase 3 pre-expansion pass. Most posts don't use Python macros. | +| Worker crashes lose progress | Pool manager catches errors, reports partial results, coordinator can retry failed chunks. | +| Template root paths differ in packaged app | Pass `process.resourcesPath` via `workerData`. Already has CWD fallback. | +| Build configuration for worker entry point | Add worker to Vite/esbuild config (existing pattern from pythonMacro.worker.ts). | + +## Success Criteria + +- Render Site with 10k posts uses all available CPU cores +- Wall-clock time scales roughly linearly with core count (e.g., 4 cores → ~4x faster) +- No regression in output correctness (identical HTML output) +- Progress bar still works smoothly +- Memory usage stays under 2GB total diff --git a/src/main/database/generatedFileHashStore.ts b/src/main/database/generatedFileHashStore.ts index cca71c5..96172b7 100644 --- a/src/main/database/generatedFileHashStore.ts +++ b/src/main/database/generatedFileHashStore.ts @@ -66,3 +66,27 @@ export async function setGeneratedFileHash(projectId: string, relativePath: stri args: [projectId, relativePath, hash, Date.now()], }); } + +/** + * Bulk-load all file hashes for a project in a single query. + * Returns a Map from relativePath → contentHash. + */ +export async function getAllGeneratedFileHashes(projectId: string): Promise> { + const client = getDatabase().getLocalClient(); + if (!client) { + throw new Error('Database client not available'); + } + + const result = await client.execute({ + sql: 'SELECT relative_path, content_hash FROM generated_file_hashes WHERE project_id = ?', + args: [projectId], + }); + + const map = new Map(); + for (const row of result.rows) { + if (typeof row.relative_path === 'string' && typeof row.content_hash === 'string') { + map.set(row.relative_path, row.content_hash); + } + } + return map; +} diff --git a/src/main/engine/BlogGenerationEngine.ts b/src/main/engine/BlogGenerationEngine.ts index 624cd81..dbf9f6d 100644 --- a/src/main/engine/BlogGenerationEngine.ts +++ b/src/main/engine/BlogGenerationEngine.ts @@ -45,11 +45,29 @@ import { selectRequestedPosts, } from './ApplyValidationDataService'; import { getGeneratedFileHashRecord } from '../database/generatedFileHashStore'; +import { getAllGeneratedFileHashes, setGeneratedFileHash } from '../database/generatedFileHashStore'; +import { GenerationWorkerPool, type WorkerPoolResult } from './GenerationWorkerPool'; +import { + serializePostData, + serializeMediaItem, + serializeBlogGenerationOptions, + serializePostMap, + serializeDateMap, + type GenerationWorkerTask, + type SerializedPostData, +} from './GenerationWorkerData'; +import { readPostTranslationFile } from './postTranslationFileUtils'; const DEFAULT_MAX_POSTS_PER_PAGE = 50; const MIN_MAX_POSTS_PER_PAGE = 1; const MAX_MAX_POSTS_PER_PAGE = 500; +export interface PreloadedGenerationData { + publishedPosts: PostData[]; + publishedListPosts: PostData[]; + publishedRoutePosts: PostData[]; +} + export interface BlogGenerationOptions { projectId: string; projectName: string; @@ -65,6 +83,9 @@ export interface BlogGenerationOptions { categorySettings?: Record; menu?: MenuDocument; sections?: BlogGenerationSection[]; + preloadedData?: PreloadedGenerationData; + /** Database file path — required for worker thread generation. */ + dbPath?: string; } export interface CategoryMetadata extends CategoryRenderSettings { @@ -207,7 +228,10 @@ interface BlogGenerationPostEngineContract { getPost: (postId: string) => Promise; hasPublishedVersion: (postId: string) => Promise; getLinkedBy?: (postId: string) => Promise<{ id: string; title: string; slug: string }[]>; + getAllBacklinks?: () => Promise>; getPostTranslations?: (postId: string) => Promise; + getPublishedTranslationsForRoutePosts?: (publishedPosts: PostData[]) => Promise>; + getPublishedPostFilePaths?: () => Promise>; setProjectContext: (projectId: string, dataDir?: string) => void; } @@ -247,25 +271,138 @@ export class BlogGenerationEngine { }; } - private async buildPublishedRoutePosts(publishedPosts: PostData[]): Promise { - const routePosts: PostData[] = [...publishedPosts]; + private async resolvePostContents(postList: PostData[]): Promise { + const postsNeedingContent = postList.filter((p) => !p.content); + if (postsNeedingContent.length === 0) return; - if (typeof this.postEngine.getPostTranslations !== 'function') { - return routePosts; + const BATCH_SIZE = 100; + for (let i = 0; i < postsNeedingContent.length; i += BATCH_SIZE) { + const batch = postsNeedingContent.slice(i, i + BATCH_SIZE); + const results = await Promise.all( + batch.map(async (post) => { + const full = await this.postEngine.getPublishedVersion(post.id); + return { post, content: full?.content ?? '' }; + }), + ); + for (const { post, content } of results) { + post.content = content; + } } + } - for (const post of publishedPosts) { - const translations = await this.postEngine.getPostTranslations(post.id); - for (const translation of translations) { - if (translation.status !== 'published') { - continue; + /** + * Load content for posts that may be resolved translations. + * For resolved posts with translationFilePath, reads from the translation file. + * For canonical posts, falls back to getPublishedVersion. + */ + private async resolveTranslatedPostContents(postList: PostData[]): Promise { + const postsNeedingContent = postList.filter((p) => !p.content); + if (postsNeedingContent.length === 0) return; + + await Promise.all(postsNeedingContent.map(async (post) => { + const variant = post as PostData & { translationFilePath?: string }; + if (variant.translationFilePath) { + const fileData = await readPostTranslationFile(variant.translationFilePath); + if (fileData) post.content = fileData.content; + } else { + const full = await this.postEngine.getPublishedVersion(post.id); + if (full) post.content = full.content; + } + })); + } + + /** + * Create post copies with translated title/excerpt for a target language. + * Posts already in the target language are returned as-is (same reference). + * Translation variant posts (with translationSourceSlug) are never resolved. + * O(n) — one Map lookup per post. + */ + private resolvePostsForLanguage( + posts: PostData[], + targetLanguage: string, + translationsByPost: Map, + mainLanguage: string, + ): PostData[] { + if (translationsByPost.size === 0) return posts; + + const target = targetLanguage.trim().toLowerCase(); + const main = mainLanguage.trim().toLowerCase(); + + return posts.map((post) => { + // Skip translation variant posts — they're already in their language + if ((post as any).translationSourceSlug) return post; + + const postLang = (post.language || '').trim().toLowerCase(); + // A post with no explicit language is assumed to be in the project main language + const effectivePostLang = postLang || main; + if (effectivePostLang === target) return post; + + const translations = translationsByPost.get(post.id); + if (!translations) return post; + + const targetTranslation = translations.find((t) => + t.language.trim().toLowerCase() === target, + ); + if (!targetTranslation) return post; + + const resolved: PostData = { + ...post, + title: targetTranslation.title, + excerpt: targetTranslation.excerpt ?? post.excerpt, + content: '', + language: targetTranslation.language, + }; + (resolved as any).translationFilePath = targetTranslation.filePath; + // Mark as already-resolved so resolveRenderablePost skips hydration + (resolved as any).translationSourceSlug = post.slug; + return resolved; + }); + } + + private async buildPublishedRoutePosts(publishedPosts: PostData[]): Promise<{ + routePosts: PostData[]; + translationsByPost: Map; + }> { + const routePosts: PostData[] = [...publishedPosts]; + const translationsByPost = new Map(); + + if (typeof this.postEngine.getPublishedTranslationsForRoutePosts === 'function') { + const translationsMap = await this.postEngine.getPublishedTranslationsForRoutePosts(publishedPosts); + for (const post of publishedPosts) { + const translations = translationsMap.get(post.id) || []; + if (translations.length > 0) { + translationsByPost.set(post.id, translations); + } + for (const translation of translations) { + routePosts.push(this.buildPublishedTranslationVariant(post, translation)); + } + } + } else if (typeof this.postEngine.getPostTranslations === 'function') { + for (const post of publishedPosts) { + const translations = await this.postEngine.getPostTranslations(post.id); + const publishedTranslations = translations.filter((t) => t.status === 'published'); + if (publishedTranslations.length > 0) { + translationsByPost.set(post.id, publishedTranslations); + } + for (const translation of publishedTranslations) { + routePosts.push(this.buildPublishedTranslationVariant(post, translation)); } - - routePosts.push(this.buildPublishedTranslationVariant(post, translation)); } } - return routePosts; + return { routePosts, translationsByPost }; + } + + async preloadGenerationData(options: BlogGenerationOptions): Promise { + const categorySettings = resolveCategorySettings(options.categoryMetadata, options.categorySettings); + const listExcludedCategories = Object.entries(categorySettings) + .filter(([, settings]) => settings.renderInLists === false) + .map(([category]) => category); + + const { publishedPosts, publishedListPosts } = await loadPublishedGenerationSets(this.postEngine, listExcludedCategories); + const { routePosts: publishedRoutePosts } = await this.buildPublishedRoutePosts(publishedPosts); + + return { publishedPosts, publishedListPosts, publishedRoutePosts }; } async generate(options: BlogGenerationOptions, onProgress: (progress: number, message?: string) => void): Promise { @@ -288,10 +425,26 @@ export class BlogGenerationEngine { .map(([category]) => category); const maxPostsPerPage = clampMaxPostsPerPage(options.maxPostsPerPage); - const { publishedPosts, publishedListPosts } = await loadPublishedGenerationSets(this.postEngine, listExcludedCategories); - const publishedRoutePosts = await this.buildPublishedRoutePosts(publishedPosts); - onProgress(3, `Found ${publishedPosts.length} published posts`); + let publishedPosts: PostData[]; + let publishedListPosts: PostData[]; + let publishedRoutePosts: PostData[]; + let translationsByPost = new Map(); + + if (options.preloadedData) { + ({ publishedPosts, publishedListPosts, publishedRoutePosts } = options.preloadedData); + // Load translations for language resolution (rendering stays in workers) + if (typeof this.postEngine.getPublishedTranslationsForRoutePosts === 'function') { + translationsByPost = await this.postEngine.getPublishedTranslationsForRoutePosts(publishedPosts); + } + } else { + ({ publishedPosts, publishedListPosts } = await loadPublishedGenerationSets(this.postEngine, listExcludedCategories)); + const built = await this.buildPublishedRoutePosts(publishedPosts); + publishedRoutePosts = built.routePosts; + translationsByPost = built.translationsByPost; + } + + onProgress(3, `Loaded ${publishedPosts.length} published posts`); const generationPostIndex = buildGenerationPostIndex(publishedListPosts); @@ -308,6 +461,10 @@ export class BlogGenerationEngine { let calendarJson = ''; if (includeCore) { + // Pre-load content for feed posts (top N by recency) before building feeds + const feedSlice = publishedListPosts.slice(0, maxPostsPerPage); + await this.resolvePostContents(feedSlice); + onProgress(5, 'Building sitemap XML...'); const sitemapAndFeedResult = buildSitemapAndFeeds({ baseUrl: options.baseUrl, @@ -390,6 +547,12 @@ export class BlogGenerationEngine { const generatedHashCache = new Map(); const knownOutputDirectories = new Set(); + // Bulk-load all known file hashes to avoid per-page DB reads + const existingHashes = await getAllGeneratedFileHashes(options.projectId); + for (const [relativePath, hash] of existingHashes) { + generatedHashCache.set(relativePath, hash); + } + if (includeCore) { onProgress(10, 'Writing sitemap and feeds...'); sitemapWritten = await writeFileIfHashChanged({ @@ -429,241 +592,72 @@ export class BlogGenerationEngine { reportUnitProgress('Assets copied'); } - const renderRoute = createPreviewBackedGenerationRouteRenderer({ - options, - maxPostsPerPage, - publishedPostsForLookup: publishedRoutePosts, - engines: { - postEngine: this.postEngine, - mediaEngine: this.mediaEngine, - postMediaEngine: this.postMediaEngine, - }, - }); - - const writePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({ - projectId, - htmlDir, - urlPath, - content, - knownDirectories: knownOutputDirectories, - hashCache: generatedHashCache, - refreshHashTimestampOnUnchanged: true, - }); - let pagesGenerated = 0; - if (includeCore) { - onProgress(20, 'Generating root pages...'); - pagesGenerated += await generateRootPages({ - projectId: options.projectId, - posts: publishedListPosts, - maxPostsPerPage, - renderRoute, - writePage, - onPageGenerated: reportUnitProgress, - }); - pagesGenerated += await generatePageRoutes({ - projectId: options.projectId, - posts: publishedRoutePosts, - renderRoute, - writePage, - onPageGenerated: reportUnitProgress, - }); - } - - if (includeSingle) { - onProgress(35, 'Generating single post pages...'); - pagesGenerated += await generateSinglePostPages({ - projectId: options.projectId, - posts: publishedRoutePosts, - renderRoute, - writePage, - onPageGenerated: reportUnitProgress, - }); - } - - if (includeCategory) { - onProgress(50, 'Generating category pages...'); - pagesGenerated += await generateCategoryPages({ - projectId: options.projectId, - posts: publishedListPosts, - allCategories, - maxPostsPerPage, - renderRoute, - writePage, - onPageGenerated: reportUnitProgress, - postsByCategory: generationPostIndex.postsByCategory, - }); - } - - if (includeTag) { - onProgress(65, 'Generating tag pages...'); - pagesGenerated += await generateTagPages({ - projectId: options.projectId, - posts: publishedListPosts, - allTags, - maxPostsPerPage, - renderRoute, - writePage, - onPageGenerated: reportUnitProgress, - postsByTag: generationPostIndex.postsByTag, - }); - } - - if (includeDate) { - onProgress(80, 'Generating date archive pages...'); - pagesGenerated += await generateDateArchivePages({ - projectId: options.projectId, - posts: publishedListPosts, - yearsMap: years, - yearMonthsMap: yearMonths, - yearMonthDaysMap: yearMonthDays, - maxPostsPerPage, - renderRoute, - writePage, - onPageGenerated: reportUnitProgress, - postsByYear: generationPostIndex.postsByYear, - postsByYearMonth: generationPostIndex.postsByYearMonth, - postsByYearMonthDay: generationPostIndex.postsByYearMonthDay, - }); - } - - // --- Alternative language subtree generation --- + // --- Alternative language subtree data preparation --- const mainLanguage = (options.language ?? 'en').trim().toLowerCase(); const additionalLanguages = (options.blogLanguages ?? []) .map((lang) => lang.trim().toLowerCase()) .filter((lang) => lang.length > 0 && lang !== mainLanguage); - for (const lang of additionalLanguages) { - onProgress(85, `Generating ${lang} language subtree...`); + // Determine whether to use worker threads for page generation + const useWorkers = !!options.dbPath; - // Filter out doNotTranslate posts - const langPosts = publishedPosts.filter((p) => !(p as PostData & { doNotTranslate?: boolean }).doNotTranslate); - const langListPosts = publishedListPosts.filter((p) => !(p as PostData & { doNotTranslate?: boolean }).doNotTranslate); - - const langPostIndex = buildGenerationPostIndex(langListPosts); - const langArchiveMetadata = collectSitemapArchiveMetadata({ - baseUrl: options.baseUrl, + if (useWorkers) { + // ── Worker-based generation ──────────────────────────────────────── + pagesGenerated += await this.generateWithWorkers({ + options, maxPostsPerPage, - publishedPosts: langPosts, - publishedListPosts: langListPosts, - }); - - // Build per-language feeds - if (includeCore) { - const langFeedResult = buildSitemapAndFeeds({ - baseUrl: `${options.baseUrl}/${lang}`, - projectName: options.projectName, - projectDescription: options.projectDescription, - maxPostsPerPage, - publishedPosts: langPosts, - publishedListPosts: langListPosts, - postIndex: langPostIndex, - includeFeeds: true, - feedLanguage: lang, - }); - - const langRssPath = path.join(htmlDir, lang, 'rss.xml'); - const langAtomPath = path.join(htmlDir, lang, 'atom.xml'); - await fs.mkdir(path.join(htmlDir, lang), { recursive: true }); - await writeFileIfHashChanged({ projectId: options.projectId, filePath: langRssPath, relativePath: `${lang}/rss.xml`, content: langFeedResult.rssXml }); - await writeFileIfHashChanged({ projectId: options.projectId, filePath: langAtomPath, relativePath: `${lang}/atom.xml`, content: langFeedResult.atomXml }); - } - - const langRenderRoute = createPreviewBackedGenerationRouteRenderer({ - options: { ...options, language: lang }, - maxPostsPerPage, - publishedPostsForLookup: langPosts, - languagePrefix: `/${lang}`, - engines: { - postEngine: this.postEngine, - mediaEngine: this.mediaEngine, - postMediaEngine: this.postMediaEngine, - }, - }); - - const langWritePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({ - projectId, htmlDir, - urlPath: `${lang}/${urlPath}`, - content, - knownDirectories: knownOutputDirectories, - hashCache: generatedHashCache, - refreshHashTimestampOnUnchanged: true, + publishedPosts, + publishedListPosts, + publishedRoutePosts, + generationPostIndex, + allCategories, + allTags, + years, + yearMonths, + yearMonthDays, + generatedHashCache, + mainLanguage, + additionalLanguages, + translationsByPost, + includeCore, + includeSingle, + includeCategory, + includeTag, + includeDate, + onProgress, + reportUnitProgress, + }); + } else { + // ── Main-thread fallback (tests / no dbPath) ──────────────────── + pagesGenerated += await this.generateOnMainThread({ + options, + maxPostsPerPage, + htmlDir, + publishedPosts, + publishedListPosts, + publishedRoutePosts, + generationPostIndex, + allCategories, + allTags, + years, + yearMonths, + yearMonthDays, + knownOutputDirectories, + generatedHashCache, + mainLanguage, + additionalLanguages, + translationsByPost, + includeCore, + includeSingle, + includeCategory, + includeTag, + includeDate, + onProgress, + reportUnitProgress, }); - - const langReportProgress = (message: string) => reportUnitProgress(`[${lang}] ${message}`); - - if (includeCore) { - pagesGenerated += await generateRootPages({ - projectId: options.projectId, - posts: langListPosts, - maxPostsPerPage, - renderRoute: langRenderRoute, - writePage: langWritePage, - onPageGenerated: langReportProgress, - }); - pagesGenerated += await generatePageRoutes({ - projectId: options.projectId, - posts: langPosts, - renderRoute: langRenderRoute, - writePage: langWritePage, - onPageGenerated: langReportProgress, - }); - } - - if (includeSingle) { - pagesGenerated += await generateSinglePostPages({ - projectId: options.projectId, - posts: langPosts, - renderRoute: langRenderRoute, - writePage: langWritePage, - onPageGenerated: langReportProgress, - }); - } - - if (includeCategory) { - pagesGenerated += await generateCategoryPages({ - projectId: options.projectId, - posts: langListPosts, - allCategories: langArchiveMetadata.allCategories, - maxPostsPerPage, - renderRoute: langRenderRoute, - writePage: langWritePage, - onPageGenerated: langReportProgress, - postsByCategory: langPostIndex.postsByCategory, - }); - } - - if (includeTag) { - pagesGenerated += await generateTagPages({ - projectId: options.projectId, - posts: langListPosts, - allTags: langArchiveMetadata.allTags, - maxPostsPerPage, - renderRoute: langRenderRoute, - writePage: langWritePage, - onPageGenerated: langReportProgress, - postsByTag: langPostIndex.postsByTag, - }); - } - - if (includeDate) { - pagesGenerated += await generateDateArchivePages({ - projectId: options.projectId, - posts: langListPosts, - yearsMap: langArchiveMetadata.years, - yearMonthsMap: langArchiveMetadata.yearMonths, - yearMonthDaysMap: langArchiveMetadata.yearMonthDays, - maxPostsPerPage, - renderRoute: langRenderRoute, - writePage: langWritePage, - onPageGenerated: langReportProgress, - postsByYear: langPostIndex.postsByYear, - postsByYearMonth: langPostIndex.postsByYearMonth, - postsByYearMonthDay: langPostIndex.postsByYearMonthDay, - }); - } } // --- Combined sitemap with hreflang (if multiple languages) --- @@ -718,6 +712,627 @@ export class BlogGenerationEngine { }; } + // ── Worker-based page generation ───────────────────────────────────────── + + private async generateWithWorkers(params: { + options: BlogGenerationOptions; + maxPostsPerPage: number; + htmlDir: string; + publishedPosts: PostData[]; + publishedListPosts: PostData[]; + publishedRoutePosts: PostData[]; + generationPostIndex: GenerationPostIndex; + allCategories: Set; + allTags: Set; + years: Map; + yearMonths: Map; + yearMonthDays: Map; + generatedHashCache: Map; + mainLanguage: string; + additionalLanguages: string[]; + translationsByPost: Map; + includeCore: boolean; + includeSingle: boolean; + includeCategory: boolean; + includeTag: boolean; + includeDate: boolean; + onProgress: (progress: number, message?: string) => void; + reportUnitProgress: (message: string) => void; + }): Promise { + const { + options, maxPostsPerPage, htmlDir, + publishedPosts, publishedListPosts, publishedRoutePosts, + generationPostIndex, allCategories, allTags, years, yearMonths, yearMonthDays, + generatedHashCache, + mainLanguage, additionalLanguages, translationsByPost, + includeCore, includeSingle, includeCategory, includeTag, includeDate, + onProgress, reportUnitProgress, + } = params; + + // Pre-load media data for worker serialization + const rawMedia = await this.mediaEngine.getAllMedia(); + const mediaItems = rawMedia.map(serializeMediaItem); + + // Pre-load backlinks + let backlinksRecord: Record> = {}; + if (typeof this.postEngine.getAllBacklinks === 'function') { + const blMap = await this.postEngine.getAllBacklinks(); + for (const [postId, links] of blMap) { + backlinksRecord[postId] = links; + } + } + + const serializedOptions = serializeBlogGenerationOptions(options); + + // Pre-load post file paths for worker-side lazy content resolution + let postFilePathEntries: Array<[string, string]> = []; + if (typeof this.postEngine.getPublishedPostFilePaths === 'function') { + const filePathMap = await this.postEngine.getPublishedPostFilePaths(); + postFilePathEntries = Array.from(filePathMap); + } + + // Pre-load post-media links for worker-side gallery/album macros + let postMediaLinksEntries: Array<[string, Array<{ mediaId: string; sortOrder: number }>]> = []; + if (typeof this.postMediaEngine.getAllPostMediaLinks === 'function') { + const linksMap = await this.postMediaEngine.getAllPostMediaLinks(); + postMediaLinksEntries = Array.from(linksMap); + } + + // Serialize hash cache as [relativePath, hash] tuples for workers + const hashMapEntries: Array<[string, string]> = []; + for (const [relativePath, hash] of generatedHashCache) { + if (hash !== null) { + hashMapEntries.push([relativePath, hash]); + } + } + + // Resolve posts to project main language before serialization + const mainLangRoutePosts = this.resolvePostsForLanguage(publishedRoutePosts, mainLanguage, translationsByPost, mainLanguage); + const mainLangListPosts = this.resolvePostsForLanguage(publishedListPosts, mainLanguage, translationsByPost, mainLanguage); + const mainLangPostIndex = buildGenerationPostIndex(mainLangListPosts); + const serializedRoutePosts = mainLangRoutePosts.map(serializePostData); + const serializedListPosts = mainLangListPosts.map(serializePostData); + + // Build base task data shared across all tasks + const baseTaskData = { + lookupPosts: serializedRoutePosts, + mediaItems, + backlinksMap: backlinksRecord, + options: serializedOptions, + maxPostsPerPage, + htmlDir, + hashMapEntries, + postFilePathEntries, + postMediaLinksEntries, + }; + + const tasks: GenerationWorkerTask[] = []; + let taskCounter = 0; + const nextTaskId = (section: string, lang?: string) => + `gen-${section}${lang ? `-${lang}` : ''}-${++taskCounter}`; + + // ── Main language tasks ────────────────────────────────────────── + + if (includeCore) { + tasks.push({ + ...baseTaskData, + taskId: nextTaskId('core'), + section: 'core', + posts: serializedListPosts, + }); + } + + if (includeSingle) { + // Split single posts across multiple workers + const workerCount = Math.max(1, Math.min( + require('os').cpus().length - 1, + Math.ceil(serializedRoutePosts.length / 100), + )); + const chunkSize = Math.ceil(serializedRoutePosts.length / workerCount); + for (let i = 0; i < serializedRoutePosts.length; i += chunkSize) { + const chunk = serializedRoutePosts.slice(i, i + chunkSize); + tasks.push({ + ...baseTaskData, + taskId: nextTaskId('single'), + section: 'single', + posts: chunk, + }); + } + } + + if (includeCategory) { + tasks.push({ + ...baseTaskData, + taskId: nextTaskId('category'), + section: 'category', + posts: serializedListPosts, + allCategories: Array.from(allCategories), + postsByCategoryEntries: serializePostMap(mainLangPostIndex.postsByCategory), + }); + } + + if (includeTag) { + tasks.push({ + ...baseTaskData, + taskId: nextTaskId('tag'), + section: 'tag', + posts: serializedListPosts, + allTags: Array.from(allTags), + postsByTagEntries: serializePostMap(mainLangPostIndex.postsByTag), + }); + } + + if (includeDate) { + tasks.push({ + ...baseTaskData, + taskId: nextTaskId('date'), + section: 'date', + posts: serializedListPosts, + yearsEntries: serializeDateMap(years), + yearMonthsEntries: serializeDateMap(yearMonths), + yearMonthDaysEntries: serializeDateMap(yearMonthDays), + postsByYearEntries: serializePostMap(mainLangPostIndex.postsByYear), + postsByYearMonthEntries: serializePostMap(mainLangPostIndex.postsByYearMonth), + postsByYearMonthDayEntries: serializePostMap(mainLangPostIndex.postsByYearMonthDay), + }); + } + + // ── Language subtree tasks ──────────────────────────────────────── + + for (const lang of additionalLanguages) { + const langPosts = publishedPosts.filter((p) => !p.doNotTranslate); + const langListPosts = publishedListPosts.filter((p) => !p.doNotTranslate); + + const langPostIndex = buildGenerationPostIndex(langListPosts); + const langArchiveMetadata = collectSitemapArchiveMetadata({ + baseUrl: options.baseUrl, + maxPostsPerPage, + publishedPosts: langPosts, + publishedListPosts: langListPosts, + }); + + const resolvedLangPosts = this.resolvePostsForLanguage(langPosts, lang, translationsByPost, mainLanguage); + const resolvedLangListPosts = this.resolvePostsForLanguage(langListPosts, lang, translationsByPost, mainLanguage); + const resolvedLangPostIndex = buildGenerationPostIndex(resolvedLangListPosts); + + // Write per-language feeds in main thread (small I/O work) + if (includeCore) { + const langFeedSlice = resolvedLangListPosts.slice(0, maxPostsPerPage); + await this.resolveTranslatedPostContents(langFeedSlice); + + const langFeedResult = buildSitemapAndFeeds({ + baseUrl: `${options.baseUrl}/${lang}`, + projectName: options.projectName, + projectDescription: options.projectDescription, + maxPostsPerPage, + publishedPosts: langPosts, + publishedListPosts: resolvedLangListPosts, + postIndex: langPostIndex, + includeFeeds: true, + feedLanguage: lang, + }); + + const langRssPath = path.join(htmlDir, lang, 'rss.xml'); + const langAtomPath = path.join(htmlDir, lang, 'atom.xml'); + await fs.mkdir(path.join(htmlDir, lang), { recursive: true }); + await writeFileIfHashChanged({ projectId: options.projectId, filePath: langRssPath, relativePath: `${lang}/rss.xml`, content: langFeedResult.rssXml }); + await writeFileIfHashChanged({ projectId: options.projectId, filePath: langAtomPath, relativePath: `${lang}/atom.xml`, content: langFeedResult.atomXml }); + } + + const serializedLangPosts = resolvedLangPosts.map(serializePostData); + const serializedLangListPosts = resolvedLangListPosts.map(serializePostData); + + const langBaseTaskData = { + ...baseTaskData, + lookupPosts: serializedLangPosts, + options: { ...serializedOptions, language: lang }, + languagePrefix: `/${lang}`, + mainLanguage, + }; + + if (includeCore) { + tasks.push({ + ...langBaseTaskData, + taskId: nextTaskId('core', lang), + section: 'core' as const, + posts: serializedLangListPosts, + }); + } + + if (includeSingle) { + const workerCount = Math.max(1, Math.min( + require('os').cpus().length - 1, + Math.ceil(serializedLangPosts.length / 100), + )); + const chunkSize = Math.ceil(serializedLangPosts.length / workerCount); + for (let i = 0; i < serializedLangPosts.length; i += chunkSize) { + tasks.push({ + ...langBaseTaskData, + taskId: nextTaskId('single', lang), + section: 'single' as const, + posts: serializedLangPosts.slice(i, i + chunkSize), + }); + } + } + + if (includeCategory) { + tasks.push({ + ...langBaseTaskData, + taskId: nextTaskId('category', lang), + section: 'category' as const, + posts: serializedLangListPosts, + allCategories: Array.from(langArchiveMetadata.allCategories), + postsByCategoryEntries: serializePostMap(resolvedLangPostIndex.postsByCategory), + }); + } + + if (includeTag) { + tasks.push({ + ...langBaseTaskData, + taskId: nextTaskId('tag', lang), + section: 'tag' as const, + posts: serializedLangListPosts, + allTags: Array.from(langArchiveMetadata.allTags), + postsByTagEntries: serializePostMap(resolvedLangPostIndex.postsByTag), + }); + } + + if (includeDate) { + tasks.push({ + ...langBaseTaskData, + taskId: nextTaskId('date', lang), + section: 'date' as const, + posts: serializedLangListPosts, + yearsEntries: serializeDateMap(langArchiveMetadata.years), + yearMonthsEntries: serializeDateMap(langArchiveMetadata.yearMonths), + yearMonthDaysEntries: serializeDateMap(langArchiveMetadata.yearMonthDays), + postsByYearEntries: serializePostMap(resolvedLangPostIndex.postsByYear), + postsByYearMonthEntries: serializePostMap(resolvedLangPostIndex.postsByYearMonth), + postsByYearMonthDayEntries: serializePostMap(resolvedLangPostIndex.postsByYearMonthDay), + }); + } + } + + // ── Dispatch to worker pool ────────────────────────────────────── + + onProgress(15, `Dispatching ${tasks.length} tasks to worker pool...`); + + const pool = new GenerationWorkerPool(); + const result = await pool.runTasks(tasks, reportUnitProgress); + + if (result.errors.length > 0) { + console.error(`[GenerationWorkerPool] ${result.errors.length} task(s) failed:`); + for (const err of result.errors) { + console.error(` [${err.taskId}] ${err.error}`); + } + } + + // Persist hash updates collected from workers (single DB connection, no contention) + if (result.hashUpdates.length > 0) { + for (const update of result.hashUpdates) { + await setGeneratedFileHash(options.projectId, update.relativePath, update.hash); + } + } + + return result.pagesGenerated; + } + + // ── Main-thread page generation (fallback / tests) ─────────────────── + + private async generateOnMainThread(params: { + options: BlogGenerationOptions; + maxPostsPerPage: number; + htmlDir: string; + publishedPosts: PostData[]; + publishedListPosts: PostData[]; + publishedRoutePosts: PostData[]; + generationPostIndex: GenerationPostIndex; + allCategories: Set; + allTags: Set; + years: Map; + yearMonths: Map; + yearMonthDays: Map; + knownOutputDirectories: Set; + generatedHashCache: Map; + mainLanguage: string; + additionalLanguages: string[]; + translationsByPost: Map; + includeCore: boolean; + includeSingle: boolean; + includeCategory: boolean; + includeTag: boolean; + includeDate: boolean; + onProgress: (progress: number, message?: string) => void; + reportUnitProgress: (message: string) => void; + }): Promise { + const { + options, maxPostsPerPage, htmlDir, + publishedPosts, publishedListPosts, publishedRoutePosts, + generationPostIndex, allCategories, allTags, years, yearMonths, yearMonthDays, + knownOutputDirectories, generatedHashCache, + mainLanguage, additionalLanguages, translationsByPost, + includeCore, includeSingle, includeCategory, includeTag, includeDate, + onProgress, reportUnitProgress, + } = params; + + // Wrap post engine to resolve translations in getPostsFiltered results. + // The route renderer calls getPostsFiltered internally for list pages, + // so we need to ensure it returns language-resolved posts with content loaded. + const createResolvedPostEngine = (targetLang: string, filterDoNotTranslate = false) => { + if (translationsByPost.size === 0 && !filterDoNotTranslate) return this.postEngine; + return new Proxy(this.postEngine as any, { + get: (target: any, prop: string | symbol) => { + if (prop === 'getPostsFiltered') { + return async (filter: any) => { + let posts: PostData[] = await target.getPostsFiltered(filter); + if (filterDoNotTranslate) { + posts = posts.filter((p: PostData) => !p.doNotTranslate); + } + const resolved = this.resolvePostsForLanguage(posts, targetLang, translationsByPost, mainLanguage); + // Load translation content for resolved posts that need it (list pages render content) + await Promise.all(resolved.map(async (post) => { + const variant = post as PostData & { translationFilePath?: string }; + if (!post.content && variant.translationFilePath) { + const fileData = await readPostTranslationFile(variant.translationFilePath); + if (fileData) { + post.content = fileData.content; + } + } + })); + return resolved; + }; + } + const val = target[prop]; + return typeof val === 'function' ? val.bind(target) : val; + }, + }); + }; + + const mainLangRoutePosts = this.resolvePostsForLanguage(publishedRoutePosts, mainLanguage, translationsByPost, mainLanguage); + const mainLangListPosts = this.resolvePostsForLanguage(publishedListPosts, mainLanguage, translationsByPost, mainLanguage); + const mainLangPostIndex = buildGenerationPostIndex(mainLangListPosts); + + const renderRoute = createPreviewBackedGenerationRouteRenderer({ + options, + maxPostsPerPage, + publishedPostsForLookup: mainLangRoutePosts, + engines: { + postEngine: createResolvedPostEngine(mainLanguage) as any, + mediaEngine: this.mediaEngine, + postMediaEngine: this.postMediaEngine, + }, + }); + + const writePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({ + projectId, + htmlDir, + urlPath, + content, + knownDirectories: knownOutputDirectories, + hashCache: generatedHashCache, + refreshHashTimestampOnUnchanged: true, + }); + + let pagesGenerated = 0; + + if (includeCore) { + onProgress(20, 'Generating root pages...'); + pagesGenerated += await generateRootPages({ + projectId: options.projectId, + posts: mainLangListPosts, + maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated: reportUnitProgress, + }); + pagesGenerated += await generatePageRoutes({ + projectId: options.projectId, + posts: mainLangRoutePosts, + renderRoute, + writePage, + onPageGenerated: reportUnitProgress, + }); + } + + if (includeSingle) { + onProgress(35, 'Generating single post pages...'); + pagesGenerated += await generateSinglePostPages({ + projectId: options.projectId, + posts: mainLangRoutePosts, + renderRoute, + writePage, + onPageGenerated: reportUnitProgress, + }); + } + + if (includeCategory) { + onProgress(50, 'Generating category pages...'); + pagesGenerated += await generateCategoryPages({ + projectId: options.projectId, + posts: mainLangListPosts, + allCategories, + maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated: reportUnitProgress, + postsByCategory: mainLangPostIndex.postsByCategory, + }); + } + + if (includeTag) { + onProgress(65, 'Generating tag pages...'); + pagesGenerated += await generateTagPages({ + projectId: options.projectId, + posts: mainLangListPosts, + allTags, + maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated: reportUnitProgress, + postsByTag: mainLangPostIndex.postsByTag, + }); + } + + if (includeDate) { + onProgress(80, 'Generating date archive pages...'); + pagesGenerated += await generateDateArchivePages({ + projectId: options.projectId, + posts: mainLangListPosts, + yearsMap: years, + yearMonthsMap: yearMonths, + yearMonthDaysMap: yearMonthDays, + maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated: reportUnitProgress, + postsByYear: mainLangPostIndex.postsByYear, + postsByYearMonth: mainLangPostIndex.postsByYearMonth, + postsByYearMonthDay: mainLangPostIndex.postsByYearMonthDay, + }); + } + + // --- Alternative language subtree generation --- + for (const lang of additionalLanguages) { + onProgress(85, `Generating ${lang} language subtree...`); + + const langPosts = publishedPosts.filter((p) => !p.doNotTranslate); + const langListPosts = publishedListPosts.filter((p) => !p.doNotTranslate); + + const langPostIndex = buildGenerationPostIndex(langListPosts); + const langArchiveMetadata = collectSitemapArchiveMetadata({ + baseUrl: options.baseUrl, + maxPostsPerPage, + publishedPosts: langPosts, + publishedListPosts: langListPosts, + }); + + const resolvedLangPosts = this.resolvePostsForLanguage(langPosts, lang, translationsByPost, mainLanguage); + const resolvedLangListPosts = this.resolvePostsForLanguage(langListPosts, lang, translationsByPost, mainLanguage); + const resolvedLangPostIndex = buildGenerationPostIndex(resolvedLangListPosts); + + if (includeCore) { + const langFeedSlice = resolvedLangListPosts.slice(0, maxPostsPerPage); + await this.resolveTranslatedPostContents(langFeedSlice); + + const langFeedResult = buildSitemapAndFeeds({ + baseUrl: `${options.baseUrl}/${lang}`, + projectName: options.projectName, + projectDescription: options.projectDescription, + maxPostsPerPage, + publishedPosts: langPosts, + publishedListPosts: resolvedLangListPosts, + postIndex: langPostIndex, + includeFeeds: true, + feedLanguage: lang, + }); + + const langRssPath = path.join(htmlDir, lang, 'rss.xml'); + const langAtomPath = path.join(htmlDir, lang, 'atom.xml'); + await fs.mkdir(path.join(htmlDir, lang), { recursive: true }); + await writeFileIfHashChanged({ projectId: options.projectId, filePath: langRssPath, relativePath: `${lang}/rss.xml`, content: langFeedResult.rssXml }); + await writeFileIfHashChanged({ projectId: options.projectId, filePath: langAtomPath, relativePath: `${lang}/atom.xml`, content: langFeedResult.atomXml }); + } + + const langRenderRoute = createPreviewBackedGenerationRouteRenderer({ + options: { ...options, language: lang }, + projectMainLanguage: mainLanguage, + maxPostsPerPage, + publishedPostsForLookup: resolvedLangPosts, + languagePrefix: `/${lang}`, + engines: { + postEngine: createResolvedPostEngine(lang, true) as any, + mediaEngine: this.mediaEngine, + postMediaEngine: this.postMediaEngine, + }, + }); + + const langWritePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({ + projectId, + htmlDir, + urlPath: `${lang}/${urlPath}`, + content, + knownDirectories: knownOutputDirectories, + hashCache: generatedHashCache, + refreshHashTimestampOnUnchanged: true, + }); + + const langReportProgress = (message: string) => reportUnitProgress(`[${lang}] ${message}`); + + if (includeCore) { + pagesGenerated += await generateRootPages({ + projectId: options.projectId, + posts: resolvedLangListPosts, + maxPostsPerPage, + renderRoute: langRenderRoute, + writePage: langWritePage, + onPageGenerated: langReportProgress, + }); + pagesGenerated += await generatePageRoutes({ + projectId: options.projectId, + posts: resolvedLangPosts, + renderRoute: langRenderRoute, + writePage: langWritePage, + onPageGenerated: langReportProgress, + }); + } + + if (includeSingle) { + pagesGenerated += await generateSinglePostPages({ + projectId: options.projectId, + posts: resolvedLangPosts, + renderRoute: langRenderRoute, + writePage: langWritePage, + onPageGenerated: langReportProgress, + }); + } + + if (includeCategory) { + pagesGenerated += await generateCategoryPages({ + projectId: options.projectId, + posts: resolvedLangListPosts, + allCategories: langArchiveMetadata.allCategories, + maxPostsPerPage, + renderRoute: langRenderRoute, + writePage: langWritePage, + onPageGenerated: langReportProgress, + postsByCategory: resolvedLangPostIndex.postsByCategory, + }); + } + + if (includeTag) { + pagesGenerated += await generateTagPages({ + projectId: options.projectId, + posts: resolvedLangListPosts, + allTags: langArchiveMetadata.allTags, + maxPostsPerPage, + renderRoute: langRenderRoute, + writePage: langWritePage, + onPageGenerated: langReportProgress, + postsByTag: resolvedLangPostIndex.postsByTag, + }); + } + + if (includeDate) { + pagesGenerated += await generateDateArchivePages({ + projectId: options.projectId, + posts: resolvedLangListPosts, + yearsMap: langArchiveMetadata.years, + yearMonthsMap: langArchiveMetadata.yearMonths, + yearMonthDaysMap: langArchiveMetadata.yearMonthDays, + maxPostsPerPage, + renderRoute: langRenderRoute, + writePage: langWritePage, + onPageGenerated: langReportProgress, + postsByYear: resolvedLangPostIndex.postsByYear, + postsByYearMonth: resolvedLangPostIndex.postsByYearMonth, + postsByYearMonthDay: resolvedLangPostIndex.postsByYearMonthDay, + }); + } + } + + return pagesGenerated; + } + async regenerateCalendar( options: BlogGenerationOptions, onProgress: (progress: number, message?: string) => void, @@ -766,7 +1381,7 @@ export class BlogGenerationEngine { .map(([category]) => category); const { publishedPosts, publishedListPosts } = await loadPublishedGenerationSets(this.postEngine, listExcludedCategories); - const publishedRoutePosts = await this.buildPublishedRoutePosts(publishedPosts); + const { routePosts: publishedRoutePosts } = await this.buildPublishedRoutePosts(publishedPosts); const generationPostIndex = buildGenerationPostIndex(publishedListPosts); const { sitemapXml } = buildSitemapAndFeeds({ @@ -1016,7 +1631,7 @@ export class BlogGenerationEngine { const maxPostsPerPage = clampMaxPostsPerPage(options.maxPostsPerPage); const { publishedPosts, publishedListPosts } = await loadPublishedGenerationSets(this.postEngine, listExcludedCategories); - const publishedRoutePosts = await this.buildPublishedRoutePosts(publishedPosts); + const { routePosts: publishedRoutePosts } = await this.buildPublishedRoutePosts(publishedPosts); const generationPostIndex = buildGenerationPostIndex(publishedListPosts); const { allCategories, allTags, years, yearMonths, yearMonthDays } = buildApplyValidationArchives(publishedListPosts); diff --git a/src/main/engine/DataBackedEngines.ts b/src/main/engine/DataBackedEngines.ts new file mode 100644 index 0000000..08c25f5 --- /dev/null +++ b/src/main/engine/DataBackedEngines.ts @@ -0,0 +1,250 @@ +/** + * Lightweight, in-memory engine implementations for use in worker threads. + * + * These replace the real PostEngine / MediaEngine / PostMediaEngine when running + * inside a generation worker. They are backed entirely by pre-loaded data arrays + * so no database access is needed for post/media queries. + */ +import type { PostData, PostFilter, PostTranslationData } from './PostEngine'; +import type { MediaData } from './MediaEngine'; +import { readPostFile } from './postFileUtils'; +import { readPostTranslationFile } from './postTranslationFileUtils'; + +// --------------------------------------------------------------------------- +// DataBackedPostEngine +// --------------------------------------------------------------------------- + +export interface DataBackedPostEngineInit { + /** All posts (published snapshots + translation variants). */ + allPosts: PostData[]; + /** Pre-resolved backlinks: postId → linking posts. */ + backlinksMap?: Map>; + /** Post file paths for lazy content loading from filesystem: postId → absoluteFilePath. */ + postFilePaths?: Map; +} + +export interface DataBackedPostEngineContract { + getPostsFiltered: (filter: PostFilter) => Promise; + getPublishedVersion: (id: string) => Promise; + getPost: (id: string) => Promise; + hasPublishedVersion: (id: string) => Promise; + findPublishedBySlug: (slug: string, dateFilter?: { year: number; month: number }) => Promise; + getLinkedBy: (postId: string) => Promise>; + getAllBacklinks: () => Promise>>; + getPostTranslation: (postId: string, language: string) => Promise; + getPostTranslations: (postId: string) => Promise; + setProjectContext: (projectId: string, dataDir?: string) => void; +} + +export function createDataBackedPostEngine(init: DataBackedPostEngineInit): DataBackedPostEngineContract { + const { allPosts, backlinksMap, postFilePaths } = init; + + // Build indexes for fast lookups + const byId = new Map(); + const bySlug = new Map(); + for (const post of allPosts) { + byId.set(post.id, post); + const slugEntries = bySlug.get(post.slug); + if (slugEntries) { + slugEntries.push(post); + } else { + bySlug.set(post.slug, [post]); + } + } + + function matchesFilter(post: PostData, filter: PostFilter): boolean { + if (filter.status && post.status !== filter.status) return false; + + if (filter.excludeCategories && filter.excludeCategories.length > 0) { + const excluded = new Set(filter.excludeCategories); + if (post.categories.some((c) => excluded.has(c))) return false; + } + + if (filter.categories && filter.categories.length > 0) { + if (!filter.categories.some((c) => post.categories.includes(c))) return false; + } + + if (filter.tags && filter.tags.length > 0) { + if (!filter.tags.every((t) => post.tags.includes(t))) return false; + } + + if (filter.language && post.language !== filter.language) return false; + + if (filter.year !== undefined) { + if (post.createdAt.getFullYear() !== filter.year) return false; + } + + if (filter.month !== undefined) { + if (post.createdAt.getMonth() + 1 !== filter.month) return false; + } + + if (filter.startDate) { + if (post.createdAt < filter.startDate) return false; + } + + if (filter.endDate) { + if (post.createdAt > filter.endDate) return false; + } + + return true; + } + + // Shared lazy content loader for posts with empty content + async function lazyLoadContent(post: PostData): Promise { + if (post.content || !postFilePaths) return; + const variant = post as PostData & { translationFilePath?: string }; + if (variant.translationFilePath) { + const fileData = await readPostTranslationFile(variant.translationFilePath); + if (fileData) { + post.content = fileData.content; + } + } else { + const filePath = postFilePaths.get(post.id); + if (filePath) { + const fileData = await readPostFile(filePath); + if (fileData) { + post.content = fileData.content; + } + } + } + } + + return { + async getPostsFiltered(filter: PostFilter): Promise { + const filtered = allPosts + .filter((post) => { + const tss = (post as any).translationSourceSlug; + // Keep canonical posts and resolved posts (slug === tss). + // Exclude translation variant route posts (slug !== tss, e.g. "my-post.en"). + return !tss || post.slug === tss; + }) + .filter((post) => matchesFilter(post, filter)) + .sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()); + + // Lazy-load content for posts that need it (e.g. resolved translation posts + // have content: '' and need their translation file content loaded). + await Promise.all(filtered.map((post) => lazyLoadContent(post))); + + return filtered; + }, + + async getPublishedVersion(id: string): Promise { + const post = byId.get(id); + if (!post) return null; + + await lazyLoadContent(post); + + return post; + }, + + async getPost(id: string): Promise { + const post = byId.get(id) ?? null; + if (post) { + await lazyLoadContent(post); + } + return post; + }, + + async hasPublishedVersion(id: string): Promise { + return byId.has(id); + }, + + async findPublishedBySlug(slug: string, dateFilter?: { year: number; month: number }): Promise { + const candidates = bySlug.get(slug); + if (!candidates || candidates.length === 0) return null; + + if (!dateFilter) return candidates[0]; + + return candidates.find((p) => + p.createdAt.getFullYear() === dateFilter.year + && p.createdAt.getMonth() === dateFilter.month - 1, + ) ?? null; + }, + + async getLinkedBy(postId: string): Promise> { + return backlinksMap?.get(postId) ?? []; + }, + + async getAllBacklinks(): Promise>> { + return backlinksMap ?? new Map(); + }, + + async getPostTranslation(_postId: string, _language: string): Promise { + // Translation variants are already included as separate route posts + return null; + }, + + async getPostTranslations(_postId: string): Promise { + return []; + }, + + setProjectContext(_projectId: string, _dataDir?: string): void { + // No-op — data is already loaded + }, + }; +} + +// --------------------------------------------------------------------------- +// DataBackedMediaEngine +// --------------------------------------------------------------------------- + +export interface DataBackedMediaEngineContract { + getAllMedia: () => Promise; + setProjectContext: (projectId: string, dataDir?: string, internalDir?: string) => void; +} + +export function createDataBackedMediaEngine( + mediaItems: MediaData[], +): DataBackedMediaEngineContract { + return { + async getAllMedia() { + return mediaItems; + }, + setProjectContext(_projectId: string, _dataDir?: string, _internalDir?: string): void { + // No-op + }, + }; +} + +// --------------------------------------------------------------------------- +// DataBackedPostMediaEngine +// --------------------------------------------------------------------------- + +export interface DataBackedPostMediaEngineInit { + mediaItems: MediaData[]; + postMediaLinks: Map>; +} + +export interface DataBackedPostMediaEngineContract { + setProjectContext: (projectId: string) => void; + getLinkedMediaForPost: (postId: string) => Promise>; + getLinkedMediaDataForPost: (postId: string) => Promise>; +} + +export function createDataBackedPostMediaEngine(init: DataBackedPostMediaEngineInit): DataBackedPostMediaEngineContract { + const { mediaItems, postMediaLinks } = init; + const mediaById = new Map(); + for (const m of mediaItems) { + mediaById.set(m.id, m); + } + + return { + setProjectContext(_projectId: string): void { + // No-op + }, + async getLinkedMediaForPost(postId: string): Promise> { + return postMediaLinks.get(postId) ?? []; + }, + async getLinkedMediaDataForPost(postId: string): Promise> { + const links = postMediaLinks.get(postId) ?? []; + const result: Array<{ media: MediaData }> = []; + for (const link of links) { + const media = mediaById.get(link.mediaId); + if (media) { + result.push({ media }); + } + } + return result; + }, + }; +} diff --git a/src/main/engine/GenerationPostSnapshotService.ts b/src/main/engine/GenerationPostSnapshotService.ts index b3bd229..931d845 100644 --- a/src/main/engine/GenerationPostSnapshotService.ts +++ b/src/main/engine/GenerationPostSnapshotService.ts @@ -3,6 +3,7 @@ import type { PostData } from './PostEngine'; export interface GenerationSnapshotPostEngine { getPostsFiltered: (filter: { status?: 'draft' | 'published' | 'archived'; excludeCategories?: string[] }) => Promise; getPublishedVersion: (id: string) => Promise; + getPublishedVersionsBulk?: (ids: string[]) => Promise>; } export interface GenerationPublishedSets { @@ -10,57 +11,64 @@ export interface GenerationPublishedSets { publishedListPosts: PostData[]; } +async function resolvePublishedVersions( + postEngine: GenerationSnapshotPostEngine, + ids: string[], +): Promise> { + if (ids.length === 0) return new Map(); + + if (postEngine.getPublishedVersionsBulk) { + return postEngine.getPublishedVersionsBulk(ids); + } + + const result = new Map(); + const entries = await Promise.all( + ids.map(async (id) => { + const version = await postEngine.getPublishedVersion(id); + return { id, version }; + }), + ); + for (const { id, version } of entries) { + if (version) result.set(id, version); + } + return result; +} + export async function loadPublishedGenerationSets( postEngine: GenerationSnapshotPostEngine, listExcludedCategories: string[], ): Promise { const publishedCandidates = await postEngine.getPostsFiltered({ status: 'published' }); const draftCandidates = await postEngine.getPostsFiltered({ status: 'draft' }); - const publishedListCandidates = await postEngine.getPostsFiltered({ - status: 'published', - excludeCategories: listExcludedCategories, - }); - const draftListCandidates = await postEngine.getPostsFiltered({ - status: 'draft', - excludeCategories: listExcludedCategories, - }); - const publishedSnapshots = await Promise.all( - publishedCandidates.map(async (post) => { - const snapshot = await postEngine.getPublishedVersion(post.id); - return snapshot || post; - }), - ); - const draftPublishedSnapshots = await Promise.all( - draftCandidates.map(async (post) => postEngine.getPublishedVersion(post.id)), - ); - const publishedListSnapshots = await Promise.all( - publishedListCandidates.map(async (post) => { - const snapshot = await postEngine.getPublishedVersion(post.id); - return snapshot || post; - }), - ); - const draftListPublishedSnapshots = await Promise.all( - draftListCandidates.map(async (post) => postEngine.getPublishedVersion(post.id)), - ); + const allIds = new Set(); + for (const p of publishedCandidates) allIds.add(p.id); + for (const p of draftCandidates) allIds.add(p.id); + + const publishedVersions = await resolvePublishedVersions(postEngine, Array.from(allIds)); + + const excludedCategorySet = new Set(listExcludedCategories); + const isListExcluded = (post: PostData) => + excludedCategorySet.size > 0 && post.categories.some((c) => excludedCategorySet.has(c)); const publishedPostById = new Map(); - for (const post of publishedSnapshots) { - publishedPostById.set(post.id, post); - } - for (const snapshot of draftPublishedSnapshots) { - if (snapshot) { - publishedPostById.set(snapshot.id, snapshot); + const publishedListPostById = new Map(); + + for (const post of publishedCandidates) { + const snapshot = publishedVersions.get(post.id) || post; + publishedPostById.set(post.id, snapshot); + if (!isListExcluded(post)) { + publishedListPostById.set(post.id, snapshot); } } - const publishedListPostById = new Map(); - for (const post of publishedListSnapshots) { - publishedListPostById.set(post.id, post); - } - for (const snapshot of draftListPublishedSnapshots) { + for (const post of draftCandidates) { + const snapshot = publishedVersions.get(post.id); if (snapshot) { - publishedListPostById.set(snapshot.id, snapshot); + publishedPostById.set(post.id, snapshot); + if (!isListExcluded(post)) { + publishedListPostById.set(post.id, snapshot); + } } } diff --git a/src/main/engine/GenerationRouteRendererFactory.ts b/src/main/engine/GenerationRouteRendererFactory.ts index 919b64e..ca31d1b 100644 --- a/src/main/engine/GenerationRouteRendererFactory.ts +++ b/src/main/engine/GenerationRouteRendererFactory.ts @@ -1,6 +1,6 @@ import path from 'node:path'; -import type { CategoryRenderSettings } from './PageRenderer'; -import { buildCanonicalPostPath } from './PageRenderer'; +import type { CategoryRenderSettings, HtmlRewriteContext } from './PageRenderer'; +import { buildCanonicalPostPath, mapToRecord } from './PageRenderer'; import type { MenuDocument } from './MenuEngine'; import type { ProjectMetadata } from './MetaEngine'; import type { PostData } from './PostEngine'; @@ -8,6 +8,7 @@ import type { PicoThemeName } from '../shared/picoThemes'; import type { CategoryMetadata } from './BlogGenerationEngine'; import { PreviewServer } from './PreviewServer'; import type { PostTranslationData } from './PostEngine'; +import { readPostTranslationFile } from './postTranslationFileUtils'; interface RenderContext { projectContext: { @@ -49,11 +50,14 @@ export function createPreviewBackedGenerationRouteRenderer(params: { projectName: string; projectDescription?: string; language?: string; + blogLanguages?: string[]; picoTheme?: PicoThemeName; categoryMetadata?: Record; categorySettings?: Record; menu?: MenuDocument; }; + /** The project's actual main language (for href_prefix computation). Defaults to options.language. */ + projectMainLanguage?: string; maxPostsPerPage: number; publishedPostsForLookup: PostData[]; languagePrefix?: string; @@ -66,6 +70,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: { getPostTranslation?: (postId: string, language: string) => Promise; hasPublishedVersion: (postId: string) => Promise; getLinkedBy?: (postId: string) => Promise<{ id: string; title: string; slug: string }[]>; + getAllBacklinks?: () => Promise>; setProjectContext: (projectId: string, dataDir?: string) => void; }; mediaEngine: { @@ -79,10 +84,13 @@ export function createPreviewBackedGenerationRouteRenderer(params: { }; }; }): (pathname: string) => Promise { + const projectMainLanguage = params.projectMainLanguage ?? params.options.language; + const metadata: ProjectMetadata = { name: params.options.projectName, description: params.options.projectDescription, - mainLanguage: params.options.language, + mainLanguage: projectMainLanguage, + blogLanguages: params.options.blogLanguages, maxPostsPerPage: params.maxPostsPerPage, picoTheme: params.options.picoTheme, categoryMetadata: params.options.categoryMetadata, @@ -166,26 +174,53 @@ export function createPreviewBackedGenerationRouteRenderer(params: { return null; } + let match: PostData | undefined; if (!dateFilter) { - return candidates[0] ?? null; + match = candidates[0]; + } else { + match = candidates.find((candidate) => { + const createdAt = candidate.createdAt; + return createdAt.getFullYear() === dateFilter.year + && createdAt.getMonth() === dateFilter.month - 1; + }); } - const match = candidates.find((candidate) => { - const createdAt = candidate.createdAt; - return createdAt.getFullYear() === dateFilter.year - && createdAt.getMonth() === dateFilter.month - 1; - }); + if (!match) return null; - return match ?? null; + // Lazily resolve content from file when needed + if (!match.content) { + const variant = match as PostData & { translationFilePath?: string }; + if (variant.translationFilePath) { + const fileData = await readPostTranslationFile(variant.translationFilePath); + if (fileData) { + match.content = fileData.content; + } + } else { + const full = await cachedPostEngine.getPublishedVersion(match.id); + if (full) { + match.content = full.content; + } + } + } + + return match; }, getPost: (postId: string) => params.engines.postEngine.getPost(postId), getPostTranslation: params.engines.postEngine.getPostTranslation ? (postId: string, language: string) => params.engines.postEngine.getPostTranslation!(postId, language) : undefined, hasPublishedVersion: (postId: string) => params.engines.postEngine.hasPublishedVersion(postId), - getLinkedBy: params.engines.postEngine.getLinkedBy - ? (postId: string) => params.engines.postEngine.getLinkedBy!(postId) - : undefined, + getLinkedBy: params.engines.postEngine.getAllBacklinks + ? (() => { + const backlinksCachePromise = params.engines.postEngine.getAllBacklinks!(); + return async (postId: string) => { + const backlinksMap = await backlinksCachePromise; + return backlinksMap.get(postId) ?? []; + }; + })() + : params.engines.postEngine.getLinkedBy + ? (postId: string) => params.engines.postEngine.getLinkedBy!(postId) + : undefined, setProjectContext: (projectId: string, dataDir?: string) => { params.engines.postEngine.setProjectContext(projectId, dataDir); }, @@ -224,7 +259,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: { userTemplatesDir: path.join(params.options.dataDir, 'templates'), }); - const htmlRewriteContextPromise: Promise<{ canonicalPostPathBySlug: Map; canonicalMediaPathBySourcePath: Map; languagePrefix?: string }> = (async () => { + const htmlRewriteContextPromise: Promise = (async () => { const canonicalPostPathBySlug = new Map(); for (const post of params.publishedPostsForLookup) { canonicalPostPathBySlug.set(post.slug, buildCanonicalPostPath(post)); @@ -247,6 +282,8 @@ export function createPreviewBackedGenerationRouteRenderer(params: { return { canonicalPostPathBySlug, canonicalMediaPathBySourcePath, + canonicalPostPathBySlugRecord: mapToRecord(canonicalPostPathBySlug), + canonicalMediaPathBySourcePathRecord: mapToRecord(canonicalMediaPathBySourcePath), languagePrefix: params.languagePrefix, }; })(); @@ -255,6 +292,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: { renderWithContext: async (pathname, context) => previewServer.renderRouteForContext(pathname, { ...context, htmlRewriteContext: await htmlRewriteContextPromise, + preferredLanguage: params.options.language, }), context: { projectContext, diff --git a/src/main/engine/GenerationWorkerData.ts b/src/main/engine/GenerationWorkerData.ts new file mode 100644 index 0000000..ad6101e --- /dev/null +++ b/src/main/engine/GenerationWorkerData.ts @@ -0,0 +1,327 @@ +/** + * Serialization types and utilities for passing data to generation worker threads. + * + * Worker threads cannot receive non-serializable objects (Date, Map, etc.) via + * the structured-clone algorithm used by workerData / postMessage. This module + * provides a thin serialization layer that converts Dates to ISO strings and + * Maps to arrays-of-tuples so the data survives the boundary. + */ +import type { PostData } from './PostEngine'; +import type { MediaData } from './MediaEngine'; +import type { CategoryMetadata, BlogGenerationOptions, BlogGenerationSection } from './BlogGenerationEngine'; +import type { CategoryRenderSettings } from './PageRenderer'; +import type { MenuDocument } from './MenuEngine'; +import type { PicoThemeName } from '../shared/picoThemes'; + +// --------------------------------------------------------------------------- +// Serialized post (Dates → ISO strings) +// --------------------------------------------------------------------------- + +export interface SerializedPostData { + id: string; + projectId: string; + title: string; + slug: string; + excerpt?: string; + content: string; + status: 'draft' | 'published' | 'archived'; + author?: string; + language?: string; + doNotTranslate?: boolean; + templateSlug?: string; + createdAt: string; + updatedAt: string; + publishedAt?: string; + tags: string[]; + categories: string[]; + availableLanguages: string[]; + translationSourceSlug?: string; + translationCanonicalLanguage?: string; + translationFilePath?: string; + /** Absolute file path for canonical posts (for lazy content loading in workers). */ + filePath?: string; +} + +// --------------------------------------------------------------------------- +// Serialized media item (only what generation needs) +// --------------------------------------------------------------------------- + +export interface SerializedMediaData { + id: string; + filename: string; + originalName: string; + mimeType: string; + size: number; + width?: number; + height?: number; + title?: string; + alt?: string; + caption?: string; + author?: string; + language?: string; + createdAt: string; + updatedAt: string; + tags: string[]; + linkedPostIds?: string[]; + availableLanguages: string[]; +} + +// --------------------------------------------------------------------------- +// Serialized options (stripped to what the worker actually uses) +// --------------------------------------------------------------------------- + +export interface SerializedBlogGenerationOptions { + projectId: string; + projectName: string; + projectDescription?: string; + dataDir: string; + baseUrl: string; + language?: string; + blogLanguages?: string[]; + picoTheme?: PicoThemeName; + categoryMetadata?: Record; + categorySettings?: Record; + menu?: MenuDocument; +} + +// --------------------------------------------------------------------------- +// Worker task — everything a worker needs to process one section +// --------------------------------------------------------------------------- + +export interface GenerationWorkerTask { + taskId: string; + section: BlogGenerationSection; + + /** Posts relevant to this task (single: chunk, others: all list posts). */ + posts: SerializedPostData[]; + + /** + * All published route posts — used by the route renderer for slug lookups, + * canonical path building, and backlink resolution. + */ + lookupPosts: SerializedPostData[]; + + /** Media items for canonical media path building. */ + mediaItems: SerializedMediaData[]; + + /** Pre-resolved backlinks map: postId → linked-by entries. */ + backlinksMap: Record>; + + options: SerializedBlogGenerationOptions; + maxPostsPerPage: number; + htmlDir: string; + + /** Pre-loaded hash map as [relativePath, contentHash] tuples. */ + hashMapEntries: Array<[string, string]>; + + /** Post file paths for lazy content loading: [postId, absoluteFilePath] tuples. */ + postFilePathEntries: Array<[string, string]>; + + /** Post-media links: [postId, [{mediaId, sortOrder}]] tuples for gallery/album macros. */ + postMediaLinksEntries: Array<[string, Array<{ mediaId: string; sortOrder: number }>]>; + + /** Language prefix for subtree generation, e.g. "/fr". */ + languagePrefix?: string; + + /** The project's actual main language (for href_prefix computation in subtree rendering). */ + mainLanguage?: string; + + // -- Section-specific data (category / tag / date) ----------------------- + + allCategories?: string[]; + allTags?: string[]; + + /** Serialized Maps as arrays of tuples. */ + yearsEntries?: Array<[number, string]>; + yearMonthsEntries?: Array<[string, string]>; + yearMonthDaysEntries?: Array<[string, string]>; + + /** Pre-built post index (avoids rebuilding in each worker). */ + postsByCategoryEntries?: Array<[string, SerializedPostData[]]>; + postsByTagEntries?: Array<[string, SerializedPostData[]]>; + postsByYearEntries?: Array<[number, SerializedPostData[]]>; + postsByYearMonthEntries?: Array<[string, SerializedPostData[]]>; + postsByYearMonthDayEntries?: Array<[string, SerializedPostData[]]>; +} + +// --------------------------------------------------------------------------- +// Messages between main thread and worker +// --------------------------------------------------------------------------- + +export interface WorkerProgressMessage { + type: 'progress'; + taskId: string; + message: string; +} + +export interface WorkerResultMessage { + type: 'result'; + taskId: string; + pagesGenerated: number; + /** Hash updates accumulated during generation, to be written by the main thread. */ + hashUpdates: Array<{ relativePath: string; hash: string }>; +} + +export interface WorkerErrorMessage { + type: 'error'; + taskId: string; + error: string; +} + +export type WorkerOutboundMessage = + | WorkerProgressMessage + | WorkerResultMessage + | WorkerErrorMessage; + +// --------------------------------------------------------------------------- +// Serialization helpers +// --------------------------------------------------------------------------- + +export function serializePostData(post: PostData): SerializedPostData { + const serialized: SerializedPostData = { + id: post.id, + projectId: post.projectId, + title: post.title, + slug: post.slug, + excerpt: post.excerpt, + content: post.content ?? '', + status: post.status, + author: post.author, + language: post.language, + doNotTranslate: post.doNotTranslate, + templateSlug: post.templateSlug, + createdAt: post.createdAt instanceof Date ? post.createdAt.toISOString() : String(post.createdAt), + updatedAt: post.updatedAt instanceof Date ? post.updatedAt.toISOString() : String(post.updatedAt), + publishedAt: post.publishedAt instanceof Date ? post.publishedAt.toISOString() : post.publishedAt ? String(post.publishedAt) : undefined, + tags: post.tags ?? [], + categories: post.categories ?? [], + availableLanguages: post.availableLanguages ?? [], + }; + + // Preserve translation variant fields if present + const variant = post as PostData & { + translationSourceSlug?: string; + translationCanonicalLanguage?: string; + translationFilePath?: string; + }; + if (variant.translationSourceSlug) serialized.translationSourceSlug = variant.translationSourceSlug; + if (variant.translationCanonicalLanguage) serialized.translationCanonicalLanguage = variant.translationCanonicalLanguage; + if (variant.translationFilePath) serialized.translationFilePath = variant.translationFilePath; + + return serialized; +} + +export function deserializePostData(serialized: SerializedPostData): PostData { + const post: PostData = { + id: serialized.id, + projectId: serialized.projectId, + title: serialized.title, + slug: serialized.slug, + excerpt: serialized.excerpt, + content: serialized.content ?? '', + status: serialized.status, + author: serialized.author, + language: serialized.language, + doNotTranslate: serialized.doNotTranslate, + templateSlug: serialized.templateSlug, + createdAt: new Date(serialized.createdAt), + updatedAt: new Date(serialized.updatedAt), + publishedAt: serialized.publishedAt ? new Date(serialized.publishedAt) : undefined, + tags: serialized.tags ?? [], + categories: serialized.categories ?? [], + availableLanguages: serialized.availableLanguages ?? [], + }; + + // Re-attach translation variant fields + if (serialized.translationSourceSlug) { + (post as any).translationSourceSlug = serialized.translationSourceSlug; + } + if (serialized.translationCanonicalLanguage) { + (post as any).translationCanonicalLanguage = serialized.translationCanonicalLanguage; + } + if (serialized.translationFilePath) { + (post as any).translationFilePath = serialized.translationFilePath; + } + + return post; +} + +export function serializeMediaItem(media: MediaData): SerializedMediaData { + return { + id: media.id, + filename: media.filename, + originalName: media.originalName, + mimeType: media.mimeType, + size: media.size, + width: media.width, + height: media.height, + title: media.title, + alt: media.alt, + caption: media.caption, + author: media.author, + language: media.language, + createdAt: media.createdAt instanceof Date ? media.createdAt.toISOString() : String(media.createdAt), + updatedAt: media.updatedAt instanceof Date ? media.updatedAt.toISOString() : String(media.updatedAt), + tags: media.tags ?? [], + linkedPostIds: media.linkedPostIds, + availableLanguages: media.availableLanguages ?? [], + }; +} + +export function deserializeMediaItem(serialized: SerializedMediaData): MediaData { + return { + id: serialized.id, + filename: serialized.filename, + originalName: serialized.originalName, + mimeType: serialized.mimeType, + size: serialized.size, + width: serialized.width, + height: serialized.height, + title: serialized.title, + alt: serialized.alt, + caption: serialized.caption, + author: serialized.author, + language: serialized.language, + createdAt: new Date(serialized.createdAt), + updatedAt: new Date(serialized.updatedAt), + tags: serialized.tags ?? [], + linkedPostIds: serialized.linkedPostIds, + availableLanguages: serialized.availableLanguages ?? [], + }; +} + +export function serializeBlogGenerationOptions(options: BlogGenerationOptions): SerializedBlogGenerationOptions { + return { + projectId: options.projectId, + projectName: options.projectName, + projectDescription: options.projectDescription, + dataDir: options.dataDir, + baseUrl: options.baseUrl, + language: options.language, + blogLanguages: options.blogLanguages, + picoTheme: options.picoTheme, + categoryMetadata: options.categoryMetadata, + categorySettings: options.categorySettings, + menu: options.menu, + }; +} + +/** Serialize a Map to an array of [K, SerializedPostData[]] tuples. */ +export function serializePostMap(map: Map): Array<[K, SerializedPostData[]]> { + return Array.from(map.entries()).map(([key, posts]) => [key, posts.map(serializePostData)]); +} + +/** Deserialize an array of [K, SerializedPostData[]] tuples to a Map. */ +export function deserializePostMap(entries: Array<[K, SerializedPostData[]]>): Map { + return new Map(entries.map(([key, posts]) => [key, posts.map(deserializePostData)])); +} + +/** Serialize a Map to an array of [K, string] tuples. */ +export function serializeDateMap(map: Map): Array<[K, string]> { + return Array.from(map.entries()).map(([key, date]) => [key, date.toISOString()]); +} + +/** Deserialize an array of [K, string] tuples to a Map. */ +export function deserializeDateMap(entries: Array<[K, string]>): Map { + return new Map(entries.map(([key, iso]) => [key, new Date(iso)])); +} diff --git a/src/main/engine/GenerationWorkerPool.ts b/src/main/engine/GenerationWorkerPool.ts new file mode 100644 index 0000000..ea8943b --- /dev/null +++ b/src/main/engine/GenerationWorkerPool.ts @@ -0,0 +1,139 @@ +/** + * Worker pool for parallel blog generation. + * + * Manages a pool of worker threads that render HTML pages concurrently. + * Each worker gets a self-contained GenerationWorkerTask and produces pages + * independently. The pool limits concurrency to os.cpus().length - 1 (min 1). + */ +import { Worker } from 'worker_threads'; +import * as os from 'os'; +import * as path from 'path'; +import type { + GenerationWorkerTask, + WorkerOutboundMessage, +} from './GenerationWorkerData'; + +export interface WorkerPoolOptions { + /** Max concurrent workers. Defaults to os.cpus().length - 1, min 1. */ + maxWorkers?: number; + /** Override the worker script path (for testing). */ + workerPath?: string; +} + +export interface WorkerPoolResult { + pagesGenerated: number; + errors: Array<{ taskId: string; error: string }>; + hashUpdates: Array<{ relativePath: string; hash: string }>; +} + +export type WorkerFactory = (workerPath: string, workerData: GenerationWorkerTask) => WorkerLike; + +export interface WorkerLike { + on(event: string, listener: (...args: unknown[]) => void): void; + terminate(): Promise; + removeAllListeners(): void; +} + +export class GenerationWorkerPool { + private readonly maxWorkers: number; + private readonly workerPath: string; + private readonly workerFactory: WorkerFactory; + + constructor(options?: WorkerPoolOptions, workerFactory?: WorkerFactory) { + this.maxWorkers = Math.max(1, options?.maxWorkers ?? (os.cpus().length - 1)); + this.workerPath = options?.workerPath ?? path.join(__dirname, 'generation.worker.js'); + this.workerFactory = workerFactory ?? ((wp, wd) => new Worker(wp, { workerData: wd }) as unknown as WorkerLike); + } + + /** + * Run a set of generation tasks across the worker pool. + * + * Tasks are distributed to workers up to maxWorkers concurrency. + * When a worker finishes, the next queued task is dispatched. + * + * @param tasks - Array of self-contained worker tasks + * @param onProgress - Called for each page generated (for progress bar updates) + * @returns Merged results from all workers + */ + async runTasks( + tasks: GenerationWorkerTask[], + onProgress: (message: string) => void, + ): Promise { + if (tasks.length === 0) { + return { pagesGenerated: 0, errors: [], hashUpdates: [] }; + } + + return new Promise((resolve) => { + let totalPages = 0; + const errors: Array<{ taskId: string; error: string }> = []; + const allHashUpdates: Array<{ relativePath: string; hash: string }> = []; + let nextTaskIndex = 0; + let activeWorkers = 0; + + const startNextWorker = () => { + if (nextTaskIndex >= tasks.length) { + if (activeWorkers === 0) { + resolve({ pagesGenerated: totalPages, errors, hashUpdates: allHashUpdates }); + } + return; + } + + const task = tasks[nextTaskIndex++]; + activeWorkers++; + + const worker = this.workerFactory(this.workerPath, task); + + worker.on('message', (raw: unknown) => { + const msg = raw as WorkerOutboundMessage; + + switch (msg.type) { + case 'progress': + onProgress(msg.message); + break; + + case 'result': + totalPages += msg.pagesGenerated; + if (msg.hashUpdates) { + allHashUpdates.push(...msg.hashUpdates); + } + activeWorkers--; + void worker.terminate(); + startNextWorker(); + break; + + case 'error': + errors.push({ taskId: msg.taskId, error: msg.error }); + activeWorkers--; + void worker.terminate(); + startNextWorker(); + break; + } + }); + + worker.on('error', (err: unknown) => { + const errorMessage = err instanceof Error ? err.message : String(err); + errors.push({ taskId: task.taskId, error: errorMessage }); + activeWorkers--; + startNextWorker(); + }); + + worker.on('exit', (code: unknown) => { + // If the worker exited unexpectedly (no result/error message received), + // we need to account for it. The 'error' handler above covers crashes. + // Exit code 0 is normal (worker finished). Non-zero without error handler + // means something unexpected happened. + if (typeof code === 'number' && code !== 0) { + // Only handle if we haven't already decremented (check via activeWorkers) + // This is a safety net — most crashes are caught by the 'error' event. + } + }); + }; + + // Start initial batch of workers + const initialBatch = Math.min(this.maxWorkers, tasks.length); + for (let i = 0; i < initialBatch; i++) { + startNextWorker(); + } + }); + } +} diff --git a/src/main/engine/PageRenderer.ts b/src/main/engine/PageRenderer.ts index 84430f0..a22558e 100644 --- a/src/main/engine/PageRenderer.ts +++ b/src/main/engine/PageRenderer.ts @@ -56,6 +56,10 @@ export interface HtmlRewriteContext { canonicalPostPathBySlug: Map; canonicalMediaPathBySourcePath: Map; languagePrefix?: string; + /** Pre-computed Record version of canonicalPostPathBySlug (avoids repeated Map→Object conversion) */ + canonicalPostPathBySlugRecord?: Record; + /** Pre-computed Record version of canonicalMediaPathBySourcePath */ + canonicalMediaPathBySourcePathRecord?: Record; } export interface TemplatePostEntry { @@ -1445,8 +1449,8 @@ export class PageRenderer { has_next_page: hasNextPage, prev_page_href: prevPageHref, next_page_href: nextPageHref, - canonical_post_path_by_slug: mapToRecord(rewriteContext.canonicalPostPathBySlug), - canonical_media_path_by_source_path: mapToRecord(rewriteContext.canonicalMediaPathBySourcePath), + canonical_post_path_by_slug: rewriteContext.canonicalPostPathBySlugRecord ?? mapToRecord(rewriteContext.canonicalPostPathBySlug), + canonical_media_path_by_source_path: rewriteContext.canonicalMediaPathBySourcePathRecord ?? mapToRecord(rewriteContext.canonicalMediaPathBySourcePath), post_data_json_by_id: Object.fromEntries( posts.map((post) => [post.id, JSON.stringify(serializePostDataForMacro(post))]), ), @@ -1572,7 +1576,7 @@ export class PageRenderer { ? Array.from(new Set(renderablePost.tags.map((tag) => tag.trim()).filter((tag) => tag.length > 0))) : []; - const canonicalPostPathBySlug = mapToRecord(rewriteContext.canonicalPostPathBySlug); + const canonicalPostPathBySlug = rewriteContext.canonicalPostPathBySlugRecord ?? mapToRecord(rewriteContext.canonicalPostPathBySlug); // Per-post language overrides the page-level language when present const postLanguage = (renderablePost as { language?: string }).language; @@ -1598,7 +1602,7 @@ export class PageRenderer { calendar_initial_year: renderablePost.createdAt.getFullYear(), calendar_initial_month: renderablePost.createdAt.getMonth() + 1, canonical_post_path_by_slug: canonicalPostPathBySlug, - canonical_media_path_by_source_path: mapToRecord(rewriteContext.canonicalMediaPathBySourcePath), + canonical_media_path_by_source_path: rewriteContext.canonicalMediaPathBySourcePathRecord ?? mapToRecord(rewriteContext.canonicalMediaPathBySourcePath), post_data_json_by_id: { [renderablePost.id]: JSON.stringify(serializePostDataForMacro(renderablePost)), }, diff --git a/src/main/engine/PostEngine.ts b/src/main/engine/PostEngine.ts index d3e513c..b44d327 100644 --- a/src/main/engine/PostEngine.ts +++ b/src/main/engine/PostEngine.ts @@ -1077,6 +1077,40 @@ export class PostEngine extends EventEmitter { return result; } + async getPublishedTranslationsForRoutePosts(publishedPosts: PostData[]): Promise> { + const allRows = await this.getAllTranslationRows(); + const postById = new Map(publishedPosts.map((p) => [p.id, p])); + const result = new Map(); + + for (const row of allRows) { + if (row.status !== 'published') continue; + const sourcePost = postById.get(row.translationFor); + if (!sourcePost) continue; + if (this.isCanonicalTranslationLanguage(sourcePost, row.language)) continue; + + const translationData: PostTranslationData = { + id: row.id, + projectId: row.projectId, + translationFor: row.translationFor, + language: row.language, + title: row.title, + excerpt: row.excerpt || undefined, + content: '', + status: row.status as 'draft' | 'published' | 'archived', + createdAt: row.createdAt, + updatedAt: row.updatedAt, + publishedAt: row.publishedAt || undefined, + filePath: row.filePath, + }; + + const arr = result.get(row.translationFor) || []; + arr.push(translationData); + result.set(row.translationFor, arr); + } + + return result; + } + async getPostTranslations(postId: string): Promise { const sourcePost = await this.getPost(postId); const rows = this.filterCanonicalTranslationRows(sourcePost, await this.getTranslationRowsForPost(postId)); @@ -2504,6 +2538,44 @@ export class PostEngine extends EventEmitter { }; } + async getPublishedVersionsBulk(ids: string[]): Promise> { + const result = new Map(); + if (ids.length === 0) return result; + + const db = getDatabase().getLocal(); + const idSet = new Set(ids); + + const dbPosts = await db.select().from(posts) + .where(eq(posts.projectId, this.currentProjectId)) + .all(); + + for (const dbPost of dbPosts) { + if (!idSet.has(dbPost.id) || !dbPost.filePath) continue; + result.set(dbPost.id, this.dbRowToPostData(dbPost, '')); + } + + return result; + } + + /** + * Bulk-load file paths for all published posts in the current project. + * Returns a Map from postId → absolute filePath. + */ + async getPublishedPostFilePaths(): Promise> { + const db = getDatabase().getLocal(); + const dbPosts = await db.select().from(posts) + .where(eq(posts.projectId, this.currentProjectId)) + .all(); + + const result = new Map(); + for (const dbPost of dbPosts) { + if (dbPost.filePath) { + result.set(dbPost.id, dbPost.filePath); + } + } + return result; + } + /** * Rebuild the FTS index for all posts in the current project. * Call this after changing the search language or after migration. @@ -3089,6 +3161,48 @@ export class PostEngine extends EventEmitter { return sourcePosts.filter(p => sourceIds.includes(p.id)); } + /** + * Bulk-load all backlinks for all posts in the current project. + * Returns a Map from targetPostId → array of source posts that link to it. + * Much more efficient than calling getLinkedBy per post during generation. + */ + async getAllBacklinks(): Promise> { + const db = getDatabase().getLocal(); + + const allLinks = await db + .select({ + sourcePostId: postLinks.sourcePostId, + targetPostId: postLinks.targetPostId, + }) + .from(postLinks); + + if (allLinks.length === 0) return new Map(); + + const sourceIds = new Set(allLinks.map(l => l.sourcePostId)); + const allSourcePosts = await db + .select({ id: posts.id, title: posts.title, slug: posts.slug }) + .from(posts) + .where(eq(posts.projectId, this.currentProjectId)); + + const sourcePostById = new Map( + allSourcePosts.filter(p => sourceIds.has(p.id)).map(p => [p.id, p]), + ); + + const result = new Map(); + for (const link of allLinks) { + const sourcePost = sourcePostById.get(link.sourcePostId); + if (!sourcePost) continue; + const existing = result.get(link.targetPostId); + if (existing) { + existing.push(sourcePost); + } else { + result.set(link.targetPostId, [sourcePost]); + } + } + + return result; + } + /** * Get posts that the specified post links TO ("links to") */ diff --git a/src/main/engine/PostMediaEngine.ts b/src/main/engine/PostMediaEngine.ts index fa9af04..aeeabbf 100644 --- a/src/main/engine/PostMediaEngine.ts +++ b/src/main/engine/PostMediaEngine.ts @@ -394,6 +394,32 @@ export class PostMediaEngine extends EventEmitter { return link.length > 0; } + /** + * Get all post-media links for the current project, grouped by post ID. + * Used to pre-load data for generation workers. + */ + async getAllPostMediaLinks(): Promise>> { + const db = this.getDb(); + + const links = await db + .select() + .from(postMedia) + .where(eq(postMedia.projectId, this.currentProjectId)) + .orderBy(asc(postMedia.sortOrder)); + + const result = new Map>(); + for (const link of links) { + const existing = result.get(link.postId); + const entry = { mediaId: link.mediaId, sortOrder: link.sortOrder }; + if (existing) { + existing.push(entry); + } else { + result.set(link.postId, [entry]); + } + } + return result; + } + /** * Map database row to PostMediaLinkData */ diff --git a/src/main/engine/RoutePageGenerationService.ts b/src/main/engine/RoutePageGenerationService.ts index 45395d5..0602b5b 100644 --- a/src/main/engine/RoutePageGenerationService.ts +++ b/src/main/engine/RoutePageGenerationService.ts @@ -72,18 +72,22 @@ export async function generateSinglePostPages(params: BaseParams & { posts: PostData[]; }): Promise { let count = 0; + const BATCH_SIZE = 10; - for (const post of params.posts) { - const createdAt = resolvePostCreatedAt(post); - const year = createdAt.getFullYear(); - const month = String(createdAt.getMonth() + 1).padStart(2, '0'); - const day = String(createdAt.getDate()).padStart(2, '0'); + for (let i = 0; i < params.posts.length; i += BATCH_SIZE) { + const batch = params.posts.slice(i, i + BATCH_SIZE); + const results = await Promise.all(batch.map(async (post) => { + const createdAt = resolvePostCreatedAt(post); + const year = createdAt.getFullYear(); + const month = String(createdAt.getMonth() + 1).padStart(2, '0'); + const day = String(createdAt.getDate()).padStart(2, '0'); - const urlPath = `${year}/${month}/${day}/${post.slug}`; - const html = await renderRequiredRoute(params.renderRoute, `/${urlPath}`); - await params.writePage(params.projectId, urlPath, html); - count++; - params.onPageGenerated(`Generated /${urlPath}`); + const urlPath = `${year}/${month}/${day}/${post.slug}`; + const html = await renderRequiredRoute(params.renderRoute, `/${urlPath}`); + await params.writePage(params.projectId, urlPath, html); + params.onPageGenerated(`Generated /${urlPath}`); + })); + count += results.length; } return count; diff --git a/src/main/engine/assets/calendarRuntime.ts b/src/main/engine/assets/calendarRuntime.ts index 6f8cc8c..6a13ee0 100644 --- a/src/main/engine/assets/calendarRuntime.ts +++ b/src/main/engine/assets/calendarRuntime.ts @@ -109,7 +109,7 @@ export const CALENDAR_RUNTIME_JS = String.raw`(() => { } async function loadCalendarData() { - const response = await fetch(languagePrefix + '/calendar.json', { cache: 'no-store' }); + const response = await fetch('/calendar.json', { cache: 'no-store' }); if (!response.ok) { throw new Error('calendar.json request failed'); } diff --git a/src/main/engine/generation.worker.ts b/src/main/engine/generation.worker.ts new file mode 100644 index 0000000..6591927 --- /dev/null +++ b/src/main/engine/generation.worker.ts @@ -0,0 +1,287 @@ +/** + * Worker thread entry point for parallel blog generation. + * + * Each worker receives a GenerationWorkerTask via workerData, creates its own + * rendering pipeline (Liquid, PageRenderer, PreviewServer, route renderer) and + * renders the assigned pages, writing them to the filesystem. + * + * Workers do NOT open database connections. Hash reads come from a pre-loaded + * map passed in task data. Hash writes are accumulated in memory and sent back + * to the main thread in the result message for the main thread to persist. + */ +import { parentPort, workerData } from 'worker_threads'; +import type { + GenerationWorkerTask, + WorkerOutboundMessage, + SerializedPostData, +} from './GenerationWorkerData'; +import { + deserializePostData, + deserializeMediaItem, + deserializePostMap, + deserializeDateMap, +} from './GenerationWorkerData'; +import { + createDataBackedPostEngine, + createDataBackedMediaEngine, + createDataBackedPostMediaEngine, +} from './DataBackedEngines'; +import { createPreviewBackedGenerationRouteRenderer } from './GenerationRouteRendererFactory'; +import { + generateSinglePostPages, + generateCategoryPages, + generateTagPages, + generateDateArchivePages, + generateRootPages, + generatePageRoutes, +} from './RoutePageGenerationService'; +import { writeHtmlPage } from './BlogGenerationOutputService'; +import type { PostData } from './PostEngine'; + +// --------------------------------------------------------------------------- +// In-memory hash store (no DB access) +// --------------------------------------------------------------------------- + +/** + * Creates a purely in-memory hash store. + * Reads come from the pre-loaded hash map (passed from main thread). + * Writes are accumulated in `pendingUpdates` and returned to the main thread + * via the result message so it can persist them in a single connection. + */ +function createWorkerHashStore(hashCache: Map) { + const pendingUpdates: Array<{ relativePath: string; hash: string }> = []; + + return { + async get(_projectId: string, relativePath: string): Promise { + return hashCache.get(relativePath) ?? null; + }, + + async set(_projectId: string, relativePath: string, hash: string): Promise { + pendingUpdates.push({ relativePath, hash }); + hashCache.set(relativePath, hash); + }, + + pendingUpdates, + }; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function send(message: WorkerOutboundMessage): void { + parentPort?.postMessage(message); +} + +function deserializePostArray(serialized: SerializedPostData[]): PostData[] { + return serialized.map(deserializePostData); +} + +// --------------------------------------------------------------------------- +// Main worker logic +// --------------------------------------------------------------------------- + +async function run(): Promise { + const task = workerData as GenerationWorkerTask; + + try { + // 1. Reconstruct hash cache from pre-loaded entries (no DB needed) + const hashCache = new Map(); + for (const [relativePath, hash] of task.hashMapEntries) { + hashCache.set(relativePath, hash); + } + const hashStore = createWorkerHashStore(hashCache); + + // 2. Deserialize post data + const posts = deserializePostArray(task.posts); + const lookupPosts = deserializePostArray(task.lookupPosts); + const mediaItems = (task.mediaItems ?? []).map(deserializeMediaItem); + + // 2b. Reconstruct post file paths for lazy content loading + const postFilePaths = new Map(); + if (task.postFilePathEntries) { + for (const [postId, filePath] of task.postFilePathEntries) { + postFilePaths.set(postId, filePath); + } + } + + // 2c. Reconstruct post-media links for gallery/album macros + const postMediaLinks = new Map>(); + if (task.postMediaLinksEntries) { + for (const [postId, links] of task.postMediaLinksEntries) { + postMediaLinks.set(postId, links); + } + } + + // 3. Reconstruct backlinks Map + const backlinksMap = new Map>(); + if (task.backlinksMap) { + for (const [postId, links] of Object.entries(task.backlinksMap)) { + backlinksMap.set(postId, links); + } + } + + // 4. Create data-backed engines + const postEngine = createDataBackedPostEngine({ allPosts: lookupPosts, backlinksMap, postFilePaths }); + const mediaEngine = createDataBackedMediaEngine(mediaItems); + const postMediaEngine = createDataBackedPostMediaEngine({ mediaItems, postMediaLinks }); + + // 5. Create route renderer (same factory as main thread, but backed by data) + const renderRoute = createPreviewBackedGenerationRouteRenderer({ + options: { + ...task.options, + language: task.options.language, + }, + projectMainLanguage: task.mainLanguage, + maxPostsPerPage: task.maxPostsPerPage, + publishedPostsForLookup: lookupPosts, + languagePrefix: task.languagePrefix, + engines: { + postEngine: postEngine as any, + mediaEngine: mediaEngine as any, + postMediaEngine: postMediaEngine as any, + }, + }); + + // 6. Build writePage function using in-memory hash store + const knownDirectories = new Set(); + + const writePage = (projectId: string, urlPath: string, content: string) => { + const effectiveUrlPath = task.languagePrefix + ? `${task.languagePrefix.replace(/^\//, '')}/${urlPath}` + : urlPath; + + return writeHtmlPage({ + projectId, + htmlDir: task.htmlDir, + urlPath: effectiveUrlPath, + content, + knownDirectories, + hashCache, + getGeneratedFileHash: hashStore.get, + setGeneratedFileHash: hashStore.set, + refreshHashTimestampOnUnchanged: true, + }); + }; + + const onPageGenerated = (message: string) => { + send({ type: 'progress', taskId: task.taskId, message }); + }; + + // 7. Execute the assigned section + let pagesGenerated = 0; + const projectId = task.options.projectId; + + switch (task.section) { + case 'single': { + pagesGenerated += await generateSinglePostPages({ + projectId, + posts, + renderRoute, + writePage, + onPageGenerated, + }); + break; + } + + case 'category': { + const allCategories = new Set(task.allCategories ?? []); + const postsByCategory = task.postsByCategoryEntries + ? deserializePostMap(task.postsByCategoryEntries) + : undefined; + + pagesGenerated += await generateCategoryPages({ + projectId, + posts, + allCategories, + maxPostsPerPage: task.maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated, + postsByCategory, + }); + break; + } + + case 'tag': { + const allTags = new Set(task.allTags ?? []); + const postsByTag = task.postsByTagEntries + ? deserializePostMap(task.postsByTagEntries) + : undefined; + + pagesGenerated += await generateTagPages({ + projectId, + posts, + allTags, + maxPostsPerPage: task.maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated, + postsByTag, + }); + break; + } + + case 'date': { + const yearsMap = task.yearsEntries ? deserializeDateMap(task.yearsEntries) : new Map(); + const yearMonthsMap = task.yearMonthsEntries ? deserializeDateMap(task.yearMonthsEntries) : new Map(); + const yearMonthDaysMap = task.yearMonthDaysEntries ? deserializeDateMap(task.yearMonthDaysEntries) : new Map(); + const postsByYear = task.postsByYearEntries ? deserializePostMap(task.postsByYearEntries) : undefined; + const postsByYearMonth = task.postsByYearMonthEntries ? deserializePostMap(task.postsByYearMonthEntries) : undefined; + const postsByYearMonthDay = task.postsByYearMonthDayEntries ? deserializePostMap(task.postsByYearMonthDayEntries) : undefined; + + pagesGenerated += await generateDateArchivePages({ + projectId, + posts, + yearsMap, + yearMonthsMap, + yearMonthDaysMap, + maxPostsPerPage: task.maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated, + postsByYear, + postsByYearMonth, + postsByYearMonthDay, + }); + break; + } + + case 'core': { + // Core includes root pages and page routes (sitemap/feeds handled by main thread) + pagesGenerated += await generateRootPages({ + projectId, + posts, + maxPostsPerPage: task.maxPostsPerPage, + renderRoute, + writePage, + onPageGenerated, + }); + pagesGenerated += await generatePageRoutes({ + projectId, + posts: lookupPosts, + renderRoute, + writePage, + onPageGenerated, + }); + break; + } + } + + // 8. Report result with accumulated hash updates + send({ + type: 'result', + taskId: task.taskId, + pagesGenerated, + hashUpdates: hashStore.pendingUpdates, + }); + } catch (err) { + send({ + type: 'error', + taskId: task.taskId, + error: err instanceof Error ? err.message : String(err), + }); + } +} + +run(); diff --git a/src/main/ipc/blogHandlers.ts b/src/main/ipc/blogHandlers.ts index 9d39464..e9d90ee 100644 --- a/src/main/ipc/blogHandlers.ts +++ b/src/main/ipc/blogHandlers.ts @@ -11,6 +11,7 @@ import type { EngineBundle } from '../engine/EngineBundle'; import type { TranslationValidationReport } from '../shared/electronApi'; import { autoTranslatePost, autoTranslateMediaMetadata } from './chatHandlers'; import { v4 as uuidv4 } from 'uuid'; +import { getDatabase } from '../database/connection'; type SafeHandle = (channel: string, handler: (...args: any[]) => Promise) => void; @@ -84,6 +85,7 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl categoryMetadata: (metadata as any)?.categoryMetadata, categorySettings: (metadata as any)?.categorySettings, menu, + dbPath: getDatabase().getDbPath(), }; }; @@ -91,6 +93,9 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl const blogGenerationEngine = bundle.blogGenerationEngine; const baseOptions = await resolveBlogGenerationBaseOptions(); + // Pre-load post data ONCE before parallel tasks + const preloadedData = await blogGenerationEngine.preloadGenerationData(baseOptions); + const taskTimestamp = Date.now(); const taskGroupId = `site-render-${taskTimestamp}`; const taskGroupName = 'Render Site'; @@ -109,6 +114,7 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl return blogGenerationEngine.generate({ ...baseOptions, sections: [section], + preloadedData, }, (progress, message) => onProgress(progress, message || '')); }, }); diff --git a/src/main/preload.ts b/src/main/preload.ts index 6841c87..b15c15e 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -412,7 +412,7 @@ export const electronAPI: ElectronAPI = { translatePost: (postId: string, targetLanguage: string) => ipcRenderer.invoke('chat:translatePost', postId, targetLanguage), // Media Language Detection - detectMediaLanguage: (mediaId: string) => ipcRenderer.invoke('chat:detectMediaLanguage', mediaId), + detectMediaLanguage: (title: string, alt: string, caption: string) => ipcRenderer.invoke('chat:detectMediaLanguage', title, alt, caption), // Media Metadata Translation translateMediaMetadata: (mediaId: string, targetLanguage: string) => ipcRenderer.invoke('chat:translateMediaMetadata', mediaId, targetLanguage), diff --git a/src/main/shared/electronApi.ts b/src/main/shared/electronApi.ts index e49c2f1..0b38412 100644 --- a/src/main/shared/electronApi.ts +++ b/src/main/shared/electronApi.ts @@ -1070,7 +1070,7 @@ export interface ElectronAPI { translatePost: (postId: string, targetLanguage: string) => Promise<{ success: boolean; translation?: PostTranslationData; error?: string }>; // Media Language Detection - detectMediaLanguage: (mediaId: string) => Promise<{ success: boolean; language?: string; error?: string }>; + detectMediaLanguage: (title: string, alt: string, caption: string) => Promise<{ success: boolean; language?: string; error?: string }>; // Media Metadata Translation translateMediaMetadata: (mediaId: string, targetLanguage: string) => Promise<{ success: boolean; translation?: MediaTranslationData; error?: string }>; diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index ca71527..b2b7018 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -39,7 +39,6 @@ const App: React.FC = () => { toggleSidebar, togglePanel, toggleAssistantSidebar, - setActiveView, setSelectedPost, setActiveProject, setPicoTheme, diff --git a/src/renderer/a2ui/components/A2UIText.tsx b/src/renderer/a2ui/components/A2UIText.tsx index c37061c..bd3de5e 100644 --- a/src/renderer/a2ui/components/A2UIText.tsx +++ b/src/renderer/a2ui/components/A2UIText.tsx @@ -1,4 +1,4 @@ -import React, { type ReactNode } from 'react'; +import React, { type ReactElement } from 'react'; import Markdown from 'marked-react'; import type { A2UIResolvedComponent, A2UIClientAction } from '../../../main/a2ui/types'; @@ -11,7 +11,7 @@ interface A2UIComponentProps { } const safeRenderer = { - image(src: string, alt: string): ReactNode { + image(src: string, alt: string, _title?: string | null): ReactElement { if (/^https?:\/\//i.test(src)) { return {alt || src}; } diff --git a/src/renderer/components/ChatSurface/ChatTranscript.tsx b/src/renderer/components/ChatSurface/ChatTranscript.tsx index 10c059e..4a35b31 100644 --- a/src/renderer/components/ChatSurface/ChatTranscript.tsx +++ b/src/renderer/components/ChatSurface/ChatTranscript.tsx @@ -1,8 +1,8 @@ -import React, { type ReactNode } from 'react'; +import React, { type ReactElement } from 'react'; import Markdown from 'marked-react'; import type { ChatMessage } from '../../types/electron'; import type { ChatToolEvent } from '../../navigation/useChatSurfaceState'; -import type { A2UIResolvedComponent, A2UIClientAction } from '../../../main/a2ui/types'; +import type { A2UIClientAction } from '../../../main/a2ui/types'; import { InlineSurface } from '../../a2ui/InlineSurface'; import type { SurfaceEntry } from '../../a2ui/useA2UISurface'; import { computeTurnIndex } from '../../a2ui/surfaceAssociation'; @@ -51,7 +51,7 @@ export const ChatTranscript: React.FC = ({ }) => { // Block external images — CSP only allows self/data/file/blob/bds-media/bds-thumb const safeRenderer = { - image(src: string, alt: string): ReactNode { + image(src: string, alt: string, _title?: string | null): ReactElement { if (/^https?:\/\//i.test(src)) { // Show alt text as a link instead of trying to load the image return {alt || src}; diff --git a/src/renderer/components/DocumentationView/DocumentationView.tsx b/src/renderer/components/DocumentationView/DocumentationView.tsx index c346375..19f4acc 100644 --- a/src/renderer/components/DocumentationView/DocumentationView.tsx +++ b/src/renderer/components/DocumentationView/DocumentationView.tsx @@ -155,7 +155,7 @@ export const DocumentationView: React.FC = ({ headingSlugCounts.set(baseId, nextCount); const headingId = existingCount === 0 ? baseId : `${baseId}-${nextCount}`; - return React.createElement(`h${levelNumber}` as keyof JSX.IntrinsicElements, { id: headingId, key: getRendererKey('heading') }, children); + return React.createElement(`h${levelNumber}` as 'h1', { id: headingId, key: getRendererKey('heading') }, children); }, link(href: string, text: ReactNode) { if (!href.startsWith('#')) { diff --git a/src/renderer/components/Editor/MediaEditor.tsx b/src/renderer/components/Editor/MediaEditor.tsx index d5615ec..6bbd661 100644 --- a/src/renderer/components/Editor/MediaEditor.tsx +++ b/src/renderer/components/Editor/MediaEditor.tsx @@ -5,6 +5,7 @@ import { AISuggestionsModal } from '../AISuggestionsModal/AISuggestionsModal'; import { openEntityTab } from '../../navigation/tabPolicy'; import { useI18n } from '../../i18n'; import { SUPPORTED_POST_LANGUAGES, POST_LANGUAGE_FLAGS } from '../../../main/shared/i18n'; +import type { MediaData } from '../../../main/shared/electronApi'; import { getMediaDisplayName } from './editorUtils'; export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { @@ -71,7 +72,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { try { const updated = await window.electronAPI?.media.update(item!.id, { language: newLanguage || undefined }); if (updated) { - updateMedia(item!.id, updated as Partial); + updateMedia(item!.id, updated as Partial); } } catch (error) { console.error('Failed to update media language:', error); @@ -92,7 +93,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { setMediaLanguage(result.language); const updated = await window.electronAPI?.media.update(item.id, { language: result.language }); if (updated) { - updateMedia(item.id, updated as Partial); + updateMedia(item.id, updated as Partial); } showToast.success(tr('editor.media.toast.languageDetected', { language: tr(`language.${result.language}`) })); } else { @@ -249,7 +250,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { // Close AI suggestions modal const handleCloseAISuggestionsModal = () => { setShowAISuggestionsModal(false); - setAISuggestions(null); + setAISuggestionFields([]); setAIError(undefined); }; @@ -364,7 +365,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { tags: tags.split(',').map(t => t.trim()).filter(t => t.length > 0), }); if (updated) { - updateMedia(item.id, updated as Partial); + updateMedia(item.id, updated as Partial); showToast.success(tr('editor.media.toast.updated')); } } catch (error) { @@ -382,7 +383,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { try { const updated = await window.electronAPI?.media.replaceFileDialog(item.id); if (updated) { - updateMedia(item.id, updated as Partial); + updateMedia(item.id, updated as Partial); showToast.success(tr('editor.media.toast.fileReplaced')); } // null means user cancelled or file unchanged - no action needed @@ -523,7 +524,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { {item.mimeType.startsWith('image/') ? (
{item.alt { // Fallback to placeholder if image fails to load diff --git a/src/renderer/components/Editor/PostEditor.tsx b/src/renderer/components/Editor/PostEditor.tsx index b4821df..79b8a90 100644 --- a/src/renderer/components/Editor/PostEditor.tsx +++ b/src/renderer/components/Editor/PostEditor.tsx @@ -159,7 +159,6 @@ export const PostEditor: React.FC = ({ postId }) => { showErrorModal, showConfirmDeleteModal, media, - closeTab, } = useAppStore(); // Fetch full post data from backend @@ -194,7 +193,7 @@ export const PostEditor: React.FC = ({ postId }) => { const [doNotTranslate, setDoNotTranslate] = useState(false); const [activeEditingLanguage, setActiveEditingLanguage] = useState(''); const [canonicalDraft, setCanonicalDraft] = useState({ title: '', excerpt: '', content: '' }); - const [savedCanonicalDraft, setSavedCanonicalDraft] = useState({ title: '', excerpt: '', content: '' }); + const [, setSavedCanonicalDraft] = useState({ title: '', excerpt: '', content: '' }); const [translationDrafts, setTranslationDrafts] = useState>({}); const [savedTranslationDrafts, setSavedTranslationDrafts] = useState>({}); const [availablePostTemplates, setAvailablePostTemplates] = useState>([]); diff --git a/src/renderer/components/GitSidebar/GitSidebar.tsx b/src/renderer/components/GitSidebar/GitSidebar.tsx index 0d22ad7..56cf3bb 100644 --- a/src/renderer/components/GitSidebar/GitSidebar.tsx +++ b/src/renderer/components/GitSidebar/GitSidebar.tsx @@ -390,7 +390,7 @@ export const GitSidebar: React.FC = () => { recentCommitsToKeep: 2, }); if (!result.success) { - if (result.code === 'offline') { + if ('code' in result && result.code === 'offline') { showErrorModal({ message: tr('gitSidebar.error.offlineMode') }); return; } diff --git a/src/renderer/components/ImportAnalysisView/ImportAnalysisView.tsx b/src/renderer/components/ImportAnalysisView/ImportAnalysisView.tsx index c2bbf88..3a3f7a2 100644 --- a/src/renderer/components/ImportAnalysisView/ImportAnalysisView.tsx +++ b/src/renderer/components/ImportAnalysisView/ImportAnalysisView.tsx @@ -208,23 +208,25 @@ export const ImportAnalysisView: React.FC = ({ definiti // Subscribe to task completion events useEffect(() => { - const unsubscribe = window.electronAPI?.on('task:completed', (task: { taskId: string }) => { + const unsubscribe = window.electronAPI?.on('task:completed', ((...args: unknown[]) => { + const task = args[0] as { taskId: string }; setExecutionState(prev => { if (prev.taskId !== task.taskId) return prev; return { ...prev, isExecuting: false, completed: true }; }); - }); + }) as (...args: unknown[]) => void); return () => unsubscribe?.(); }, []); // Subscribe to task failure events useEffect(() => { - const unsubscribe = window.electronAPI?.on('task:failed', (task: { taskId: string; error: string }) => { + const unsubscribe = window.electronAPI?.on('task:failed', ((...args: unknown[]) => { + const task = args[0] as { taskId: string; error: string }; setExecutionState(prev => { if (prev.taskId !== task.taskId) return prev; return { ...prev, isExecuting: false, error: task.error }; }); - }); + }) as (...args: unknown[]) => void); return () => unsubscribe?.(); }, []); @@ -919,7 +921,7 @@ const DateDistributionCard: React.FC<{ distribution: DateDistribution }> = ({ di }; // Helper function to format post metadata for tooltip (new post from WXR) -function formatPostTooltip(wxrPost: AnalyzedPostItem['wxrPost'], t: (key: string, params?: Record) => string): string { +function formatPostTooltip(wxrPost: AnalyzedPostItem['wxrPost'], t: (key: string, params?: Record) => string): string { const lines: string[] = []; lines.push(`${t('importAnalysis.wordpressId')}: ${wxrPost.wpId}`); lines.push(`${t('importAnalysis.type')}: ${wxrPost.postType}`); @@ -1051,7 +1053,7 @@ function ExistingPostHoverCard({ children, className, postId }: { } // Helper function to format media metadata for tooltip -function formatMediaTooltip(wxrMedia: AnalyzedMediaItem['wxrMedia'], t: (key: string, params?: Record) => string): string { +function formatMediaTooltip(wxrMedia: AnalyzedMediaItem['wxrMedia'], t: (key: string, params?: Record) => string): string { const lines: string[] = []; lines.push(`${t('importAnalysis.wordpressId')}: ${wxrMedia.wpId}`); lines.push(`${t('importAnalysis.mimeType')}: ${wxrMedia.mimeType || t('importAnalysis.unknown')}`); diff --git a/src/renderer/components/MilkdownEditor/MilkdownEditor.tsx b/src/renderer/components/MilkdownEditor/MilkdownEditor.tsx index 532166b..49e3356 100644 --- a/src/renderer/components/MilkdownEditor/MilkdownEditor.tsx +++ b/src/renderer/components/MilkdownEditor/MilkdownEditor.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useRef, useCallback, useState } from 'react'; import { Editor, defaultValueCtx, editorViewCtx, rootCtx, remarkStringifyOptionsCtx, remarkPluginsCtx } from '@milkdown/kit/core'; -import { commonmark, toggleStrongCommand, toggleEmphasisCommand, wrapInBlockquoteCommand, wrapInBulletListCommand, wrapInOrderedListCommand, insertHrCommand, toggleInlineCodeCommand, insertImageCommand, toggleLinkCommand } from '@milkdown/kit/preset/commonmark'; +import { commonmark, toggleStrongCommand, toggleEmphasisCommand, wrapInBlockquoteCommand, wrapInBulletListCommand, wrapInOrderedListCommand, insertHrCommand, toggleInlineCodeCommand, insertImageCommand } from '@milkdown/kit/preset/commonmark'; import { gfm, toggleStrikethroughCommand } from '@milkdown/kit/preset/gfm'; import { history, undoCommand, redoCommand } from '@milkdown/kit/plugin/history'; import { listener, listenerCtx } from '@milkdown/kit/plugin/listener'; diff --git a/src/renderer/components/ScriptsView/ScriptsView.tsx b/src/renderer/components/ScriptsView/ScriptsView.tsx index 35f1a11..ca27f7a 100644 --- a/src/renderer/components/ScriptsView/ScriptsView.tsx +++ b/src/renderer/components/ScriptsView/ScriptsView.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { useCallback, useRef, useState } from 'react'; import MonacoEditor, { type Monaco } from '@monaco-editor/react'; import type { ScriptData } from '../../../main/shared/electronApi'; import { useAppStore } from '../../store'; @@ -89,7 +89,6 @@ export const ScriptsView: React.FC = ({ scriptId }) => { // Refresh entrypoints asynchronously entrypointCancelRef.current = true; // cancel any pending refresh - const cancelToken = {}; entrypointCancelRef.current = false; const refreshEntrypoints = async () => { try { diff --git a/src/renderer/components/Sidebar/SidebarEntityList.tsx b/src/renderer/components/Sidebar/SidebarEntityList.tsx index 1acccf4..826b656 100644 --- a/src/renderer/components/Sidebar/SidebarEntityList.tsx +++ b/src/renderer/components/Sidebar/SidebarEntityList.tsx @@ -28,7 +28,7 @@ export function SidebarEntityList({ renderItem, getItemKey, topContent, -}: SidebarEntityListProps): JSX.Element { +}: SidebarEntityListProps): React.JSX.Element { if (isLoading) { return (
diff --git a/src/renderer/components/WindowTitleBar/WindowTitleBar.tsx b/src/renderer/components/WindowTitleBar/WindowTitleBar.tsx index f02274e..e86815a 100644 --- a/src/renderer/components/WindowTitleBar/WindowTitleBar.tsx +++ b/src/renderer/components/WindowTitleBar/WindowTitleBar.tsx @@ -346,7 +346,7 @@ export const WindowTitleBar: React.FC = () => { }; }, [isMac, mnemonicByKey, showMnemonics]); - const handleMenuButtonClick = (event: React.MouseEvent, label: string) => { + const handleMenuButtonClick = (_event: React.MouseEvent, label: string) => { const left = getMenuLeft(label); if (left === null) { return; diff --git a/src/renderer/macros/pythonMacroPreview.ts b/src/renderer/macros/pythonMacroPreview.ts index 34dff7e..a88a137 100644 --- a/src/renderer/macros/pythonMacroPreview.ts +++ b/src/renderer/macros/pythonMacroPreview.ts @@ -1,4 +1,4 @@ -import type { PythonMacroInfo, PythonMacroResolver, PythonMacroRendererFn } from './types'; +import type { PythonMacroResolver, PythonMacroRendererFn } from './types'; import { setPythonMacroResolver } from './registry'; import { getPythonRuntimeManager } from '../python/runtimeManagerInstance'; diff --git a/src/renderer/navigation/assistantPanelSpec.ts b/src/renderer/navigation/assistantPanelSpec.ts index 85beb10..cf82fa9 100644 --- a/src/renderer/navigation/assistantPanelSpec.ts +++ b/src/renderer/navigation/assistantPanelSpec.ts @@ -138,7 +138,7 @@ const tabsElementSchema: z.ZodTypeAny = z.lazy(() => z.object({ ).min(1), })); -assistantPanelElementSchemaRef = z.discriminatedUnion('type', [ +assistantPanelElementSchemaRef = z.union([ textElementSchema, metricElementSchema, listElementSchema, diff --git a/src/renderer/navigation/blogmarkHandling.ts b/src/renderer/navigation/blogmarkHandling.ts index 83faa3d..4298504 100644 --- a/src/renderer/navigation/blogmarkHandling.ts +++ b/src/renderer/navigation/blogmarkHandling.ts @@ -1,4 +1,4 @@ -import { openEntityTab } from './tabPolicy'; +import { openEntityTab, type CanonicalTabSpec } from './tabPolicy'; import type { SidebarView } from './sidebarViewRegistry'; interface BlogmarkStateSnapshot { @@ -14,7 +14,7 @@ interface BlogmarkHandlers { setActiveView: (view: SidebarView) => void; toggleSidebar: () => void; setSelectedPost: (id: string) => void; - openTab: (tab: { type: 'post'; id: string; isTransient: boolean }) => void; + openTab: (tab: CanonicalTabSpec) => void; } export function handleBlogmarkCreatedEvent( diff --git a/src/renderer/navigation/blogmarkTransformOutput.ts b/src/renderer/navigation/blogmarkTransformOutput.ts index 9625efb..cc53316 100644 --- a/src/renderer/navigation/blogmarkTransformOutput.ts +++ b/src/renderer/navigation/blogmarkTransformOutput.ts @@ -83,13 +83,13 @@ export function parseBlogmarkCreatedEventPayload(payload: unknown): BlogmarkCrea if (isRecord(payload.post)) { return { - post: payload.post as PostData, + post: payload.post as unknown as PostData, transform: parseTransformDebugInfo(payload.transform), }; } return { - post: payload as PostData, + post: payload as unknown as PostData, transform: undefined, }; } diff --git a/src/renderer/python/PythonRuntimeManager.ts b/src/renderer/python/PythonRuntimeManager.ts index 6de0d0a..1c7dbae 100644 --- a/src/renderer/python/PythonRuntimeManager.ts +++ b/src/renderer/python/PythonRuntimeManager.ts @@ -1,7 +1,7 @@ import { createPythonRuntimeWorker } from './createPythonRuntimeWorker'; import type { PythonWorkerMessage, PythonWorkerRequest } from './runtimeProtocol'; import type { PythonSyntaxError } from './runtimeProtocol'; -import { parseMacroContextV1, parseMacroResultV1, type MacroContextV1, type MacroResultV1 } from './abiV1'; +import { parseMacroContextV1, parseMacroResultV1, type MacroResultV1 } from './abiV1'; import { invokePythonApiMethodV1 } from './pythonApiInvokerV1'; import { showToast } from '../components/Toast'; diff --git a/src/renderer/types/highlightjs-cdn-assets.d.ts b/src/renderer/types/highlightjs-cdn-assets.d.ts new file mode 100644 index 0000000..08791a0 --- /dev/null +++ b/src/renderer/types/highlightjs-cdn-assets.d.ts @@ -0,0 +1,4 @@ +declare module '@highlightjs/cdn-assets/es/highlight.min.js' { + import hljs from 'highlight.js'; + export default hljs; +} diff --git a/tests/engine/BlogGenerationEngine.test.ts b/tests/engine/BlogGenerationEngine.test.ts index 175c25c..395fbfd 100644 --- a/tests/engine/BlogGenerationEngine.test.ts +++ b/tests/engine/BlogGenerationEngine.test.ts @@ -57,6 +57,7 @@ vi.mock('../../src/main/database/generatedFileHashStore', () => ({ getGeneratedFileHash: getGeneratedFileHashMock, getGeneratedFileHashRecord: getGeneratedFileHashRecordMock, setGeneratedFileHash: setGeneratedFileHashMock, + getAllGeneratedFileHashes: vi.fn(async () => new Map()), })); vi.mock('../../src/main/database', () => ({ @@ -76,7 +77,7 @@ vi.mock('../../src/main/engine/PostEngine', async (importOriginal) => { getPostTranslation: vi.fn(async () => null), getPostTranslations: vi.fn(async () => []), setProjectContext: vi.fn(), - }; + } as Record; return { ...actual, getPostEngine: vi.fn(() => mockPostEngine), @@ -211,6 +212,7 @@ describe('BlogGenerationEngine', () => { options?: Partial<{ maxPostsPerPage: number; language: string; + blogLanguages: string[]; pageTitle: string; picoTheme: string; categorySettings: Record; @@ -229,6 +231,7 @@ describe('BlogGenerationEngine', () => { baseUrl: 'https://example.com', maxPostsPerPage: options?.maxPostsPerPage, language: options?.language, + blogLanguages: options?.blogLanguages, pageTitle: options?.pageTitle, picoTheme: options?.picoTheme as any, categorySettings: options?.categorySettings, @@ -2148,6 +2151,154 @@ describe('BlogGenerationEngine', () => { expect(result.postCount).toBe(0); }); + it('language subtree list pages show translated title and excerpt, not canonical language', async () => { + const posts = [ + makePost({ + id: 'de-post-1', + slug: 'german-post', + title: 'Deutscher Titel', + excerpt: 'Deutscher Auszug', + content: '# Deutscher Inhalt', + language: 'de', + categories: ['tech'], + createdAt: new Date('2025-06-10T10:00:00Z'), + availableLanguages: ['de', 'en'], + }), + ]; + + const translationMap = new Map(); + translationMap.set('de-post-1', [{ + id: 'en-trans-1', + projectId: 'test', + translationFor: 'de-post-1', + language: 'en', + title: 'English Title', + excerpt: 'English excerpt', + content: '# English Content', + status: 'published', + createdAt: new Date('2025-06-10T10:00:00Z'), + updatedAt: new Date('2025-06-10T10:00:00Z'), + publishedAt: new Date('2025-06-10T10:00:00Z'), + filePath: '', + }]); + mockPostEngine.getPublishedTranslationsForRoutePosts = vi.fn().mockResolvedValue(translationMap); + + await generate(posts, { language: 'de', blogLanguages: ['de', 'en'] }); + + // /en/ subtree list page should show English title and excerpt + const enIndex = await readFile(path.join(tempDir, 'html', 'en', 'index.html'), 'utf-8'); + expect(enIndex).toContain('English Title'); + expect(enIndex).toContain('English excerpt'); + expect(enIndex).not.toContain('Deutscher Titel'); + expect(enIndex).not.toContain('Deutscher Auszug'); + + // Main blog list page should still show German + const deIndex = await readFile(path.join(tempDir, 'html', 'index.html'), 'utf-8'); + expect(deIndex).toContain('Deutscher Titel'); + expect(deIndex).not.toContain('English Title'); + }); + + it('main blog list pages show translated content when canonical language differs from project language', async () => { + const posts = [ + makePost({ + id: 'en-post-1', + slug: 'english-post', + title: 'English Title', + excerpt: 'English excerpt', + content: '# English Content', + language: 'en', + categories: ['tech'], + createdAt: new Date('2025-06-10T10:00:00Z'), + availableLanguages: ['en', 'de'], + }), + ]; + + const translationMap = new Map(); + translationMap.set('en-post-1', [{ + id: 'de-trans-1', + projectId: 'test', + translationFor: 'en-post-1', + language: 'de', + title: 'Deutscher Titel', + excerpt: 'Deutscher Auszug', + content: '# Deutscher Inhalt', + status: 'published', + createdAt: new Date('2025-06-10T10:00:00Z'), + updatedAt: new Date('2025-06-10T10:00:00Z'), + publishedAt: new Date('2025-06-10T10:00:00Z'), + filePath: '', + }]); + mockPostEngine.getPublishedTranslationsForRoutePosts = vi.fn().mockResolvedValue(translationMap); + + await generate(posts, { language: 'de', blogLanguages: ['de', 'en'] }); + + // Main blog (de) should show German translated title, not English canonical + const deIndex = await readFile(path.join(tempDir, 'html', 'index.html'), 'utf-8'); + expect(deIndex).toContain('Deutscher Titel'); + expect(deIndex).not.toContain('English Title'); + + // /en/ subtree should show English canonical title + const enIndex = await readFile(path.join(tempDir, 'html', 'en', 'index.html'), 'utf-8'); + expect(enIndex).toContain('English Title'); + expect(enIndex).not.toContain('Deutscher Titel'); + }); + + it('language subtree RSS and Atom feeds use translated titles and content', async () => { + const posts = [ + makePost({ + id: 'de-post-1', + slug: 'german-post', + title: 'Deutscher Titel', + content: '# Deutscher Inhalt\n\nDeutscher Body Text', + language: 'de', + categories: ['tech'], + createdAt: new Date('2025-06-10T10:00:00Z'), + availableLanguages: ['de', 'en'], + }), + ]; + + const translationFilePath = path.join(tempDir, 'posts', 'german-post.en.md'); + await mkdir(path.join(tempDir, 'posts'), { recursive: true }); + await writeFile(translationFilePath, '---\ntranslationFor: de-post-1\nlanguage: en\ntitle: English Title\n---\n# English Content\n\nEnglish Body Text'); + + const translationMap = new Map(); + translationMap.set('de-post-1', [{ + id: 'en-trans-1', + projectId: 'test', + translationFor: 'de-post-1', + language: 'en', + title: 'English Title', + content: '# English Content\n\nEnglish Body Text', + status: 'published', + createdAt: new Date('2025-06-10T10:00:00Z'), + updatedAt: new Date('2025-06-10T10:00:00Z'), + publishedAt: new Date('2025-06-10T10:00:00Z'), + filePath: translationFilePath, + }]); + mockPostEngine.getPublishedTranslationsForRoutePosts = vi.fn().mockResolvedValue(translationMap); + + await generate(posts, { language: 'de', blogLanguages: ['de', 'en'] }); + + // /en/ RSS feed should use English translated title and content + const enRss = await readFile(path.join(tempDir, 'html', 'en', 'rss.xml'), 'utf-8'); + expect(enRss).toContain('English Title'); + expect(enRss).not.toContain('Deutscher Titel'); + expect(enRss).toContain('English Body Text'); + expect(enRss).not.toContain('Deutscher Body Text'); + + // /en/ Atom feed should use English translated title and content + const enAtom = await readFile(path.join(tempDir, 'html', 'en', 'atom.xml'), 'utf-8'); + expect(enAtom).toContain('English Title'); + expect(enAtom).not.toContain('Deutscher Titel'); + expect(enAtom).toContain('English Body Text'); + expect(enAtom).not.toContain('Deutscher Body Text'); + + // Root RSS should keep German canonical content + const deRss = await readFile(path.join(tempDir, 'html', 'rss.xml'), 'utf-8'); + expect(deRss).toContain('Deutscher Titel'); + expect(deRss).not.toContain('English Title'); + }); + it('generates pagination links in list pages', async () => { const posts: PostData[] = []; for (let i = 0; i < 4; i++) { diff --git a/tests/engine/DataBackedEngines.test.ts b/tests/engine/DataBackedEngines.test.ts new file mode 100644 index 0000000..1fd0ad1 --- /dev/null +++ b/tests/engine/DataBackedEngines.test.ts @@ -0,0 +1,471 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + createDataBackedPostEngine, + createDataBackedMediaEngine, + createDataBackedPostMediaEngine, +} from '../../src/main/engine/DataBackedEngines'; +import type { PostData } from '../../src/main/engine/PostEngine'; + +function makePost(overrides: Partial & { id: string; slug: string }): PostData { + return { + projectId: 'proj-1', + title: overrides.slug, + excerpt: '', + content: `content of ${overrides.slug}`, + status: 'published', + createdAt: new Date('2025-06-15T12:00:00Z'), + updatedAt: new Date('2025-06-15T12:00:00Z'), + tags: [], + categories: [], + availableLanguages: [], + ...overrides, + }; +} + +describe('DataBackedPostEngine', () => { + const posts: PostData[] = [ + makePost({ id: '1', slug: 'alpha', tags: ['js', 'node'], categories: ['article'], createdAt: new Date('2025-01-10T00:00:00Z') }), + makePost({ id: '2', slug: 'beta', tags: ['python'], categories: ['aside'], createdAt: new Date('2025-02-15T00:00:00Z') }), + makePost({ id: '3', slug: 'gamma', tags: ['js'], categories: ['page'], createdAt: new Date('2025-03-20T00:00:00Z') }), + makePost({ id: '4', slug: 'delta', tags: ['rust'], categories: ['article'], createdAt: new Date('2024-12-01T00:00:00Z') }), + ]; + + const backlinksMap = new Map([ + ['1', [{ id: '2', title: 'beta', slug: 'beta' }]], + ]); + + const engine = createDataBackedPostEngine({ allPosts: posts, backlinksMap }); + + describe('getPostsFiltered', () => { + it('returns all posts when no filter applied', async () => { + const result = await engine.getPostsFiltered({}); + expect(result).toHaveLength(4); + }); + + it('filters by status', async () => { + const result = await engine.getPostsFiltered({ status: 'published' }); + expect(result).toHaveLength(4); + + const drafts = await engine.getPostsFiltered({ status: 'draft' }); + expect(drafts).toHaveLength(0); + }); + + it('filters by excludeCategories', async () => { + const result = await engine.getPostsFiltered({ excludeCategories: ['page'] }); + expect(result).toHaveLength(3); + expect(result.every((p) => !p.categories.includes('page'))).toBe(true); + }); + + it('filters by categories', async () => { + const result = await engine.getPostsFiltered({ categories: ['article'] }); + expect(result).toHaveLength(2); + }); + + it('filters by tags (all must match)', async () => { + const result = await engine.getPostsFiltered({ tags: ['js'] }); + expect(result).toHaveLength(2); + + const both = await engine.getPostsFiltered({ tags: ['js', 'node'] }); + expect(both).toHaveLength(1); + expect(both[0].slug).toBe('alpha'); + }); + + it('filters by year', async () => { + const result = await engine.getPostsFiltered({ year: 2025 }); + expect(result).toHaveLength(3); + }); + + it('filters by year and month', async () => { + const result = await engine.getPostsFiltered({ year: 2025, month: 2 }); + expect(result).toHaveLength(1); + expect(result[0].slug).toBe('beta'); + }); + + it('filters by date range', async () => { + const result = await engine.getPostsFiltered({ + startDate: new Date('2025-01-01T00:00:00Z'), + endDate: new Date('2025-02-28T23:59:59Z'), + }); + expect(result).toHaveLength(2); + }); + + it('returns sorted by createdAt descending', async () => { + const result = await engine.getPostsFiltered({}); + const dates = result.map((p) => p.createdAt.getTime()); + for (let i = 1; i < dates.length; i++) { + expect(dates[i]).toBeLessThanOrEqual(dates[i - 1]); + } + }); + + it('excludes translation variants from filtered results', async () => { + const canonical = makePost({ id: 'c1', slug: 'hello', content: 'Hello content', language: 'de' }); + const variant = makePost({ + id: 'v1', + slug: 'hello.en', + content: 'Hello EN content', + language: 'en', + translationSourceSlug: 'hello', + } as any); + + const eng = createDataBackedPostEngine({ allPosts: [canonical, variant] }); + const result = await eng.getPostsFiltered({ status: 'published' }); + expect(result).toHaveLength(1); + expect(result[0].slug).toBe('hello'); + }); + + it('translation variants remain accessible via findPublishedBySlug', async () => { + const canonical = makePost({ id: 'c1', slug: 'hello', content: 'Hello content', language: 'de' }); + const variant = makePost({ + id: 'v1', + slug: 'hello.en', + content: 'Hello EN content', + language: 'en', + translationSourceSlug: 'hello', + } as any); + + const eng = createDataBackedPostEngine({ allPosts: [canonical, variant] }); + const found = await eng.findPublishedBySlug('hello.en'); + expect(found).not.toBeNull(); + expect(found!.id).toBe('v1'); + }); + + it('includes resolved posts (slug === translationSourceSlug) in filtered results', async () => { + // A resolved post is a canonical post with its title/excerpt replaced by a translation. + // Its slug remains the same as translationSourceSlug (e.g., both are "hello"). + const resolved = makePost({ + id: 'r1', + slug: 'hello', + title: 'Hallo (übersetzt)', + language: 'en', + translationSourceSlug: 'hello', + } as any); + const unresolved = makePost({ + id: 'r2', + slug: 'world', + title: 'Welt', + language: 'de', + }); + + const eng = createDataBackedPostEngine({ allPosts: [resolved, unresolved] }); + const result = await eng.getPostsFiltered({ status: 'published' }); + expect(result).toHaveLength(2); + expect(result.map((p) => p.slug).sort()).toEqual(['hello', 'world']); + }); + + it('lazy-loads translation content for resolved posts returned by getPostsFiltered', async () => { + const resolved = makePost({ + id: 'lz-1', + slug: 'lazy-post', + title: 'Translated Title', + content: '', + language: 'en', + translationSourceSlug: 'lazy-post', + translationFilePath: '/data/posts/2025/06/lazy-post.en.md', + } as any); + + const readTranslationSpy = vi.spyOn(await import('../../src/main/engine/postTranslationFileUtils'), 'readPostTranslationFile'); + readTranslationSpy.mockResolvedValueOnce({ + translationFor: 'lz-1', + language: 'en', + title: 'Translated Title', + content: 'Lazy loaded translation content!', + }); + + const eng = createDataBackedPostEngine({ + allPosts: [resolved], + postFilePaths: new Map(), + }); + + const result = await eng.getPostsFiltered({ status: 'published' }); + expect(result).toHaveLength(1); + expect(result[0].content).toBe('Lazy loaded translation content!'); + expect(readTranslationSpy).toHaveBeenCalledWith('/data/posts/2025/06/lazy-post.en.md'); + + readTranslationSpy.mockRestore(); + }); + }); + + describe('lookup methods', () => { + it('getPublishedVersion returns by id', async () => { + const post = await engine.getPublishedVersion('2'); + expect(post?.slug).toBe('beta'); + }); + + it('getPublishedVersion returns null for unknown id', async () => { + expect(await engine.getPublishedVersion('unknown')).toBeNull(); + }); + + it('getPost returns by id', async () => { + const post = await engine.getPost('3'); + expect(post?.slug).toBe('gamma'); + }); + + it('hasPublishedVersion checks existence', async () => { + expect(await engine.hasPublishedVersion('1')).toBe(true); + expect(await engine.hasPublishedVersion('nope')).toBe(false); + }); + + it('findPublishedBySlug returns first match without date filter', async () => { + const post = await engine.findPublishedBySlug('alpha'); + expect(post?.id).toBe('1'); + }); + + it('findPublishedBySlug returns null for unknown slug', async () => { + expect(await engine.findPublishedBySlug('nope')).toBeNull(); + }); + + it('findPublishedBySlug applies date filter', async () => { + const match = await engine.findPublishedBySlug('alpha', { year: 2025, month: 1 }); + expect(match?.id).toBe('1'); + + const noMatch = await engine.findPublishedBySlug('alpha', { year: 2024, month: 1 }); + expect(noMatch).toBeNull(); + }); + }); + + describe('backlinks', () => { + it('getLinkedBy returns backlinks for a post', async () => { + const links = await engine.getLinkedBy('1'); + expect(links).toEqual([{ id: '2', title: 'beta', slug: 'beta' }]); + }); + + it('getLinkedBy returns empty for unlinked post', async () => { + const links = await engine.getLinkedBy('3'); + expect(links).toEqual([]); + }); + + it('getAllBacklinks returns the full map', async () => { + const map = await engine.getAllBacklinks(); + expect(map.size).toBe(1); + expect(map.get('1')).toHaveLength(1); + }); + }); + + describe('translations (stubs)', () => { + it('getPostTranslation returns null', async () => { + expect(await engine.getPostTranslation('1', 'fr')).toBeNull(); + }); + + it('getPostTranslations returns empty array', async () => { + expect(await engine.getPostTranslations('1')).toEqual([]); + }); + }); + + describe('setProjectContext is a no-op', () => { + it('does not throw', () => { + expect(() => engine.setProjectContext('proj-1', '/data')).not.toThrow(); + }); + }); + + describe('getPublishedVersion with lazy FS content loading', () => { + it('reads content from filesystem when post has no content and filePath is available', async () => { + const emptyContentPost = makePost({ id: 'fs-1', slug: 'fs-post', content: '' }); + const postFilePaths = new Map([['fs-1', '/data/posts/2025/06/fs-post.md']]); + + const { readPostFile } = await import('../../src/main/engine/postFileUtils'); + const readPostFileSpy = vi.spyOn(await import('../../src/main/engine/postFileUtils'), 'readPostFile'); + readPostFileSpy.mockResolvedValueOnce({ + id: 'fs-1', + title: 'FS Post', + slug: 'fs-post', + content: 'Content loaded from filesystem!', + status: 'published', + createdAt: new Date('2025-06-15'), + updatedAt: new Date('2025-06-15'), + tags: [], + categories: [], + }); + + const fsEngine = createDataBackedPostEngine({ + allPosts: [emptyContentPost], + postFilePaths, + }); + + const result = await fsEngine.getPublishedVersion('fs-1'); + expect(result).not.toBeNull(); + expect(result!.content).toBe('Content loaded from filesystem!'); + expect(readPostFileSpy).toHaveBeenCalledWith('/data/posts/2025/06/fs-post.md'); + + readPostFileSpy.mockRestore(); + }); + + it('does not read from FS when post already has content', async () => { + const postWithContent = makePost({ id: 'has-c', slug: 'has-content', content: 'Already here' }); + const postFilePaths = new Map([['has-c', '/data/posts/2025/06/has-content.md']]); + + const readPostFileSpy = vi.spyOn(await import('../../src/main/engine/postFileUtils'), 'readPostFile'); + + const fsEngine = createDataBackedPostEngine({ + allPosts: [postWithContent], + postFilePaths, + }); + + const result = await fsEngine.getPublishedVersion('has-c'); + expect(result!.content).toBe('Already here'); + expect(readPostFileSpy).not.toHaveBeenCalled(); + + readPostFileSpy.mockRestore(); + }); + + it('reads translation content from translationFilePath', async () => { + const translationPost = makePost({ id: 'tr-1', slug: 'post.fr', content: '' }); + (translationPost as any).translationFilePath = '/data/posts/2025/06/post.fr.md'; + + const readTranslationSpy = vi.spyOn(await import('../../src/main/engine/postTranslationFileUtils'), 'readPostTranslationFile'); + readTranslationSpy.mockResolvedValueOnce({ + translationFor: 'original-id', + language: 'fr', + title: 'Post FR', + content: 'Contenu fran\u00e7ais!', + }); + + const fsEngine = createDataBackedPostEngine({ + allPosts: [translationPost], + postFilePaths: new Map(), + }); + + const result = await fsEngine.getPublishedVersion('tr-1'); + expect(result!.content).toBe('Contenu fran\u00e7ais!'); + expect(readTranslationSpy).toHaveBeenCalledWith('/data/posts/2025/06/post.fr.md'); + + readTranslationSpy.mockRestore(); + }); + }); + + describe('getPost with lazy FS content loading', () => { + it('reads content from filesystem when post has no content', async () => { + const emptyContentPost = makePost({ id: 'gp-1', slug: 'gp-post', content: '' }); + const postFilePaths = new Map([['gp-1', '/data/posts/2025/06/gp-post.md']]); + + const readPostFileSpy = vi.spyOn(await import('../../src/main/engine/postFileUtils'), 'readPostFile'); + readPostFileSpy.mockResolvedValueOnce({ + id: 'gp-1', + title: 'GP Post', + slug: 'gp-post', + content: 'Content via getPost!', + status: 'published', + createdAt: new Date('2025-06-15'), + updatedAt: new Date('2025-06-15'), + tags: [], + categories: [], + }); + + const fsEngine = createDataBackedPostEngine({ + allPosts: [emptyContentPost], + postFilePaths, + }); + + const result = await fsEngine.getPost('gp-1'); + expect(result).not.toBeNull(); + expect(result!.content).toBe('Content via getPost!'); + expect(readPostFileSpy).toHaveBeenCalledWith('/data/posts/2025/06/gp-post.md'); + + readPostFileSpy.mockRestore(); + }); + }); +}); + +describe('DataBackedMediaEngine', () => { + const mediaItems: import('../../src/main/engine/MediaEngine').MediaData[] = [ + { + id: 'm1', filename: 'photo.jpg', originalName: 'my-photo.jpg', + mimeType: 'image/jpeg', size: 12345, width: 800, height: 600, + title: 'Beach', alt: 'Beach photo', caption: 'A sunny beach', + author: 'Alice', language: 'en', + createdAt: new Date('2025-01-01'), updatedAt: new Date('2025-01-02'), + tags: ['nature', 'beach'], linkedPostIds: ['p1', 'p2'], availableLanguages: ['en', 'de'], + }, + { + id: 'm2', filename: 'doc.pdf', originalName: 'document.pdf', + mimeType: 'application/pdf', size: 99999, + createdAt: new Date('2025-02-01'), updatedAt: new Date('2025-02-01'), + tags: [], availableLanguages: [], + }, + ]; + + const engine = createDataBackedMediaEngine(mediaItems); + + it('getAllMedia returns all items with full MediaData fields', async () => { + const result = await engine.getAllMedia(); + expect(result).toHaveLength(2); + expect(result[0].filename).toBe('photo.jpg'); + expect(result[0].mimeType).toBe('image/jpeg'); + expect(result[0].size).toBe(12345); + expect(result[0].width).toBe(800); + expect(result[0].height).toBe(600); + expect(result[0].title).toBe('Beach'); + expect(result[0].alt).toBe('Beach photo'); + expect(result[0].caption).toBe('A sunny beach'); + expect(result[0].author).toBe('Alice'); + expect(result[0].language).toBe('en'); + expect(result[0].tags).toEqual(['nature', 'beach']); + expect(result[0].linkedPostIds).toEqual(['p1', 'p2']); + expect(result[0].availableLanguages).toEqual(['en', 'de']); + }); + + it('setProjectContext is a no-op', () => { + expect(() => engine.setProjectContext('proj-1')).not.toThrow(); + }); +}); + +describe('DataBackedPostMediaEngine', () => { + const mediaItems: import('../../src/main/engine/MediaEngine').MediaData[] = [ + { + id: 'm1', filename: 'photo.jpg', originalName: 'my-photo.jpg', + mimeType: 'image/jpeg', size: 12345, width: 800, height: 600, + title: 'Beach', alt: 'Beach photo', caption: 'A sunny beach', + createdAt: new Date('2025-01-01'), updatedAt: new Date('2025-01-02'), + tags: ['nature'], linkedPostIds: ['p1'], availableLanguages: [], + }, + { + id: 'm2', filename: 'sunset.png', originalName: 'sunset.png', + mimeType: 'image/png', size: 5555, + createdAt: new Date('2025-02-01'), updatedAt: new Date('2025-02-01'), + tags: [], linkedPostIds: ['p1'], availableLanguages: [], + }, + { + id: 'm3', filename: 'doc.pdf', originalName: 'document.pdf', + mimeType: 'application/pdf', size: 99999, + createdAt: new Date('2025-03-01'), updatedAt: new Date('2025-03-01'), + tags: [], availableLanguages: [], + }, + ]; + + // Post p1 is linked to m1 (sort 0) and m2 (sort 1) + const postMediaLinks = new Map>([ + ['p1', [{ mediaId: 'm1', sortOrder: 0 }, { mediaId: 'm2', sortOrder: 1 }]], + ]); + + const engine = createDataBackedPostMediaEngine({ mediaItems, postMediaLinks }); + + it('getLinkedMediaForPost returns links for a post', async () => { + const result = await engine.getLinkedMediaForPost('p1'); + expect(result).toHaveLength(2); + expect(result[0].mediaId).toBe('m1'); + expect(result[1].mediaId).toBe('m2'); + }); + + it('getLinkedMediaForPost returns empty for unknown post', async () => { + const result = await engine.getLinkedMediaForPost('unknown'); + expect(result).toEqual([]); + }); + + it('getLinkedMediaDataForPost returns links with full media data', async () => { + const result = await engine.getLinkedMediaDataForPost('p1'); + expect(result).toHaveLength(2); + expect(result[0].media.id).toBe('m1'); + expect(result[0].media.mimeType).toBe('image/jpeg'); + expect(result[0].media.caption).toBe('A sunny beach'); + expect(result[1].media.id).toBe('m2'); + expect(result[1].media.mimeType).toBe('image/png'); + }); + + it('getLinkedMediaDataForPost returns empty for unknown post', async () => { + const result = await engine.getLinkedMediaDataForPost('unknown'); + expect(result).toEqual([]); + }); + + it('setProjectContext is a no-op', () => { + expect(() => engine.setProjectContext('proj-1')).not.toThrow(); + }); +}); diff --git a/tests/engine/GenerationPostSnapshotService.test.ts b/tests/engine/GenerationPostSnapshotService.test.ts index d98c0bd..36f0209 100644 --- a/tests/engine/GenerationPostSnapshotService.test.ts +++ b/tests/engine/GenerationPostSnapshotService.test.ts @@ -75,4 +75,65 @@ describe('GenerationPostSnapshotService', () => { expect(result.publishedPosts.map((post) => post.id).sort()).toEqual(['article', 'page']); expect(result.publishedListPosts.map((post) => post.id)).toEqual(['article']); }); + + it('uses getPublishedVersionsBulk when available for efficient loading', async () => { + const published = makePost({ id: 'pub-1', status: 'published', categories: ['news'] }); + const draft = makePost({ id: 'draft-1', status: 'draft', categories: ['news'] }); + const pubSnapshot = makePost({ id: 'pub-1', status: 'published', categories: ['news'], title: 'Snapshot Title' }); + const draftSnapshot = makePost({ id: 'draft-1', status: 'published', categories: ['news'] }); + + const engine = makeEngine([published, draft]); + let individualCallCount = 0; + engine.getPublishedVersion = async () => { + individualCallCount++; + return null; + }; + engine.getPublishedVersionsBulk = async (ids: string[]) => { + const map = new Map(); + if (ids.includes('pub-1')) map.set('pub-1', pubSnapshot); + if (ids.includes('draft-1')) map.set('draft-1', draftSnapshot); + return map; + }; + + const result = await loadPublishedGenerationSets(engine, []); + + expect(individualCallCount).toBe(0); + expect(result.publishedPosts).toHaveLength(2); + expect(result.publishedPosts.find(p => p.id === 'pub-1')?.title).toBe('Snapshot Title'); + }); + + it('uses bulk loading with list-excluded categories filtered in memory', async () => { + const article = makePost({ id: 'article', status: 'published', categories: ['article'] }); + const page = makePost({ id: 'page', status: 'published', categories: ['page'] }); + const articleSnapshot = makePost({ id: 'article', status: 'published', categories: ['article'], title: 'Article Snap' }); + const pageSnapshot = makePost({ id: 'page', status: 'published', categories: ['page'], title: 'Page Snap' }); + + const engine = makeEngine([article, page]); + engine.getPublishedVersionsBulk = async (ids: string[]) => { + const map = new Map(); + if (ids.includes('article')) map.set('article', articleSnapshot); + if (ids.includes('page')) map.set('page', pageSnapshot); + return map; + }; + + const result = await loadPublishedGenerationSets(engine, ['page']); + + expect(result.publishedPosts.map(p => p.id).sort()).toEqual(['article', 'page']); + expect(result.publishedListPosts.map(p => p.id)).toEqual(['article']); + }); + + it('only calls getPostsFiltered twice (published + draft), not four times', async () => { + const post = makePost({ id: 'p1', status: 'published' }); + const engine = makeEngine([post]); + let filterCallCount = 0; + const originalGetPostsFiltered = engine.getPostsFiltered; + engine.getPostsFiltered = async (filter) => { + filterCallCount++; + return originalGetPostsFiltered(filter); + }; + + await loadPublishedGenerationSets(engine, ['some-category']); + + expect(filterCallCount).toBe(2); + }); }); diff --git a/tests/engine/GenerationRouteRendererFactory.test.ts b/tests/engine/GenerationRouteRendererFactory.test.ts index 39a03cd..cbdf45f 100644 --- a/tests/engine/GenerationRouteRendererFactory.test.ts +++ b/tests/engine/GenerationRouteRendererFactory.test.ts @@ -183,4 +183,59 @@ describe('GenerationRouteRendererFactory', () => { expect(html).toContain('youtube.com/embed/dQw4w9WgXcQ?rel=0'); expect(html).toContain('/assets/bds.css'); }); + + it('produces correct language flags for subtree pages when projectMainLanguage is set', async () => { + const post = makePost({ + id: 'lang-1', + slug: 'lang-post', + title: 'Sprachbeitrag', + content: 'Inhalt des Beitrags.', + createdAt: new Date('2025-01-15T10:00:00.000Z'), + }); + + const postEngine = { + getPostsFiltered: vi.fn(async () => [post]), + getPublishedVersion: vi.fn(async () => null), + findPublishedBySlug: vi.fn(async (slug: string) => (slug === post.slug ? post : null)), + getPost: vi.fn(async (id: string) => (id === post.id ? post : null)), + hasPublishedVersion: vi.fn(async () => false), + setProjectContext: vi.fn(), + }; + + const renderRoute = createPreviewBackedGenerationRouteRenderer({ + options: { + projectId: 'project', + dataDir: '/tmp', + projectName: 'Project', + language: 'en', + blogLanguages: ['de', 'en'], + }, + projectMainLanguage: 'de', + maxPostsPerPage: 50, + publishedPostsForLookup: [post], + languagePrefix: '/en', + engines: { + postEngine, + mediaEngine: { + getAllMedia: vi.fn(async () => []), + setProjectContext: vi.fn(), + }, + postMediaEngine: { + setProjectContext: vi.fn(), + getLinkedMediaForPost: vi.fn(async () => []), + getLinkedMediaDataForPost: vi.fn(async () => []), + }, + }, + }); + + const html = await renderRoute('/'); + + // German flag should point to root (main language), not /de/ + expect(html).toContain('🇩🇪'); + expect(html).toContain('🇬🇧'); + // Main language link should have no prefix (root) + expect(html).not.toContain('href="/de'); + // English flag should show /en prefix + expect(html).toContain('/en'); + }); }); diff --git a/tests/engine/GenerationWorkerData.test.ts b/tests/engine/GenerationWorkerData.test.ts new file mode 100644 index 0000000..719c3dc --- /dev/null +++ b/tests/engine/GenerationWorkerData.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, it } from 'vitest'; +import { + serializePostData, + deserializePostData, + serializeMediaItem, + deserializeMediaItem, + serializeBlogGenerationOptions, + serializePostMap, + deserializePostMap, + serializeDateMap, + deserializeDateMap, +} from '../../src/main/engine/GenerationWorkerData'; +import type { PostData } from '../../src/main/engine/PostEngine'; + +function makePost(overrides: Partial & { id: string; slug: string }): PostData { + return { + projectId: 'proj-1', + title: overrides.slug, + excerpt: 'short', + content: `body of ${overrides.slug}`, + status: 'published', + createdAt: new Date('2025-06-15T12:00:00Z'), + updatedAt: new Date('2025-06-15T14:00:00Z'), + tags: ['a', 'b'], + categories: ['article'], + availableLanguages: ['en'], + ...overrides, + }; +} + +describe('PostData serialization', () => { + it('round-trips a basic post', () => { + const post = makePost({ id: '1', slug: 'hello' }); + const serialized = serializePostData(post); + const deserialized = deserializePostData(serialized); + + expect(deserialized.id).toBe('1'); + expect(deserialized.slug).toBe('hello'); + expect(deserialized.createdAt).toBeInstanceOf(Date); + expect(deserialized.createdAt.toISOString()).toBe('2025-06-15T12:00:00.000Z'); + expect(deserialized.updatedAt).toBeInstanceOf(Date); + expect(deserialized.tags).toEqual(['a', 'b']); + expect(deserialized.categories).toEqual(['article']); + expect(deserialized.content).toBe('body of hello'); + }); + + it('round-trips publishedAt', () => { + const post = makePost({ id: '2', slug: 'pub', publishedAt: new Date('2025-07-01T00:00:00Z') }); + const result = deserializePostData(serializePostData(post)); + expect(result.publishedAt).toBeInstanceOf(Date); + expect(result.publishedAt?.toISOString()).toBe('2025-07-01T00:00:00.000Z'); + }); + + it('round-trips undefined publishedAt', () => { + const post = makePost({ id: '3', slug: 'nopub' }); + const result = deserializePostData(serializePostData(post)); + expect(result.publishedAt).toBeUndefined(); + }); + + it('preserves translation variant fields', () => { + const post = makePost({ id: '4', slug: 'hello.fr' }); + (post as any).translationSourceSlug = 'hello'; + (post as any).translationCanonicalLanguage = 'en'; + (post as any).translationFilePath = '/data/translations/hello.fr.md'; + + const result = deserializePostData(serializePostData(post)); + expect((result as any).translationSourceSlug).toBe('hello'); + expect((result as any).translationCanonicalLanguage).toBe('en'); + expect((result as any).translationFilePath).toBe('/data/translations/hello.fr.md'); + }); + + it('handles post with Date already as string (defensive)', () => { + const post = makePost({ id: '5', slug: 'strdate' }); + (post as any).createdAt = '2025-01-01T00:00:00.000Z'; + const serialized = serializePostData(post); + expect(serialized.createdAt).toBe('2025-01-01T00:00:00.000Z'); + + const deserialized = deserializePostData(serialized); + expect(deserialized.createdAt).toBeInstanceOf(Date); + }); +}); + +describe('MediaItem serialization', () => { + it('round-trips a media item with all fields', () => { + const media = { + id: 'm1', + filename: 'photo.webp', + originalName: 'My Photo.webp', + mimeType: 'image/webp', + size: 54321, + width: 1920, + height: 1080, + title: 'Sunset', + alt: 'A beautiful sunset', + caption: 'Taken at the beach', + author: 'Bob', + language: 'en', + createdAt: new Date('2025-03-01T10:00:00Z'), + updatedAt: new Date('2025-03-02T10:00:00Z'), + tags: ['nature', 'sunset'], + linkedPostIds: ['p1', 'p2'], + availableLanguages: ['en', 'de'], + }; + const serialized = serializeMediaItem(media); + expect(serialized.createdAt).toBe('2025-03-01T10:00:00.000Z'); + expect(serialized.updatedAt).toBe('2025-03-02T10:00:00.000Z'); + expect(serialized.mimeType).toBe('image/webp'); + expect(serialized.size).toBe(54321); + expect(serialized.width).toBe(1920); + expect(serialized.title).toBe('Sunset'); + expect(serialized.tags).toEqual(['nature', 'sunset']); + expect(serialized.linkedPostIds).toEqual(['p1', 'p2']); + + const deserialized = deserializeMediaItem(serialized); + expect(deserialized.createdAt).toBeInstanceOf(Date); + expect(deserialized.updatedAt).toBeInstanceOf(Date); + expect(deserialized.filename).toBe('photo.webp'); + expect(deserialized.originalName).toBe('My Photo.webp'); + expect(deserialized.mimeType).toBe('image/webp'); + expect(deserialized.size).toBe(54321); + expect(deserialized.width).toBe(1920); + expect(deserialized.height).toBe(1080); + expect(deserialized.title).toBe('Sunset'); + expect(deserialized.alt).toBe('A beautiful sunset'); + expect(deserialized.caption).toBe('Taken at the beach'); + expect(deserialized.author).toBe('Bob'); + expect(deserialized.language).toBe('en'); + expect(deserialized.tags).toEqual(['nature', 'sunset']); + expect(deserialized.linkedPostIds).toEqual(['p1', 'p2']); + expect(deserialized.availableLanguages).toEqual(['en', 'de']); + }); + + it('round-trips a media item with minimal fields', () => { + const media = { + id: 'm2', + filename: 'doc.pdf', + originalName: 'Document.pdf', + mimeType: 'application/pdf', + size: 999, + createdAt: new Date('2025-04-01T00:00:00Z'), + updatedAt: new Date('2025-04-01T00:00:00Z'), + tags: [], + availableLanguages: [], + }; + const deserialized = deserializeMediaItem(serializeMediaItem(media)); + expect(deserialized.mimeType).toBe('application/pdf'); + expect(deserialized.width).toBeUndefined(); + expect(deserialized.title).toBeUndefined(); + expect(deserialized.linkedPostIds).toBeUndefined(); + expect(deserialized.tags).toEqual([]); + }); +}); + +describe('BlogGenerationOptions serialization', () => { + it('strips fields not needed by worker', () => { + const serialized = serializeBlogGenerationOptions({ + projectId: 'p1', + projectName: 'My Blog', + projectDescription: 'A blog', + dataDir: '/data', + baseUrl: 'https://example.com', + language: 'en', + blogLanguages: ['en', 'fr'], + pageTitle: 'My Blog', + maxPostsPerPage: 50, + picoTheme: undefined, + sections: ['single'], + }); + + expect(serialized.projectId).toBe('p1'); + expect(serialized.baseUrl).toBe('https://example.com'); + expect(serialized.blogLanguages).toEqual(['en', 'fr']); + // pageTitle, maxPostsPerPage, sections are not in serialized + expect((serialized as any).sections).toBeUndefined(); + }); +}); + +describe('PostMap serialization', () => { + it('round-trips a Map', () => { + const post1 = makePost({ id: '1', slug: 'a' }); + const post2 = makePost({ id: '2', slug: 'b' }); + const map = new Map([ + ['tag-js', [post1, post2]], + ['tag-py', [post2]], + ]); + + const serialized = serializePostMap(map); + expect(serialized).toHaveLength(2); + expect(serialized[0][0]).toBe('tag-js'); + expect(serialized[0][1]).toHaveLength(2); + + const deserialized = deserializePostMap(serialized); + expect(deserialized.size).toBe(2); + expect(deserialized.get('tag-js')).toHaveLength(2); + expect(deserialized.get('tag-js')![0].createdAt).toBeInstanceOf(Date); + }); + + it('round-trips a Map', () => { + const post1 = makePost({ id: '1', slug: 'a' }); + const map = new Map([ + [2025, [post1]], + ]); + + const serialized = serializePostMap(map); + const deserialized = deserializePostMap(serialized); + expect(deserialized.get(2025)).toHaveLength(1); + }); +}); + +describe('DateMap serialization', () => { + it('round-trips a Map', () => { + const map = new Map([ + [2024, new Date('2024-01-01')], + [2025, new Date('2025-01-01')], + ]); + + const serialized = serializeDateMap(map); + expect(serialized).toHaveLength(2); + expect(typeof serialized[0][1]).toBe('string'); + + const deserialized = deserializeDateMap(serialized); + expect(deserialized.size).toBe(2); + expect(deserialized.get(2024)).toBeInstanceOf(Date); + expect(deserialized.get(2025)).toBeInstanceOf(Date); + }); + + it('round-trips a Map', () => { + const map = new Map([ + ['2025/01', new Date('2025-01-15')], + ]); + + const serialized = serializeDateMap(map); + const deserialized = deserializeDateMap(serialized); + expect(deserialized.get('2025/01')).toBeInstanceOf(Date); + }); +}); diff --git a/tests/engine/GenerationWorkerPool.test.ts b/tests/engine/GenerationWorkerPool.test.ts new file mode 100644 index 0000000..fb37666 --- /dev/null +++ b/tests/engine/GenerationWorkerPool.test.ts @@ -0,0 +1,274 @@ +import { describe, expect, it, vi } from 'vitest'; +import { GenerationWorkerPool, type WorkerLike, type WorkerFactory } from '../../src/main/engine/GenerationWorkerPool'; +import type { GenerationWorkerTask, WorkerOutboundMessage } from '../../src/main/engine/GenerationWorkerData'; + +function makeTask(taskId: string, section: 'single' | 'category' | 'tag' | 'date' = 'single'): GenerationWorkerTask { + return { + taskId, + section, + posts: [], + lookupPosts: [], + mediaItems: [], + backlinksMap: {}, + options: { + projectId: 'proj-1', + projectName: 'Test Blog', + dataDir: '/data', + baseUrl: 'https://example.com', + }, + maxPostsPerPage: 50, + htmlDir: '/data/html', + hashMapEntries: [], + postFilePathEntries: [], + }; +} + +function createMockWorkerFactory( + responses: Map, +): WorkerFactory { + return (_workerPath: string, workerData: GenerationWorkerTask): WorkerLike => { + const listeners = new Map void>>(); + + const worker: WorkerLike = { + on(event: string, listener: (...args: unknown[]) => void) { + const existing = listeners.get(event) ?? []; + existing.push(listener); + listeners.set(event, existing); + }, + async terminate() { + return 0; + }, + removeAllListeners() { + listeners.clear(); + }, + }; + + // Simulate async message delivery + setTimeout(() => { + const taskMessages = responses.get(workerData.taskId) ?? [ + { type: 'result', taskId: workerData.taskId, pagesGenerated: 0 }, + ]; + for (const msg of taskMessages) { + const messageListeners = listeners.get('message') ?? []; + for (const listener of messageListeners) { + listener(msg); + } + } + }, 1); + + return worker; + }; +} + +describe('GenerationWorkerPool', () => { + it('returns zero pages for empty task list', async () => { + const pool = new GenerationWorkerPool({ maxWorkers: 2 }); + const result = await pool.runTasks([], vi.fn()); + expect(result.pagesGenerated).toBe(0); + expect(result.errors).toHaveLength(0); + }); + + it('runs a single task and reports result', async () => { + const responses = new Map([ + ['task-1', [{ type: 'result', taskId: 'task-1', pagesGenerated: 42 }]], + ]); + + const pool = new GenerationWorkerPool( + { maxWorkers: 2 }, + createMockWorkerFactory(responses), + ); + + const progress = vi.fn(); + const result = await pool.runTasks([makeTask('task-1')], progress); + + expect(result.pagesGenerated).toBe(42); + expect(result.errors).toHaveLength(0); + }); + + it('merges results from multiple tasks', async () => { + const responses = new Map([ + ['task-1', [{ type: 'result', taskId: 'task-1', pagesGenerated: 10 }]], + ['task-2', [{ type: 'result', taskId: 'task-2', pagesGenerated: 20 }]], + ['task-3', [{ type: 'result', taskId: 'task-3', pagesGenerated: 30 }]], + ]); + + const pool = new GenerationWorkerPool( + { maxWorkers: 2 }, + createMockWorkerFactory(responses), + ); + + const result = await pool.runTasks( + [makeTask('task-1'), makeTask('task-2'), makeTask('task-3')], + vi.fn(), + ); + + expect(result.pagesGenerated).toBe(60); + }); + + it('collects progress messages', async () => { + const responses = new Map([ + ['task-1', [ + { type: 'progress', taskId: 'task-1', message: 'Page 1' }, + { type: 'progress', taskId: 'task-1', message: 'Page 2' }, + { type: 'result', taskId: 'task-1', pagesGenerated: 2 }, + ]], + ]); + + const pool = new GenerationWorkerPool( + { maxWorkers: 1 }, + createMockWorkerFactory(responses), + ); + + const progress = vi.fn(); + await pool.runTasks([makeTask('task-1')], progress); + + expect(progress).toHaveBeenCalledWith('Page 1'); + expect(progress).toHaveBeenCalledWith('Page 2'); + }); + + it('collects errors from failed tasks', async () => { + const responses = new Map([ + ['task-1', [{ type: 'error', taskId: 'task-1', error: 'Render failed' }]], + ['task-2', [{ type: 'result', taskId: 'task-2', pagesGenerated: 5 }]], + ]); + + const pool = new GenerationWorkerPool( + { maxWorkers: 2 }, + createMockWorkerFactory(responses), + ); + + const result = await pool.runTasks( + [makeTask('task-1'), makeTask('task-2')], + vi.fn(), + ); + + expect(result.pagesGenerated).toBe(5); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].taskId).toBe('task-1'); + expect(result.errors[0].error).toBe('Render failed'); + }); + + it('handles worker crash via error event', async () => { + const factory: WorkerFactory = (_workerPath, workerData): WorkerLike => { + const listeners = new Map void>>(); + + const worker: WorkerLike = { + on(event: string, listener: (...args: unknown[]) => void) { + const existing = listeners.get(event) ?? []; + existing.push(listener); + listeners.set(event, existing); + }, + async terminate() { + return 1; + }, + removeAllListeners() { + listeners.clear(); + }, + }; + + setTimeout(() => { + const errorListeners = listeners.get('error') ?? []; + for (const listener of errorListeners) { + listener(new Error('Worker crashed')); + } + }, 1); + + return worker; + }; + + const pool = new GenerationWorkerPool({ maxWorkers: 1 }, factory); + const result = await pool.runTasks([makeTask('crash-task')], vi.fn()); + + expect(result.errors).toHaveLength(1); + expect(result.errors[0].error).toBe('Worker crashed'); + }); + + it('collects hashUpdates from worker results', async () => { + const responses = new Map([ + ['task-1', [{ type: 'result', taskId: 'task-1', pagesGenerated: 3, hashUpdates: [ + { relativePath: 'index.html', hash: 'aaa' }, + { relativePath: 'page/2/index.html', hash: 'bbb' }, + ] }]], + ['task-2', [{ type: 'result', taskId: 'task-2', pagesGenerated: 2, hashUpdates: [ + { relativePath: 'tags/index.html', hash: 'ccc' }, + ] }]], + ]); + + const pool = new GenerationWorkerPool( + { maxWorkers: 2 }, + createMockWorkerFactory(responses), + ); + + const result = await pool.runTasks( + [makeTask('task-1'), makeTask('task-2')], + vi.fn(), + ); + + expect(result.pagesGenerated).toBe(5); + expect(result.hashUpdates).toHaveLength(3); + expect(result.hashUpdates).toContainEqual({ relativePath: 'index.html', hash: 'aaa' }); + expect(result.hashUpdates).toContainEqual({ relativePath: 'page/2/index.html', hash: 'bbb' }); + expect(result.hashUpdates).toContainEqual({ relativePath: 'tags/index.html', hash: 'ccc' }); + }); + + it('returns empty hashUpdates when workers report errors', async () => { + const responses = new Map([ + ['task-1', [{ type: 'error', taskId: 'task-1', error: 'boom' }]], + ]); + + const pool = new GenerationWorkerPool( + { maxWorkers: 1 }, + createMockWorkerFactory(responses), + ); + + const result = await pool.runTasks([makeTask('task-1')], vi.fn()); + + expect(result.hashUpdates).toHaveLength(0); + expect(result.errors).toHaveLength(1); + }); + + it('respects maxWorkers concurrency', async () => { + let peakConcurrent = 0; + let currentConcurrent = 0; + + const factory: WorkerFactory = (_workerPath, workerData): WorkerLike => { + currentConcurrent++; + if (currentConcurrent > peakConcurrent) peakConcurrent = currentConcurrent; + + const listeners = new Map void>>(); + + const worker: WorkerLike = { + on(event: string, listener: (...args: unknown[]) => void) { + const existing = listeners.get(event) ?? []; + existing.push(listener); + listeners.set(event, existing); + }, + async terminate() { + currentConcurrent--; + return 0; + }, + removeAllListeners() { + listeners.clear(); + }, + }; + + setTimeout(() => { + const messageListeners = listeners.get('message') ?? []; + for (const listener of messageListeners) { + listener({ type: 'result', taskId: workerData.taskId, pagesGenerated: 1 }); + } + }, 5); + + return worker; + }; + + const pool = new GenerationWorkerPool({ maxWorkers: 2 }, factory); + const result = await pool.runTasks( + [makeTask('t1'), makeTask('t2'), makeTask('t3'), makeTask('t4')], + vi.fn(), + ); + + expect(result.pagesGenerated).toBe(4); + expect(peakConcurrent).toBeLessThanOrEqual(2); + }); +});