Feature/worker threads generation (#43)

* Add worker threads architecture plan for blog generation

* fix: tries to optimize rendering, still slow

* feat: moved site rendering into web worker

* fix: calendar grabs from central data source for calendar

* fix: feeds now use blog language content and not canonical content

---------

Co-authored-by: hugo <hugoms@me.com>
This commit is contained in:
Georg Bauer
2026-03-09 22:49:25 +01:00
committed by GitHub
parent b855d61524
commit 4f9be93c6d
42 changed files with 3617 additions and 346 deletions

View File

@@ -66,3 +66,27 @@ export async function setGeneratedFileHash(projectId: string, relativePath: stri
args: [projectId, relativePath, hash, Date.now()],
});
}
/**
* Bulk-load all file hashes for a project in a single query.
* Returns a Map from relativePath → contentHash.
*/
export async function getAllGeneratedFileHashes(projectId: string): Promise<Map<string, string>> {
const client = getDatabase().getLocalClient();
if (!client) {
throw new Error('Database client not available');
}
const result = await client.execute({
sql: 'SELECT relative_path, content_hash FROM generated_file_hashes WHERE project_id = ?',
args: [projectId],
});
const map = new Map<string, string>();
for (const row of result.rows) {
if (typeof row.relative_path === 'string' && typeof row.content_hash === 'string') {
map.set(row.relative_path, row.content_hash);
}
}
return map;
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,250 @@
/**
* Lightweight, in-memory engine implementations for use in worker threads.
*
* These replace the real PostEngine / MediaEngine / PostMediaEngine when running
* inside a generation worker. They are backed entirely by pre-loaded data arrays
* so no database access is needed for post/media queries.
*/
import type { PostData, PostFilter, PostTranslationData } from './PostEngine';
import type { MediaData } from './MediaEngine';
import { readPostFile } from './postFileUtils';
import { readPostTranslationFile } from './postTranslationFileUtils';
// ---------------------------------------------------------------------------
// DataBackedPostEngine
// ---------------------------------------------------------------------------
export interface DataBackedPostEngineInit {
/** All posts (published snapshots + translation variants). */
allPosts: PostData[];
/** Pre-resolved backlinks: postId → linking posts. */
backlinksMap?: Map<string, Array<{ id: string; title: string; slug: string }>>;
/** Post file paths for lazy content loading from filesystem: postId → absoluteFilePath. */
postFilePaths?: Map<string, string>;
}
export interface DataBackedPostEngineContract {
getPostsFiltered: (filter: PostFilter) => Promise<PostData[]>;
getPublishedVersion: (id: string) => Promise<PostData | null>;
getPost: (id: string) => Promise<PostData | null>;
hasPublishedVersion: (id: string) => Promise<boolean>;
findPublishedBySlug: (slug: string, dateFilter?: { year: number; month: number }) => Promise<PostData | null>;
getLinkedBy: (postId: string) => Promise<Array<{ id: string; title: string; slug: string }>>;
getAllBacklinks: () => Promise<Map<string, Array<{ id: string; title: string; slug: string }>>>;
getPostTranslation: (postId: string, language: string) => Promise<PostTranslationData | null>;
getPostTranslations: (postId: string) => Promise<PostTranslationData[]>;
setProjectContext: (projectId: string, dataDir?: string) => void;
}
export function createDataBackedPostEngine(init: DataBackedPostEngineInit): DataBackedPostEngineContract {
const { allPosts, backlinksMap, postFilePaths } = init;
// Build indexes for fast lookups
const byId = new Map<string, PostData>();
const bySlug = new Map<string, PostData[]>();
for (const post of allPosts) {
byId.set(post.id, post);
const slugEntries = bySlug.get(post.slug);
if (slugEntries) {
slugEntries.push(post);
} else {
bySlug.set(post.slug, [post]);
}
}
function matchesFilter(post: PostData, filter: PostFilter): boolean {
if (filter.status && post.status !== filter.status) return false;
if (filter.excludeCategories && filter.excludeCategories.length > 0) {
const excluded = new Set(filter.excludeCategories);
if (post.categories.some((c) => excluded.has(c))) return false;
}
if (filter.categories && filter.categories.length > 0) {
if (!filter.categories.some((c) => post.categories.includes(c))) return false;
}
if (filter.tags && filter.tags.length > 0) {
if (!filter.tags.every((t) => post.tags.includes(t))) return false;
}
if (filter.language && post.language !== filter.language) return false;
if (filter.year !== undefined) {
if (post.createdAt.getFullYear() !== filter.year) return false;
}
if (filter.month !== undefined) {
if (post.createdAt.getMonth() + 1 !== filter.month) return false;
}
if (filter.startDate) {
if (post.createdAt < filter.startDate) return false;
}
if (filter.endDate) {
if (post.createdAt > filter.endDate) return false;
}
return true;
}
// Shared lazy content loader for posts with empty content
async function lazyLoadContent(post: PostData): Promise<void> {
if (post.content || !postFilePaths) return;
const variant = post as PostData & { translationFilePath?: string };
if (variant.translationFilePath) {
const fileData = await readPostTranslationFile(variant.translationFilePath);
if (fileData) {
post.content = fileData.content;
}
} else {
const filePath = postFilePaths.get(post.id);
if (filePath) {
const fileData = await readPostFile(filePath);
if (fileData) {
post.content = fileData.content;
}
}
}
}
return {
async getPostsFiltered(filter: PostFilter): Promise<PostData[]> {
const filtered = allPosts
.filter((post) => {
const tss = (post as any).translationSourceSlug;
// Keep canonical posts and resolved posts (slug === tss).
// Exclude translation variant route posts (slug !== tss, e.g. "my-post.en").
return !tss || post.slug === tss;
})
.filter((post) => matchesFilter(post, filter))
.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
// Lazy-load content for posts that need it (e.g. resolved translation posts
// have content: '' and need their translation file content loaded).
await Promise.all(filtered.map((post) => lazyLoadContent(post)));
return filtered;
},
async getPublishedVersion(id: string): Promise<PostData | null> {
const post = byId.get(id);
if (!post) return null;
await lazyLoadContent(post);
return post;
},
async getPost(id: string): Promise<PostData | null> {
const post = byId.get(id) ?? null;
if (post) {
await lazyLoadContent(post);
}
return post;
},
async hasPublishedVersion(id: string): Promise<boolean> {
return byId.has(id);
},
async findPublishedBySlug(slug: string, dateFilter?: { year: number; month: number }): Promise<PostData | null> {
const candidates = bySlug.get(slug);
if (!candidates || candidates.length === 0) return null;
if (!dateFilter) return candidates[0];
return candidates.find((p) =>
p.createdAt.getFullYear() === dateFilter.year
&& p.createdAt.getMonth() === dateFilter.month - 1,
) ?? null;
},
async getLinkedBy(postId: string): Promise<Array<{ id: string; title: string; slug: string }>> {
return backlinksMap?.get(postId) ?? [];
},
async getAllBacklinks(): Promise<Map<string, Array<{ id: string; title: string; slug: string }>>> {
return backlinksMap ?? new Map();
},
async getPostTranslation(_postId: string, _language: string): Promise<PostTranslationData | null> {
// Translation variants are already included as separate route posts
return null;
},
async getPostTranslations(_postId: string): Promise<PostTranslationData[]> {
return [];
},
setProjectContext(_projectId: string, _dataDir?: string): void {
// No-op — data is already loaded
},
};
}
// ---------------------------------------------------------------------------
// DataBackedMediaEngine
// ---------------------------------------------------------------------------
export interface DataBackedMediaEngineContract {
getAllMedia: () => Promise<MediaData[]>;
setProjectContext: (projectId: string, dataDir?: string, internalDir?: string) => void;
}
export function createDataBackedMediaEngine(
mediaItems: MediaData[],
): DataBackedMediaEngineContract {
return {
async getAllMedia() {
return mediaItems;
},
setProjectContext(_projectId: string, _dataDir?: string, _internalDir?: string): void {
// No-op
},
};
}
// ---------------------------------------------------------------------------
// DataBackedPostMediaEngine
// ---------------------------------------------------------------------------
export interface DataBackedPostMediaEngineInit {
mediaItems: MediaData[];
postMediaLinks: Map<string, Array<{ mediaId: string; sortOrder: number }>>;
}
export interface DataBackedPostMediaEngineContract {
setProjectContext: (projectId: string) => void;
getLinkedMediaForPost: (postId: string) => Promise<Array<{ mediaId: string; sortOrder: number }>>;
getLinkedMediaDataForPost: (postId: string) => Promise<Array<{ media: MediaData }>>;
}
export function createDataBackedPostMediaEngine(init: DataBackedPostMediaEngineInit): DataBackedPostMediaEngineContract {
const { mediaItems, postMediaLinks } = init;
const mediaById = new Map<string, MediaData>();
for (const m of mediaItems) {
mediaById.set(m.id, m);
}
return {
setProjectContext(_projectId: string): void {
// No-op
},
async getLinkedMediaForPost(postId: string): Promise<Array<{ mediaId: string; sortOrder: number }>> {
return postMediaLinks.get(postId) ?? [];
},
async getLinkedMediaDataForPost(postId: string): Promise<Array<{ media: MediaData }>> {
const links = postMediaLinks.get(postId) ?? [];
const result: Array<{ media: MediaData }> = [];
for (const link of links) {
const media = mediaById.get(link.mediaId);
if (media) {
result.push({ media });
}
}
return result;
},
};
}

View File

@@ -3,6 +3,7 @@ import type { PostData } from './PostEngine';
export interface GenerationSnapshotPostEngine {
getPostsFiltered: (filter: { status?: 'draft' | 'published' | 'archived'; excludeCategories?: string[] }) => Promise<PostData[]>;
getPublishedVersion: (id: string) => Promise<PostData | null>;
getPublishedVersionsBulk?: (ids: string[]) => Promise<Map<string, PostData>>;
}
export interface GenerationPublishedSets {
@@ -10,57 +11,64 @@ export interface GenerationPublishedSets {
publishedListPosts: PostData[];
}
async function resolvePublishedVersions(
postEngine: GenerationSnapshotPostEngine,
ids: string[],
): Promise<Map<string, PostData>> {
if (ids.length === 0) return new Map();
if (postEngine.getPublishedVersionsBulk) {
return postEngine.getPublishedVersionsBulk(ids);
}
const result = new Map<string, PostData>();
const entries = await Promise.all(
ids.map(async (id) => {
const version = await postEngine.getPublishedVersion(id);
return { id, version };
}),
);
for (const { id, version } of entries) {
if (version) result.set(id, version);
}
return result;
}
export async function loadPublishedGenerationSets(
postEngine: GenerationSnapshotPostEngine,
listExcludedCategories: string[],
): Promise<GenerationPublishedSets> {
const publishedCandidates = await postEngine.getPostsFiltered({ status: 'published' });
const draftCandidates = await postEngine.getPostsFiltered({ status: 'draft' });
const publishedListCandidates = await postEngine.getPostsFiltered({
status: 'published',
excludeCategories: listExcludedCategories,
});
const draftListCandidates = await postEngine.getPostsFiltered({
status: 'draft',
excludeCategories: listExcludedCategories,
});
const publishedSnapshots = await Promise.all(
publishedCandidates.map(async (post) => {
const snapshot = await postEngine.getPublishedVersion(post.id);
return snapshot || post;
}),
);
const draftPublishedSnapshots = await Promise.all(
draftCandidates.map(async (post) => postEngine.getPublishedVersion(post.id)),
);
const publishedListSnapshots = await Promise.all(
publishedListCandidates.map(async (post) => {
const snapshot = await postEngine.getPublishedVersion(post.id);
return snapshot || post;
}),
);
const draftListPublishedSnapshots = await Promise.all(
draftListCandidates.map(async (post) => postEngine.getPublishedVersion(post.id)),
);
const allIds = new Set<string>();
for (const p of publishedCandidates) allIds.add(p.id);
for (const p of draftCandidates) allIds.add(p.id);
const publishedVersions = await resolvePublishedVersions(postEngine, Array.from(allIds));
const excludedCategorySet = new Set(listExcludedCategories);
const isListExcluded = (post: PostData) =>
excludedCategorySet.size > 0 && post.categories.some((c) => excludedCategorySet.has(c));
const publishedPostById = new Map<string, PostData>();
for (const post of publishedSnapshots) {
publishedPostById.set(post.id, post);
}
for (const snapshot of draftPublishedSnapshots) {
if (snapshot) {
publishedPostById.set(snapshot.id, snapshot);
const publishedListPostById = new Map<string, PostData>();
for (const post of publishedCandidates) {
const snapshot = publishedVersions.get(post.id) || post;
publishedPostById.set(post.id, snapshot);
if (!isListExcluded(post)) {
publishedListPostById.set(post.id, snapshot);
}
}
const publishedListPostById = new Map<string, PostData>();
for (const post of publishedListSnapshots) {
publishedListPostById.set(post.id, post);
}
for (const snapshot of draftListPublishedSnapshots) {
for (const post of draftCandidates) {
const snapshot = publishedVersions.get(post.id);
if (snapshot) {
publishedListPostById.set(snapshot.id, snapshot);
publishedPostById.set(post.id, snapshot);
if (!isListExcluded(post)) {
publishedListPostById.set(post.id, snapshot);
}
}
}

View File

@@ -1,6 +1,6 @@
import path from 'node:path';
import type { CategoryRenderSettings } from './PageRenderer';
import { buildCanonicalPostPath } from './PageRenderer';
import type { CategoryRenderSettings, HtmlRewriteContext } from './PageRenderer';
import { buildCanonicalPostPath, mapToRecord } from './PageRenderer';
import type { MenuDocument } from './MenuEngine';
import type { ProjectMetadata } from './MetaEngine';
import type { PostData } from './PostEngine';
@@ -8,6 +8,7 @@ import type { PicoThemeName } from '../shared/picoThemes';
import type { CategoryMetadata } from './BlogGenerationEngine';
import { PreviewServer } from './PreviewServer';
import type { PostTranslationData } from './PostEngine';
import { readPostTranslationFile } from './postTranslationFileUtils';
interface RenderContext {
projectContext: {
@@ -49,11 +50,14 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
projectName: string;
projectDescription?: string;
language?: string;
blogLanguages?: string[];
picoTheme?: PicoThemeName;
categoryMetadata?: Record<string, CategoryMetadata>;
categorySettings?: Record<string, CategoryRenderSettings>;
menu?: MenuDocument;
};
/** The project's actual main language (for href_prefix computation). Defaults to options.language. */
projectMainLanguage?: string;
maxPostsPerPage: number;
publishedPostsForLookup: PostData[];
languagePrefix?: string;
@@ -66,6 +70,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
getPostTranslation?: (postId: string, language: string) => Promise<PostTranslationData | null>;
hasPublishedVersion: (postId: string) => Promise<boolean>;
getLinkedBy?: (postId: string) => Promise<{ id: string; title: string; slug: string }[]>;
getAllBacklinks?: () => Promise<Map<string, { id: string; title: string; slug: string }[]>>;
setProjectContext: (projectId: string, dataDir?: string) => void;
};
mediaEngine: {
@@ -79,10 +84,13 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
};
};
}): (pathname: string) => Promise<string | null> {
const projectMainLanguage = params.projectMainLanguage ?? params.options.language;
const metadata: ProjectMetadata = {
name: params.options.projectName,
description: params.options.projectDescription,
mainLanguage: params.options.language,
mainLanguage: projectMainLanguage,
blogLanguages: params.options.blogLanguages,
maxPostsPerPage: params.maxPostsPerPage,
picoTheme: params.options.picoTheme,
categoryMetadata: params.options.categoryMetadata,
@@ -166,26 +174,53 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
return null;
}
let match: PostData | undefined;
if (!dateFilter) {
return candidates[0] ?? null;
match = candidates[0];
} else {
match = candidates.find((candidate) => {
const createdAt = candidate.createdAt;
return createdAt.getFullYear() === dateFilter.year
&& createdAt.getMonth() === dateFilter.month - 1;
});
}
const match = candidates.find((candidate) => {
const createdAt = candidate.createdAt;
return createdAt.getFullYear() === dateFilter.year
&& createdAt.getMonth() === dateFilter.month - 1;
});
if (!match) return null;
return match ?? null;
// Lazily resolve content from file when needed
if (!match.content) {
const variant = match as PostData & { translationFilePath?: string };
if (variant.translationFilePath) {
const fileData = await readPostTranslationFile(variant.translationFilePath);
if (fileData) {
match.content = fileData.content;
}
} else {
const full = await cachedPostEngine.getPublishedVersion(match.id);
if (full) {
match.content = full.content;
}
}
}
return match;
},
getPost: (postId: string) => params.engines.postEngine.getPost(postId),
getPostTranslation: params.engines.postEngine.getPostTranslation
? (postId: string, language: string) => params.engines.postEngine.getPostTranslation!(postId, language)
: undefined,
hasPublishedVersion: (postId: string) => params.engines.postEngine.hasPublishedVersion(postId),
getLinkedBy: params.engines.postEngine.getLinkedBy
? (postId: string) => params.engines.postEngine.getLinkedBy!(postId)
: undefined,
getLinkedBy: params.engines.postEngine.getAllBacklinks
? (() => {
const backlinksCachePromise = params.engines.postEngine.getAllBacklinks!();
return async (postId: string) => {
const backlinksMap = await backlinksCachePromise;
return backlinksMap.get(postId) ?? [];
};
})()
: params.engines.postEngine.getLinkedBy
? (postId: string) => params.engines.postEngine.getLinkedBy!(postId)
: undefined,
setProjectContext: (projectId: string, dataDir?: string) => {
params.engines.postEngine.setProjectContext(projectId, dataDir);
},
@@ -224,7 +259,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
userTemplatesDir: path.join(params.options.dataDir, 'templates'),
});
const htmlRewriteContextPromise: Promise<{ canonicalPostPathBySlug: Map<string, string>; canonicalMediaPathBySourcePath: Map<string, string>; languagePrefix?: string }> = (async () => {
const htmlRewriteContextPromise: Promise<HtmlRewriteContext> = (async () => {
const canonicalPostPathBySlug = new Map<string, string>();
for (const post of params.publishedPostsForLookup) {
canonicalPostPathBySlug.set(post.slug, buildCanonicalPostPath(post));
@@ -247,6 +282,8 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
return {
canonicalPostPathBySlug,
canonicalMediaPathBySourcePath,
canonicalPostPathBySlugRecord: mapToRecord(canonicalPostPathBySlug),
canonicalMediaPathBySourcePathRecord: mapToRecord(canonicalMediaPathBySourcePath),
languagePrefix: params.languagePrefix,
};
})();
@@ -255,6 +292,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
renderWithContext: async (pathname, context) => previewServer.renderRouteForContext(pathname, {
...context,
htmlRewriteContext: await htmlRewriteContextPromise,
preferredLanguage: params.options.language,
}),
context: {
projectContext,

View File

@@ -0,0 +1,327 @@
/**
* Serialization types and utilities for passing data to generation worker threads.
*
* Worker threads cannot receive non-serializable objects (Date, Map, etc.) via
* the structured-clone algorithm used by workerData / postMessage. This module
* provides a thin serialization layer that converts Dates to ISO strings and
* Maps to arrays-of-tuples so the data survives the boundary.
*/
import type { PostData } from './PostEngine';
import type { MediaData } from './MediaEngine';
import type { CategoryMetadata, BlogGenerationOptions, BlogGenerationSection } from './BlogGenerationEngine';
import type { CategoryRenderSettings } from './PageRenderer';
import type { MenuDocument } from './MenuEngine';
import type { PicoThemeName } from '../shared/picoThemes';
// ---------------------------------------------------------------------------
// Serialized post (Dates → ISO strings)
// ---------------------------------------------------------------------------
export interface SerializedPostData {
id: string;
projectId: string;
title: string;
slug: string;
excerpt?: string;
content: string;
status: 'draft' | 'published' | 'archived';
author?: string;
language?: string;
doNotTranslate?: boolean;
templateSlug?: string;
createdAt: string;
updatedAt: string;
publishedAt?: string;
tags: string[];
categories: string[];
availableLanguages: string[];
translationSourceSlug?: string;
translationCanonicalLanguage?: string;
translationFilePath?: string;
/** Absolute file path for canonical posts (for lazy content loading in workers). */
filePath?: string;
}
// ---------------------------------------------------------------------------
// Serialized media item (only what generation needs)
// ---------------------------------------------------------------------------
export interface SerializedMediaData {
id: string;
filename: string;
originalName: string;
mimeType: string;
size: number;
width?: number;
height?: number;
title?: string;
alt?: string;
caption?: string;
author?: string;
language?: string;
createdAt: string;
updatedAt: string;
tags: string[];
linkedPostIds?: string[];
availableLanguages: string[];
}
// ---------------------------------------------------------------------------
// Serialized options (stripped to what the worker actually uses)
// ---------------------------------------------------------------------------
export interface SerializedBlogGenerationOptions {
projectId: string;
projectName: string;
projectDescription?: string;
dataDir: string;
baseUrl: string;
language?: string;
blogLanguages?: string[];
picoTheme?: PicoThemeName;
categoryMetadata?: Record<string, CategoryMetadata>;
categorySettings?: Record<string, CategoryRenderSettings>;
menu?: MenuDocument;
}
// ---------------------------------------------------------------------------
// Worker task — everything a worker needs to process one section
// ---------------------------------------------------------------------------
export interface GenerationWorkerTask {
taskId: string;
section: BlogGenerationSection;
/** Posts relevant to this task (single: chunk, others: all list posts). */
posts: SerializedPostData[];
/**
* All published route posts — used by the route renderer for slug lookups,
* canonical path building, and backlink resolution.
*/
lookupPosts: SerializedPostData[];
/** Media items for canonical media path building. */
mediaItems: SerializedMediaData[];
/** Pre-resolved backlinks map: postId → linked-by entries. */
backlinksMap: Record<string, Array<{ id: string; title: string; slug: string }>>;
options: SerializedBlogGenerationOptions;
maxPostsPerPage: number;
htmlDir: string;
/** Pre-loaded hash map as [relativePath, contentHash] tuples. */
hashMapEntries: Array<[string, string]>;
/** Post file paths for lazy content loading: [postId, absoluteFilePath] tuples. */
postFilePathEntries: Array<[string, string]>;
/** Post-media links: [postId, [{mediaId, sortOrder}]] tuples for gallery/album macros. */
postMediaLinksEntries: Array<[string, Array<{ mediaId: string; sortOrder: number }>]>;
/** Language prefix for subtree generation, e.g. "/fr". */
languagePrefix?: string;
/** The project's actual main language (for href_prefix computation in subtree rendering). */
mainLanguage?: string;
// -- Section-specific data (category / tag / date) -----------------------
allCategories?: string[];
allTags?: string[];
/** Serialized Maps as arrays of tuples. */
yearsEntries?: Array<[number, string]>;
yearMonthsEntries?: Array<[string, string]>;
yearMonthDaysEntries?: Array<[string, string]>;
/** Pre-built post index (avoids rebuilding in each worker). */
postsByCategoryEntries?: Array<[string, SerializedPostData[]]>;
postsByTagEntries?: Array<[string, SerializedPostData[]]>;
postsByYearEntries?: Array<[number, SerializedPostData[]]>;
postsByYearMonthEntries?: Array<[string, SerializedPostData[]]>;
postsByYearMonthDayEntries?: Array<[string, SerializedPostData[]]>;
}
// ---------------------------------------------------------------------------
// Messages between main thread and worker
// ---------------------------------------------------------------------------
export interface WorkerProgressMessage {
type: 'progress';
taskId: string;
message: string;
}
export interface WorkerResultMessage {
type: 'result';
taskId: string;
pagesGenerated: number;
/** Hash updates accumulated during generation, to be written by the main thread. */
hashUpdates: Array<{ relativePath: string; hash: string }>;
}
export interface WorkerErrorMessage {
type: 'error';
taskId: string;
error: string;
}
export type WorkerOutboundMessage =
| WorkerProgressMessage
| WorkerResultMessage
| WorkerErrorMessage;
// ---------------------------------------------------------------------------
// Serialization helpers
// ---------------------------------------------------------------------------
export function serializePostData(post: PostData): SerializedPostData {
const serialized: SerializedPostData = {
id: post.id,
projectId: post.projectId,
title: post.title,
slug: post.slug,
excerpt: post.excerpt,
content: post.content ?? '',
status: post.status,
author: post.author,
language: post.language,
doNotTranslate: post.doNotTranslate,
templateSlug: post.templateSlug,
createdAt: post.createdAt instanceof Date ? post.createdAt.toISOString() : String(post.createdAt),
updatedAt: post.updatedAt instanceof Date ? post.updatedAt.toISOString() : String(post.updatedAt),
publishedAt: post.publishedAt instanceof Date ? post.publishedAt.toISOString() : post.publishedAt ? String(post.publishedAt) : undefined,
tags: post.tags ?? [],
categories: post.categories ?? [],
availableLanguages: post.availableLanguages ?? [],
};
// Preserve translation variant fields if present
const variant = post as PostData & {
translationSourceSlug?: string;
translationCanonicalLanguage?: string;
translationFilePath?: string;
};
if (variant.translationSourceSlug) serialized.translationSourceSlug = variant.translationSourceSlug;
if (variant.translationCanonicalLanguage) serialized.translationCanonicalLanguage = variant.translationCanonicalLanguage;
if (variant.translationFilePath) serialized.translationFilePath = variant.translationFilePath;
return serialized;
}
export function deserializePostData(serialized: SerializedPostData): PostData {
const post: PostData = {
id: serialized.id,
projectId: serialized.projectId,
title: serialized.title,
slug: serialized.slug,
excerpt: serialized.excerpt,
content: serialized.content ?? '',
status: serialized.status,
author: serialized.author,
language: serialized.language,
doNotTranslate: serialized.doNotTranslate,
templateSlug: serialized.templateSlug,
createdAt: new Date(serialized.createdAt),
updatedAt: new Date(serialized.updatedAt),
publishedAt: serialized.publishedAt ? new Date(serialized.publishedAt) : undefined,
tags: serialized.tags ?? [],
categories: serialized.categories ?? [],
availableLanguages: serialized.availableLanguages ?? [],
};
// Re-attach translation variant fields
if (serialized.translationSourceSlug) {
(post as any).translationSourceSlug = serialized.translationSourceSlug;
}
if (serialized.translationCanonicalLanguage) {
(post as any).translationCanonicalLanguage = serialized.translationCanonicalLanguage;
}
if (serialized.translationFilePath) {
(post as any).translationFilePath = serialized.translationFilePath;
}
return post;
}
export function serializeMediaItem(media: MediaData): SerializedMediaData {
return {
id: media.id,
filename: media.filename,
originalName: media.originalName,
mimeType: media.mimeType,
size: media.size,
width: media.width,
height: media.height,
title: media.title,
alt: media.alt,
caption: media.caption,
author: media.author,
language: media.language,
createdAt: media.createdAt instanceof Date ? media.createdAt.toISOString() : String(media.createdAt),
updatedAt: media.updatedAt instanceof Date ? media.updatedAt.toISOString() : String(media.updatedAt),
tags: media.tags ?? [],
linkedPostIds: media.linkedPostIds,
availableLanguages: media.availableLanguages ?? [],
};
}
export function deserializeMediaItem(serialized: SerializedMediaData): MediaData {
return {
id: serialized.id,
filename: serialized.filename,
originalName: serialized.originalName,
mimeType: serialized.mimeType,
size: serialized.size,
width: serialized.width,
height: serialized.height,
title: serialized.title,
alt: serialized.alt,
caption: serialized.caption,
author: serialized.author,
language: serialized.language,
createdAt: new Date(serialized.createdAt),
updatedAt: new Date(serialized.updatedAt),
tags: serialized.tags ?? [],
linkedPostIds: serialized.linkedPostIds,
availableLanguages: serialized.availableLanguages ?? [],
};
}
export function serializeBlogGenerationOptions(options: BlogGenerationOptions): SerializedBlogGenerationOptions {
return {
projectId: options.projectId,
projectName: options.projectName,
projectDescription: options.projectDescription,
dataDir: options.dataDir,
baseUrl: options.baseUrl,
language: options.language,
blogLanguages: options.blogLanguages,
picoTheme: options.picoTheme,
categoryMetadata: options.categoryMetadata,
categorySettings: options.categorySettings,
menu: options.menu,
};
}
/** Serialize a Map<K, PostData[]> to an array of [K, SerializedPostData[]] tuples. */
export function serializePostMap<K extends string | number>(map: Map<K, PostData[]>): Array<[K, SerializedPostData[]]> {
return Array.from(map.entries()).map(([key, posts]) => [key, posts.map(serializePostData)]);
}
/** Deserialize an array of [K, SerializedPostData[]] tuples to a Map<K, PostData[]>. */
export function deserializePostMap<K extends string | number>(entries: Array<[K, SerializedPostData[]]>): Map<K, PostData[]> {
return new Map(entries.map(([key, posts]) => [key, posts.map(deserializePostData)]));
}
/** Serialize a Map<K, Date> to an array of [K, string] tuples. */
export function serializeDateMap<K extends string | number>(map: Map<K, Date>): Array<[K, string]> {
return Array.from(map.entries()).map(([key, date]) => [key, date.toISOString()]);
}
/** Deserialize an array of [K, string] tuples to a Map<K, Date>. */
export function deserializeDateMap<K extends string | number>(entries: Array<[K, string]>): Map<K, Date> {
return new Map(entries.map(([key, iso]) => [key, new Date(iso)]));
}

View File

@@ -0,0 +1,139 @@
/**
* Worker pool for parallel blog generation.
*
* Manages a pool of worker threads that render HTML pages concurrently.
* Each worker gets a self-contained GenerationWorkerTask and produces pages
* independently. The pool limits concurrency to os.cpus().length - 1 (min 1).
*/
import { Worker } from 'worker_threads';
import * as os from 'os';
import * as path from 'path';
import type {
GenerationWorkerTask,
WorkerOutboundMessage,
} from './GenerationWorkerData';
export interface WorkerPoolOptions {
/** Max concurrent workers. Defaults to os.cpus().length - 1, min 1. */
maxWorkers?: number;
/** Override the worker script path (for testing). */
workerPath?: string;
}
export interface WorkerPoolResult {
pagesGenerated: number;
errors: Array<{ taskId: string; error: string }>;
hashUpdates: Array<{ relativePath: string; hash: string }>;
}
export type WorkerFactory = (workerPath: string, workerData: GenerationWorkerTask) => WorkerLike;
export interface WorkerLike {
on(event: string, listener: (...args: unknown[]) => void): void;
terminate(): Promise<number>;
removeAllListeners(): void;
}
export class GenerationWorkerPool {
private readonly maxWorkers: number;
private readonly workerPath: string;
private readonly workerFactory: WorkerFactory;
constructor(options?: WorkerPoolOptions, workerFactory?: WorkerFactory) {
this.maxWorkers = Math.max(1, options?.maxWorkers ?? (os.cpus().length - 1));
this.workerPath = options?.workerPath ?? path.join(__dirname, 'generation.worker.js');
this.workerFactory = workerFactory ?? ((wp, wd) => new Worker(wp, { workerData: wd }) as unknown as WorkerLike);
}
/**
* Run a set of generation tasks across the worker pool.
*
* Tasks are distributed to workers up to maxWorkers concurrency.
* When a worker finishes, the next queued task is dispatched.
*
* @param tasks - Array of self-contained worker tasks
* @param onProgress - Called for each page generated (for progress bar updates)
* @returns Merged results from all workers
*/
async runTasks(
tasks: GenerationWorkerTask[],
onProgress: (message: string) => void,
): Promise<WorkerPoolResult> {
if (tasks.length === 0) {
return { pagesGenerated: 0, errors: [], hashUpdates: [] };
}
return new Promise<WorkerPoolResult>((resolve) => {
let totalPages = 0;
const errors: Array<{ taskId: string; error: string }> = [];
const allHashUpdates: Array<{ relativePath: string; hash: string }> = [];
let nextTaskIndex = 0;
let activeWorkers = 0;
const startNextWorker = () => {
if (nextTaskIndex >= tasks.length) {
if (activeWorkers === 0) {
resolve({ pagesGenerated: totalPages, errors, hashUpdates: allHashUpdates });
}
return;
}
const task = tasks[nextTaskIndex++];
activeWorkers++;
const worker = this.workerFactory(this.workerPath, task);
worker.on('message', (raw: unknown) => {
const msg = raw as WorkerOutboundMessage;
switch (msg.type) {
case 'progress':
onProgress(msg.message);
break;
case 'result':
totalPages += msg.pagesGenerated;
if (msg.hashUpdates) {
allHashUpdates.push(...msg.hashUpdates);
}
activeWorkers--;
void worker.terminate();
startNextWorker();
break;
case 'error':
errors.push({ taskId: msg.taskId, error: msg.error });
activeWorkers--;
void worker.terminate();
startNextWorker();
break;
}
});
worker.on('error', (err: unknown) => {
const errorMessage = err instanceof Error ? err.message : String(err);
errors.push({ taskId: task.taskId, error: errorMessage });
activeWorkers--;
startNextWorker();
});
worker.on('exit', (code: unknown) => {
// If the worker exited unexpectedly (no result/error message received),
// we need to account for it. The 'error' handler above covers crashes.
// Exit code 0 is normal (worker finished). Non-zero without error handler
// means something unexpected happened.
if (typeof code === 'number' && code !== 0) {
// Only handle if we haven't already decremented (check via activeWorkers)
// This is a safety net — most crashes are caught by the 'error' event.
}
});
};
// Start initial batch of workers
const initialBatch = Math.min(this.maxWorkers, tasks.length);
for (let i = 0; i < initialBatch; i++) {
startNextWorker();
}
});
}
}

View File

@@ -56,6 +56,10 @@ export interface HtmlRewriteContext {
canonicalPostPathBySlug: Map<string, string>;
canonicalMediaPathBySourcePath: Map<string, string>;
languagePrefix?: string;
/** Pre-computed Record version of canonicalPostPathBySlug (avoids repeated Map→Object conversion) */
canonicalPostPathBySlugRecord?: Record<string, string>;
/** Pre-computed Record version of canonicalMediaPathBySourcePath */
canonicalMediaPathBySourcePathRecord?: Record<string, string>;
}
export interface TemplatePostEntry {
@@ -1445,8 +1449,8 @@ export class PageRenderer {
has_next_page: hasNextPage,
prev_page_href: prevPageHref,
next_page_href: nextPageHref,
canonical_post_path_by_slug: mapToRecord(rewriteContext.canonicalPostPathBySlug),
canonical_media_path_by_source_path: mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
canonical_post_path_by_slug: rewriteContext.canonicalPostPathBySlugRecord ?? mapToRecord(rewriteContext.canonicalPostPathBySlug),
canonical_media_path_by_source_path: rewriteContext.canonicalMediaPathBySourcePathRecord ?? mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
post_data_json_by_id: Object.fromEntries(
posts.map((post) => [post.id, JSON.stringify(serializePostDataForMacro(post))]),
),
@@ -1572,7 +1576,7 @@ export class PageRenderer {
? Array.from(new Set(renderablePost.tags.map((tag) => tag.trim()).filter((tag) => tag.length > 0)))
: [];
const canonicalPostPathBySlug = mapToRecord(rewriteContext.canonicalPostPathBySlug);
const canonicalPostPathBySlug = rewriteContext.canonicalPostPathBySlugRecord ?? mapToRecord(rewriteContext.canonicalPostPathBySlug);
// Per-post language overrides the page-level language when present
const postLanguage = (renderablePost as { language?: string }).language;
@@ -1598,7 +1602,7 @@ export class PageRenderer {
calendar_initial_year: renderablePost.createdAt.getFullYear(),
calendar_initial_month: renderablePost.createdAt.getMonth() + 1,
canonical_post_path_by_slug: canonicalPostPathBySlug,
canonical_media_path_by_source_path: mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
canonical_media_path_by_source_path: rewriteContext.canonicalMediaPathBySourcePathRecord ?? mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
post_data_json_by_id: {
[renderablePost.id]: JSON.stringify(serializePostDataForMacro(renderablePost)),
},

View File

@@ -1077,6 +1077,40 @@ export class PostEngine extends EventEmitter {
return result;
}
async getPublishedTranslationsForRoutePosts(publishedPosts: PostData[]): Promise<Map<string, PostTranslationData[]>> {
const allRows = await this.getAllTranslationRows();
const postById = new Map(publishedPosts.map((p) => [p.id, p]));
const result = new Map<string, PostTranslationData[]>();
for (const row of allRows) {
if (row.status !== 'published') continue;
const sourcePost = postById.get(row.translationFor);
if (!sourcePost) continue;
if (this.isCanonicalTranslationLanguage(sourcePost, row.language)) continue;
const translationData: PostTranslationData = {
id: row.id,
projectId: row.projectId,
translationFor: row.translationFor,
language: row.language,
title: row.title,
excerpt: row.excerpt || undefined,
content: '',
status: row.status as 'draft' | 'published' | 'archived',
createdAt: row.createdAt,
updatedAt: row.updatedAt,
publishedAt: row.publishedAt || undefined,
filePath: row.filePath,
};
const arr = result.get(row.translationFor) || [];
arr.push(translationData);
result.set(row.translationFor, arr);
}
return result;
}
async getPostTranslations(postId: string): Promise<PostTranslationData[]> {
const sourcePost = await this.getPost(postId);
const rows = this.filterCanonicalTranslationRows(sourcePost, await this.getTranslationRowsForPost(postId));
@@ -2504,6 +2538,44 @@ export class PostEngine extends EventEmitter {
};
}
async getPublishedVersionsBulk(ids: string[]): Promise<Map<string, PostData>> {
const result = new Map<string, PostData>();
if (ids.length === 0) return result;
const db = getDatabase().getLocal();
const idSet = new Set(ids);
const dbPosts = await db.select().from(posts)
.where(eq(posts.projectId, this.currentProjectId))
.all();
for (const dbPost of dbPosts) {
if (!idSet.has(dbPost.id) || !dbPost.filePath) continue;
result.set(dbPost.id, this.dbRowToPostData(dbPost, ''));
}
return result;
}
/**
* Bulk-load file paths for all published posts in the current project.
* Returns a Map from postId → absolute filePath.
*/
async getPublishedPostFilePaths(): Promise<Map<string, string>> {
const db = getDatabase().getLocal();
const dbPosts = await db.select().from(posts)
.where(eq(posts.projectId, this.currentProjectId))
.all();
const result = new Map<string, string>();
for (const dbPost of dbPosts) {
if (dbPost.filePath) {
result.set(dbPost.id, dbPost.filePath);
}
}
return result;
}
/**
* Rebuild the FTS index for all posts in the current project.
* Call this after changing the search language or after migration.
@@ -3089,6 +3161,48 @@ export class PostEngine extends EventEmitter {
return sourcePosts.filter(p => sourceIds.includes(p.id));
}
/**
* Bulk-load all backlinks for all posts in the current project.
* Returns a Map from targetPostId → array of source posts that link to it.
* Much more efficient than calling getLinkedBy per post during generation.
*/
async getAllBacklinks(): Promise<Map<string, { id: string; title: string; slug: string }[]>> {
const db = getDatabase().getLocal();
const allLinks = await db
.select({
sourcePostId: postLinks.sourcePostId,
targetPostId: postLinks.targetPostId,
})
.from(postLinks);
if (allLinks.length === 0) return new Map();
const sourceIds = new Set(allLinks.map(l => l.sourcePostId));
const allSourcePosts = await db
.select({ id: posts.id, title: posts.title, slug: posts.slug })
.from(posts)
.where(eq(posts.projectId, this.currentProjectId));
const sourcePostById = new Map(
allSourcePosts.filter(p => sourceIds.has(p.id)).map(p => [p.id, p]),
);
const result = new Map<string, { id: string; title: string; slug: string }[]>();
for (const link of allLinks) {
const sourcePost = sourcePostById.get(link.sourcePostId);
if (!sourcePost) continue;
const existing = result.get(link.targetPostId);
if (existing) {
existing.push(sourcePost);
} else {
result.set(link.targetPostId, [sourcePost]);
}
}
return result;
}
/**
* Get posts that the specified post links TO ("links to")
*/

View File

@@ -394,6 +394,32 @@ export class PostMediaEngine extends EventEmitter {
return link.length > 0;
}
/**
* Get all post-media links for the current project, grouped by post ID.
* Used to pre-load data for generation workers.
*/
async getAllPostMediaLinks(): Promise<Map<string, Array<{ mediaId: string; sortOrder: number }>>> {
const db = this.getDb();
const links = await db
.select()
.from(postMedia)
.where(eq(postMedia.projectId, this.currentProjectId))
.orderBy(asc(postMedia.sortOrder));
const result = new Map<string, Array<{ mediaId: string; sortOrder: number }>>();
for (const link of links) {
const existing = result.get(link.postId);
const entry = { mediaId: link.mediaId, sortOrder: link.sortOrder };
if (existing) {
existing.push(entry);
} else {
result.set(link.postId, [entry]);
}
}
return result;
}
/**
* Map database row to PostMediaLinkData
*/

View File

@@ -72,18 +72,22 @@ export async function generateSinglePostPages(params: BaseParams & {
posts: PostData[];
}): Promise<number> {
let count = 0;
const BATCH_SIZE = 10;
for (const post of params.posts) {
const createdAt = resolvePostCreatedAt(post);
const year = createdAt.getFullYear();
const month = String(createdAt.getMonth() + 1).padStart(2, '0');
const day = String(createdAt.getDate()).padStart(2, '0');
for (let i = 0; i < params.posts.length; i += BATCH_SIZE) {
const batch = params.posts.slice(i, i + BATCH_SIZE);
const results = await Promise.all(batch.map(async (post) => {
const createdAt = resolvePostCreatedAt(post);
const year = createdAt.getFullYear();
const month = String(createdAt.getMonth() + 1).padStart(2, '0');
const day = String(createdAt.getDate()).padStart(2, '0');
const urlPath = `${year}/${month}/${day}/${post.slug}`;
const html = await renderRequiredRoute(params.renderRoute, `/${urlPath}`);
await params.writePage(params.projectId, urlPath, html);
count++;
params.onPageGenerated(`Generated /${urlPath}`);
const urlPath = `${year}/${month}/${day}/${post.slug}`;
const html = await renderRequiredRoute(params.renderRoute, `/${urlPath}`);
await params.writePage(params.projectId, urlPath, html);
params.onPageGenerated(`Generated /${urlPath}`);
}));
count += results.length;
}
return count;

View File

@@ -109,7 +109,7 @@ export const CALENDAR_RUNTIME_JS = String.raw`(() => {
}
async function loadCalendarData() {
const response = await fetch(languagePrefix + '/calendar.json', { cache: 'no-store' });
const response = await fetch('/calendar.json', { cache: 'no-store' });
if (!response.ok) {
throw new Error('calendar.json request failed');
}

View File

@@ -0,0 +1,287 @@
/**
* Worker thread entry point for parallel blog generation.
*
* Each worker receives a GenerationWorkerTask via workerData, creates its own
* rendering pipeline (Liquid, PageRenderer, PreviewServer, route renderer) and
* renders the assigned pages, writing them to the filesystem.
*
* Workers do NOT open database connections. Hash reads come from a pre-loaded
* map passed in task data. Hash writes are accumulated in memory and sent back
* to the main thread in the result message for the main thread to persist.
*/
import { parentPort, workerData } from 'worker_threads';
import type {
GenerationWorkerTask,
WorkerOutboundMessage,
SerializedPostData,
} from './GenerationWorkerData';
import {
deserializePostData,
deserializeMediaItem,
deserializePostMap,
deserializeDateMap,
} from './GenerationWorkerData';
import {
createDataBackedPostEngine,
createDataBackedMediaEngine,
createDataBackedPostMediaEngine,
} from './DataBackedEngines';
import { createPreviewBackedGenerationRouteRenderer } from './GenerationRouteRendererFactory';
import {
generateSinglePostPages,
generateCategoryPages,
generateTagPages,
generateDateArchivePages,
generateRootPages,
generatePageRoutes,
} from './RoutePageGenerationService';
import { writeHtmlPage } from './BlogGenerationOutputService';
import type { PostData } from './PostEngine';
// ---------------------------------------------------------------------------
// In-memory hash store (no DB access)
// ---------------------------------------------------------------------------
/**
* Creates a purely in-memory hash store.
* Reads come from the pre-loaded hash map (passed from main thread).
* Writes are accumulated in `pendingUpdates` and returned to the main thread
* via the result message so it can persist them in a single connection.
*/
function createWorkerHashStore(hashCache: Map<string, string | null>) {
const pendingUpdates: Array<{ relativePath: string; hash: string }> = [];
return {
async get(_projectId: string, relativePath: string): Promise<string | null> {
return hashCache.get(relativePath) ?? null;
},
async set(_projectId: string, relativePath: string, hash: string): Promise<void> {
pendingUpdates.push({ relativePath, hash });
hashCache.set(relativePath, hash);
},
pendingUpdates,
};
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function send(message: WorkerOutboundMessage): void {
parentPort?.postMessage(message);
}
function deserializePostArray(serialized: SerializedPostData[]): PostData[] {
return serialized.map(deserializePostData);
}
// ---------------------------------------------------------------------------
// Main worker logic
// ---------------------------------------------------------------------------
async function run(): Promise<void> {
const task = workerData as GenerationWorkerTask;
try {
// 1. Reconstruct hash cache from pre-loaded entries (no DB needed)
const hashCache = new Map<string, string | null>();
for (const [relativePath, hash] of task.hashMapEntries) {
hashCache.set(relativePath, hash);
}
const hashStore = createWorkerHashStore(hashCache);
// 2. Deserialize post data
const posts = deserializePostArray(task.posts);
const lookupPosts = deserializePostArray(task.lookupPosts);
const mediaItems = (task.mediaItems ?? []).map(deserializeMediaItem);
// 2b. Reconstruct post file paths for lazy content loading
const postFilePaths = new Map<string, string>();
if (task.postFilePathEntries) {
for (const [postId, filePath] of task.postFilePathEntries) {
postFilePaths.set(postId, filePath);
}
}
// 2c. Reconstruct post-media links for gallery/album macros
const postMediaLinks = new Map<string, Array<{ mediaId: string; sortOrder: number }>>();
if (task.postMediaLinksEntries) {
for (const [postId, links] of task.postMediaLinksEntries) {
postMediaLinks.set(postId, links);
}
}
// 3. Reconstruct backlinks Map
const backlinksMap = new Map<string, Array<{ id: string; title: string; slug: string }>>();
if (task.backlinksMap) {
for (const [postId, links] of Object.entries(task.backlinksMap)) {
backlinksMap.set(postId, links);
}
}
// 4. Create data-backed engines
const postEngine = createDataBackedPostEngine({ allPosts: lookupPosts, backlinksMap, postFilePaths });
const mediaEngine = createDataBackedMediaEngine(mediaItems);
const postMediaEngine = createDataBackedPostMediaEngine({ mediaItems, postMediaLinks });
// 5. Create route renderer (same factory as main thread, but backed by data)
const renderRoute = createPreviewBackedGenerationRouteRenderer({
options: {
...task.options,
language: task.options.language,
},
projectMainLanguage: task.mainLanguage,
maxPostsPerPage: task.maxPostsPerPage,
publishedPostsForLookup: lookupPosts,
languagePrefix: task.languagePrefix,
engines: {
postEngine: postEngine as any,
mediaEngine: mediaEngine as any,
postMediaEngine: postMediaEngine as any,
},
});
// 6. Build writePage function using in-memory hash store
const knownDirectories = new Set<string>();
const writePage = (projectId: string, urlPath: string, content: string) => {
const effectiveUrlPath = task.languagePrefix
? `${task.languagePrefix.replace(/^\//, '')}/${urlPath}`
: urlPath;
return writeHtmlPage({
projectId,
htmlDir: task.htmlDir,
urlPath: effectiveUrlPath,
content,
knownDirectories,
hashCache,
getGeneratedFileHash: hashStore.get,
setGeneratedFileHash: hashStore.set,
refreshHashTimestampOnUnchanged: true,
});
};
const onPageGenerated = (message: string) => {
send({ type: 'progress', taskId: task.taskId, message });
};
// 7. Execute the assigned section
let pagesGenerated = 0;
const projectId = task.options.projectId;
switch (task.section) {
case 'single': {
pagesGenerated += await generateSinglePostPages({
projectId,
posts,
renderRoute,
writePage,
onPageGenerated,
});
break;
}
case 'category': {
const allCategories = new Set(task.allCategories ?? []);
const postsByCategory = task.postsByCategoryEntries
? deserializePostMap(task.postsByCategoryEntries)
: undefined;
pagesGenerated += await generateCategoryPages({
projectId,
posts,
allCategories,
maxPostsPerPage: task.maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
postsByCategory,
});
break;
}
case 'tag': {
const allTags = new Set(task.allTags ?? []);
const postsByTag = task.postsByTagEntries
? deserializePostMap(task.postsByTagEntries)
: undefined;
pagesGenerated += await generateTagPages({
projectId,
posts,
allTags,
maxPostsPerPage: task.maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
postsByTag,
});
break;
}
case 'date': {
const yearsMap = task.yearsEntries ? deserializeDateMap(task.yearsEntries) : new Map();
const yearMonthsMap = task.yearMonthsEntries ? deserializeDateMap(task.yearMonthsEntries) : new Map();
const yearMonthDaysMap = task.yearMonthDaysEntries ? deserializeDateMap(task.yearMonthDaysEntries) : new Map();
const postsByYear = task.postsByYearEntries ? deserializePostMap(task.postsByYearEntries) : undefined;
const postsByYearMonth = task.postsByYearMonthEntries ? deserializePostMap(task.postsByYearMonthEntries) : undefined;
const postsByYearMonthDay = task.postsByYearMonthDayEntries ? deserializePostMap(task.postsByYearMonthDayEntries) : undefined;
pagesGenerated += await generateDateArchivePages({
projectId,
posts,
yearsMap,
yearMonthsMap,
yearMonthDaysMap,
maxPostsPerPage: task.maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
postsByYear,
postsByYearMonth,
postsByYearMonthDay,
});
break;
}
case 'core': {
// Core includes root pages and page routes (sitemap/feeds handled by main thread)
pagesGenerated += await generateRootPages({
projectId,
posts,
maxPostsPerPage: task.maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
});
pagesGenerated += await generatePageRoutes({
projectId,
posts: lookupPosts,
renderRoute,
writePage,
onPageGenerated,
});
break;
}
}
// 8. Report result with accumulated hash updates
send({
type: 'result',
taskId: task.taskId,
pagesGenerated,
hashUpdates: hashStore.pendingUpdates,
});
} catch (err) {
send({
type: 'error',
taskId: task.taskId,
error: err instanceof Error ? err.message : String(err),
});
}
}
run();

View File

@@ -11,6 +11,7 @@ import type { EngineBundle } from '../engine/EngineBundle';
import type { TranslationValidationReport } from '../shared/electronApi';
import { autoTranslatePost, autoTranslateMediaMetadata } from './chatHandlers';
import { v4 as uuidv4 } from 'uuid';
import { getDatabase } from '../database/connection';
type SafeHandle = (channel: string, handler: (...args: any[]) => Promise<any>) => void;
@@ -84,6 +85,7 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl
categoryMetadata: (metadata as any)?.categoryMetadata,
categorySettings: (metadata as any)?.categorySettings,
menu,
dbPath: getDatabase().getDbPath(),
};
};
@@ -91,6 +93,9 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl
const blogGenerationEngine = bundle.blogGenerationEngine;
const baseOptions = await resolveBlogGenerationBaseOptions();
// Pre-load post data ONCE before parallel tasks
const preloadedData = await blogGenerationEngine.preloadGenerationData(baseOptions);
const taskTimestamp = Date.now();
const taskGroupId = `site-render-${taskTimestamp}`;
const taskGroupName = 'Render Site';
@@ -109,6 +114,7 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl
return blogGenerationEngine.generate({
...baseOptions,
sections: [section],
preloadedData,
}, (progress, message) => onProgress(progress, message || ''));
},
});

View File

@@ -412,7 +412,7 @@ export const electronAPI: ElectronAPI = {
translatePost: (postId: string, targetLanguage: string) => ipcRenderer.invoke('chat:translatePost', postId, targetLanguage),
// Media Language Detection
detectMediaLanguage: (mediaId: string) => ipcRenderer.invoke('chat:detectMediaLanguage', mediaId),
detectMediaLanguage: (title: string, alt: string, caption: string) => ipcRenderer.invoke('chat:detectMediaLanguage', title, alt, caption),
// Media Metadata Translation
translateMediaMetadata: (mediaId: string, targetLanguage: string) => ipcRenderer.invoke('chat:translateMediaMetadata', mediaId, targetLanguage),

View File

@@ -1070,7 +1070,7 @@ export interface ElectronAPI {
translatePost: (postId: string, targetLanguage: string) => Promise<{ success: boolean; translation?: PostTranslationData; error?: string }>;
// Media Language Detection
detectMediaLanguage: (mediaId: string) => Promise<{ success: boolean; language?: string; error?: string }>;
detectMediaLanguage: (title: string, alt: string, caption: string) => Promise<{ success: boolean; language?: string; error?: string }>;
// Media Metadata Translation
translateMediaMetadata: (mediaId: string, targetLanguage: string) => Promise<{ success: boolean; translation?: MediaTranslationData; error?: string }>;

View File

@@ -39,7 +39,6 @@ const App: React.FC = () => {
toggleSidebar,
togglePanel,
toggleAssistantSidebar,
setActiveView,
setSelectedPost,
setActiveProject,
setPicoTheme,

View File

@@ -1,4 +1,4 @@
import React, { type ReactNode } from 'react';
import React, { type ReactElement } from 'react';
import Markdown from 'marked-react';
import type { A2UIResolvedComponent, A2UIClientAction } from '../../../main/a2ui/types';
@@ -11,7 +11,7 @@ interface A2UIComponentProps {
}
const safeRenderer = {
image(src: string, alt: string): ReactNode {
image(src: string, alt: string, _title?: string | null): ReactElement {
if (/^https?:\/\//i.test(src)) {
return <a href={src} key={src} title={alt}>{alt || src}</a>;
}

View File

@@ -1,8 +1,8 @@
import React, { type ReactNode } from 'react';
import React, { type ReactElement } from 'react';
import Markdown from 'marked-react';
import type { ChatMessage } from '../../types/electron';
import type { ChatToolEvent } from '../../navigation/useChatSurfaceState';
import type { A2UIResolvedComponent, A2UIClientAction } from '../../../main/a2ui/types';
import type { A2UIClientAction } from '../../../main/a2ui/types';
import { InlineSurface } from '../../a2ui/InlineSurface';
import type { SurfaceEntry } from '../../a2ui/useA2UISurface';
import { computeTurnIndex } from '../../a2ui/surfaceAssociation';
@@ -51,7 +51,7 @@ export const ChatTranscript: React.FC<ChatTranscriptProps> = ({
}) => {
// Block external images — CSP only allows self/data/file/blob/bds-media/bds-thumb
const safeRenderer = {
image(src: string, alt: string): ReactNode {
image(src: string, alt: string, _title?: string | null): ReactElement {
if (/^https?:\/\//i.test(src)) {
// Show alt text as a link instead of trying to load the image
return <a href={src} key={src} title={alt}>{alt || src}</a>;

View File

@@ -155,7 +155,7 @@ export const DocumentationView: React.FC<DocumentationViewProps> = ({
headingSlugCounts.set(baseId, nextCount);
const headingId = existingCount === 0 ? baseId : `${baseId}-${nextCount}`;
return React.createElement(`h${levelNumber}` as keyof JSX.IntrinsicElements, { id: headingId, key: getRendererKey('heading') }, children);
return React.createElement(`h${levelNumber}` as 'h1', { id: headingId, key: getRendererKey('heading') }, children);
},
link(href: string, text: ReactNode) {
if (!href.startsWith('#')) {

View File

@@ -5,6 +5,7 @@ import { AISuggestionsModal } from '../AISuggestionsModal/AISuggestionsModal';
import { openEntityTab } from '../../navigation/tabPolicy';
import { useI18n } from '../../i18n';
import { SUPPORTED_POST_LANGUAGES, POST_LANGUAGE_FLAGS } from '../../../main/shared/i18n';
import type { MediaData } from '../../../main/shared/electronApi';
import { getMediaDisplayName } from './editorUtils';
export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
@@ -71,7 +72,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
try {
const updated = await window.electronAPI?.media.update(item!.id, { language: newLanguage || undefined });
if (updated) {
updateMedia(item!.id, updated as Partial<typeof item>);
updateMedia(item!.id, updated as Partial<MediaData>);
}
} catch (error) {
console.error('Failed to update media language:', error);
@@ -92,7 +93,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
setMediaLanguage(result.language);
const updated = await window.electronAPI?.media.update(item.id, { language: result.language });
if (updated) {
updateMedia(item.id, updated as Partial<typeof item>);
updateMedia(item.id, updated as Partial<MediaData>);
}
showToast.success(tr('editor.media.toast.languageDetected', { language: tr(`language.${result.language}`) }));
} else {
@@ -249,7 +250,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
// Close AI suggestions modal
const handleCloseAISuggestionsModal = () => {
setShowAISuggestionsModal(false);
setAISuggestions(null);
setAISuggestionFields([]);
setAIError(undefined);
};
@@ -364,7 +365,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
tags: tags.split(',').map(t => t.trim()).filter(t => t.length > 0),
});
if (updated) {
updateMedia(item.id, updated as Partial<typeof item>);
updateMedia(item.id, updated as Partial<MediaData>);
showToast.success(tr('editor.media.toast.updated'));
}
} catch (error) {
@@ -382,7 +383,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
try {
const updated = await window.electronAPI?.media.replaceFileDialog(item.id);
if (updated) {
updateMedia(item.id, updated as Partial<typeof item>);
updateMedia(item.id, updated as Partial<MediaData>);
showToast.success(tr('editor.media.toast.fileReplaced'));
}
// null means user cancelled or file unchanged - no action needed
@@ -523,7 +524,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
{item.mimeType.startsWith('image/') ? (
<div className="media-preview-image">
<img
src={`bds-media://${item.id}?t=${item.updatedAt instanceof Date ? item.updatedAt.getTime() : item.updatedAt}`}
src={`bds-media://${item.id}?t=${item.updatedAt}`}
alt={item.alt || item.originalName}
onError={(e) => {
// Fallback to placeholder if image fails to load

View File

@@ -159,7 +159,6 @@ export const PostEditor: React.FC<PostEditorProps> = ({ postId }) => {
showErrorModal,
showConfirmDeleteModal,
media,
closeTab,
} = useAppStore();
// Fetch full post data from backend
@@ -194,7 +193,7 @@ export const PostEditor: React.FC<PostEditorProps> = ({ postId }) => {
const [doNotTranslate, setDoNotTranslate] = useState(false);
const [activeEditingLanguage, setActiveEditingLanguage] = useState('');
const [canonicalDraft, setCanonicalDraft] = useState<EditableContentDraft>({ title: '', excerpt: '', content: '' });
const [savedCanonicalDraft, setSavedCanonicalDraft] = useState<EditableContentDraft>({ title: '', excerpt: '', content: '' });
const [, setSavedCanonicalDraft] = useState<EditableContentDraft>({ title: '', excerpt: '', content: '' });
const [translationDrafts, setTranslationDrafts] = useState<Record<string, import('../../../main/shared/electronApi').PostTranslationData>>({});
const [savedTranslationDrafts, setSavedTranslationDrafts] = useState<Record<string, import('../../../main/shared/electronApi').PostTranslationData>>({});
const [availablePostTemplates, setAvailablePostTemplates] = useState<Array<{ slug: string; title: string }>>([]);

View File

@@ -390,7 +390,7 @@ export const GitSidebar: React.FC = () => {
recentCommitsToKeep: 2,
});
if (!result.success) {
if (result.code === 'offline') {
if ('code' in result && result.code === 'offline') {
showErrorModal({ message: tr('gitSidebar.error.offlineMode') });
return;
}

View File

@@ -208,23 +208,25 @@ export const ImportAnalysisView: React.FC<ImportAnalysisViewProps> = ({ definiti
// Subscribe to task completion events
useEffect(() => {
const unsubscribe = window.electronAPI?.on('task:completed', (task: { taskId: string }) => {
const unsubscribe = window.electronAPI?.on('task:completed', ((...args: unknown[]) => {
const task = args[0] as { taskId: string };
setExecutionState(prev => {
if (prev.taskId !== task.taskId) return prev;
return { ...prev, isExecuting: false, completed: true };
});
});
}) as (...args: unknown[]) => void);
return () => unsubscribe?.();
}, []);
// Subscribe to task failure events
useEffect(() => {
const unsubscribe = window.electronAPI?.on('task:failed', (task: { taskId: string; error: string }) => {
const unsubscribe = window.electronAPI?.on('task:failed', ((...args: unknown[]) => {
const task = args[0] as { taskId: string; error: string };
setExecutionState(prev => {
if (prev.taskId !== task.taskId) return prev;
return { ...prev, isExecuting: false, error: task.error };
});
});
}) as (...args: unknown[]) => void);
return () => unsubscribe?.();
}, []);
@@ -919,7 +921,7 @@ const DateDistributionCard: React.FC<{ distribution: DateDistribution }> = ({ di
};
// Helper function to format post metadata for tooltip (new post from WXR)
function formatPostTooltip(wxrPost: AnalyzedPostItem['wxrPost'], t: (key: string, params?: Record<string, unknown>) => string): string {
function formatPostTooltip(wxrPost: AnalyzedPostItem['wxrPost'], t: (key: string, params?: Record<string, string | number>) => string): string {
const lines: string[] = [];
lines.push(`${t('importAnalysis.wordpressId')}: ${wxrPost.wpId}`);
lines.push(`${t('importAnalysis.type')}: ${wxrPost.postType}`);
@@ -1051,7 +1053,7 @@ function ExistingPostHoverCard({ children, className, postId }: {
}
// Helper function to format media metadata for tooltip
function formatMediaTooltip(wxrMedia: AnalyzedMediaItem['wxrMedia'], t: (key: string, params?: Record<string, unknown>) => string): string {
function formatMediaTooltip(wxrMedia: AnalyzedMediaItem['wxrMedia'], t: (key: string, params?: Record<string, string | number>) => string): string {
const lines: string[] = [];
lines.push(`${t('importAnalysis.wordpressId')}: ${wxrMedia.wpId}`);
lines.push(`${t('importAnalysis.mimeType')}: ${wxrMedia.mimeType || t('importAnalysis.unknown')}`);

View File

@@ -1,6 +1,6 @@
import React, { useEffect, useRef, useCallback, useState } from 'react';
import { Editor, defaultValueCtx, editorViewCtx, rootCtx, remarkStringifyOptionsCtx, remarkPluginsCtx } from '@milkdown/kit/core';
import { commonmark, toggleStrongCommand, toggleEmphasisCommand, wrapInBlockquoteCommand, wrapInBulletListCommand, wrapInOrderedListCommand, insertHrCommand, toggleInlineCodeCommand, insertImageCommand, toggleLinkCommand } from '@milkdown/kit/preset/commonmark';
import { commonmark, toggleStrongCommand, toggleEmphasisCommand, wrapInBlockquoteCommand, wrapInBulletListCommand, wrapInOrderedListCommand, insertHrCommand, toggleInlineCodeCommand, insertImageCommand } from '@milkdown/kit/preset/commonmark';
import { gfm, toggleStrikethroughCommand } from '@milkdown/kit/preset/gfm';
import { history, undoCommand, redoCommand } from '@milkdown/kit/plugin/history';
import { listener, listenerCtx } from '@milkdown/kit/plugin/listener';

View File

@@ -1,4 +1,4 @@
import React, { useCallback, useEffect, useRef, useState } from 'react';
import React, { useCallback, useRef, useState } from 'react';
import MonacoEditor, { type Monaco } from '@monaco-editor/react';
import type { ScriptData } from '../../../main/shared/electronApi';
import { useAppStore } from '../../store';
@@ -89,7 +89,6 @@ export const ScriptsView: React.FC<ScriptsViewProps> = ({ scriptId }) => {
// Refresh entrypoints asynchronously
entrypointCancelRef.current = true; // cancel any pending refresh
const cancelToken = {};
entrypointCancelRef.current = false;
const refreshEntrypoints = async () => {
try {

View File

@@ -28,7 +28,7 @@ export function SidebarEntityList<TItem>({
renderItem,
getItemKey,
topContent,
}: SidebarEntityListProps<TItem>): JSX.Element {
}: SidebarEntityListProps<TItem>): React.JSX.Element {
if (isLoading) {
return (
<div className="chat-list">

View File

@@ -346,7 +346,7 @@ export const WindowTitleBar: React.FC = () => {
};
}, [isMac, mnemonicByKey, showMnemonics]);
const handleMenuButtonClick = (event: React.MouseEvent<HTMLButtonElement>, label: string) => {
const handleMenuButtonClick = (_event: React.MouseEvent<HTMLButtonElement>, label: string) => {
const left = getMenuLeft(label);
if (left === null) {
return;

View File

@@ -1,4 +1,4 @@
import type { PythonMacroInfo, PythonMacroResolver, PythonMacroRendererFn } from './types';
import type { PythonMacroResolver, PythonMacroRendererFn } from './types';
import { setPythonMacroResolver } from './registry';
import { getPythonRuntimeManager } from '../python/runtimeManagerInstance';

View File

@@ -138,7 +138,7 @@ const tabsElementSchema: z.ZodTypeAny = z.lazy(() => z.object({
).min(1),
}));
assistantPanelElementSchemaRef = z.discriminatedUnion('type', [
assistantPanelElementSchemaRef = z.union([
textElementSchema,
metricElementSchema,
listElementSchema,

View File

@@ -1,4 +1,4 @@
import { openEntityTab } from './tabPolicy';
import { openEntityTab, type CanonicalTabSpec } from './tabPolicy';
import type { SidebarView } from './sidebarViewRegistry';
interface BlogmarkStateSnapshot {
@@ -14,7 +14,7 @@ interface BlogmarkHandlers {
setActiveView: (view: SidebarView) => void;
toggleSidebar: () => void;
setSelectedPost: (id: string) => void;
openTab: (tab: { type: 'post'; id: string; isTransient: boolean }) => void;
openTab: (tab: CanonicalTabSpec) => void;
}
export function handleBlogmarkCreatedEvent(

View File

@@ -83,13 +83,13 @@ export function parseBlogmarkCreatedEventPayload(payload: unknown): BlogmarkCrea
if (isRecord(payload.post)) {
return {
post: payload.post as PostData,
post: payload.post as unknown as PostData,
transform: parseTransformDebugInfo(payload.transform),
};
}
return {
post: payload as PostData,
post: payload as unknown as PostData,
transform: undefined,
};
}

View File

@@ -1,7 +1,7 @@
import { createPythonRuntimeWorker } from './createPythonRuntimeWorker';
import type { PythonWorkerMessage, PythonWorkerRequest } from './runtimeProtocol';
import type { PythonSyntaxError } from './runtimeProtocol';
import { parseMacroContextV1, parseMacroResultV1, type MacroContextV1, type MacroResultV1 } from './abiV1';
import { parseMacroContextV1, parseMacroResultV1, type MacroResultV1 } from './abiV1';
import { invokePythonApiMethodV1 } from './pythonApiInvokerV1';
import { showToast } from '../components/Toast';

View File

@@ -0,0 +1,4 @@
declare module '@highlightjs/cdn-assets/es/highlight.min.js' {
import hljs from 'highlight.js';
export default hljs;
}