Feature/worker threads generation (#43)
* Add worker threads architecture plan for blog generation * fix: tries to optimize rendering, still slow * feat: moved site rendering into web worker * fix: calendar grabs from central data source for calendar * fix: feeds now use blog language content and not canonical content --------- Co-authored-by: hugo <hugoms@me.com>
This commit is contained in:
@@ -66,3 +66,27 @@ export async function setGeneratedFileHash(projectId: string, relativePath: stri
|
||||
args: [projectId, relativePath, hash, Date.now()],
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk-load all file hashes for a project in a single query.
|
||||
* Returns a Map from relativePath → contentHash.
|
||||
*/
|
||||
export async function getAllGeneratedFileHashes(projectId: string): Promise<Map<string, string>> {
|
||||
const client = getDatabase().getLocalClient();
|
||||
if (!client) {
|
||||
throw new Error('Database client not available');
|
||||
}
|
||||
|
||||
const result = await client.execute({
|
||||
sql: 'SELECT relative_path, content_hash FROM generated_file_hashes WHERE project_id = ?',
|
||||
args: [projectId],
|
||||
});
|
||||
|
||||
const map = new Map<string, string>();
|
||||
for (const row of result.rows) {
|
||||
if (typeof row.relative_path === 'string' && typeof row.content_hash === 'string') {
|
||||
map.set(row.relative_path, row.content_hash);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
250
src/main/engine/DataBackedEngines.ts
Normal file
250
src/main/engine/DataBackedEngines.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* Lightweight, in-memory engine implementations for use in worker threads.
|
||||
*
|
||||
* These replace the real PostEngine / MediaEngine / PostMediaEngine when running
|
||||
* inside a generation worker. They are backed entirely by pre-loaded data arrays
|
||||
* so no database access is needed for post/media queries.
|
||||
*/
|
||||
import type { PostData, PostFilter, PostTranslationData } from './PostEngine';
|
||||
import type { MediaData } from './MediaEngine';
|
||||
import { readPostFile } from './postFileUtils';
|
||||
import { readPostTranslationFile } from './postTranslationFileUtils';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DataBackedPostEngine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface DataBackedPostEngineInit {
|
||||
/** All posts (published snapshots + translation variants). */
|
||||
allPosts: PostData[];
|
||||
/** Pre-resolved backlinks: postId → linking posts. */
|
||||
backlinksMap?: Map<string, Array<{ id: string; title: string; slug: string }>>;
|
||||
/** Post file paths for lazy content loading from filesystem: postId → absoluteFilePath. */
|
||||
postFilePaths?: Map<string, string>;
|
||||
}
|
||||
|
||||
export interface DataBackedPostEngineContract {
|
||||
getPostsFiltered: (filter: PostFilter) => Promise<PostData[]>;
|
||||
getPublishedVersion: (id: string) => Promise<PostData | null>;
|
||||
getPost: (id: string) => Promise<PostData | null>;
|
||||
hasPublishedVersion: (id: string) => Promise<boolean>;
|
||||
findPublishedBySlug: (slug: string, dateFilter?: { year: number; month: number }) => Promise<PostData | null>;
|
||||
getLinkedBy: (postId: string) => Promise<Array<{ id: string; title: string; slug: string }>>;
|
||||
getAllBacklinks: () => Promise<Map<string, Array<{ id: string; title: string; slug: string }>>>;
|
||||
getPostTranslation: (postId: string, language: string) => Promise<PostTranslationData | null>;
|
||||
getPostTranslations: (postId: string) => Promise<PostTranslationData[]>;
|
||||
setProjectContext: (projectId: string, dataDir?: string) => void;
|
||||
}
|
||||
|
||||
export function createDataBackedPostEngine(init: DataBackedPostEngineInit): DataBackedPostEngineContract {
|
||||
const { allPosts, backlinksMap, postFilePaths } = init;
|
||||
|
||||
// Build indexes for fast lookups
|
||||
const byId = new Map<string, PostData>();
|
||||
const bySlug = new Map<string, PostData[]>();
|
||||
for (const post of allPosts) {
|
||||
byId.set(post.id, post);
|
||||
const slugEntries = bySlug.get(post.slug);
|
||||
if (slugEntries) {
|
||||
slugEntries.push(post);
|
||||
} else {
|
||||
bySlug.set(post.slug, [post]);
|
||||
}
|
||||
}
|
||||
|
||||
function matchesFilter(post: PostData, filter: PostFilter): boolean {
|
||||
if (filter.status && post.status !== filter.status) return false;
|
||||
|
||||
if (filter.excludeCategories && filter.excludeCategories.length > 0) {
|
||||
const excluded = new Set(filter.excludeCategories);
|
||||
if (post.categories.some((c) => excluded.has(c))) return false;
|
||||
}
|
||||
|
||||
if (filter.categories && filter.categories.length > 0) {
|
||||
if (!filter.categories.some((c) => post.categories.includes(c))) return false;
|
||||
}
|
||||
|
||||
if (filter.tags && filter.tags.length > 0) {
|
||||
if (!filter.tags.every((t) => post.tags.includes(t))) return false;
|
||||
}
|
||||
|
||||
if (filter.language && post.language !== filter.language) return false;
|
||||
|
||||
if (filter.year !== undefined) {
|
||||
if (post.createdAt.getFullYear() !== filter.year) return false;
|
||||
}
|
||||
|
||||
if (filter.month !== undefined) {
|
||||
if (post.createdAt.getMonth() + 1 !== filter.month) return false;
|
||||
}
|
||||
|
||||
if (filter.startDate) {
|
||||
if (post.createdAt < filter.startDate) return false;
|
||||
}
|
||||
|
||||
if (filter.endDate) {
|
||||
if (post.createdAt > filter.endDate) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Shared lazy content loader for posts with empty content
|
||||
async function lazyLoadContent(post: PostData): Promise<void> {
|
||||
if (post.content || !postFilePaths) return;
|
||||
const variant = post as PostData & { translationFilePath?: string };
|
||||
if (variant.translationFilePath) {
|
||||
const fileData = await readPostTranslationFile(variant.translationFilePath);
|
||||
if (fileData) {
|
||||
post.content = fileData.content;
|
||||
}
|
||||
} else {
|
||||
const filePath = postFilePaths.get(post.id);
|
||||
if (filePath) {
|
||||
const fileData = await readPostFile(filePath);
|
||||
if (fileData) {
|
||||
post.content = fileData.content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async getPostsFiltered(filter: PostFilter): Promise<PostData[]> {
|
||||
const filtered = allPosts
|
||||
.filter((post) => {
|
||||
const tss = (post as any).translationSourceSlug;
|
||||
// Keep canonical posts and resolved posts (slug === tss).
|
||||
// Exclude translation variant route posts (slug !== tss, e.g. "my-post.en").
|
||||
return !tss || post.slug === tss;
|
||||
})
|
||||
.filter((post) => matchesFilter(post, filter))
|
||||
.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
||||
|
||||
// Lazy-load content for posts that need it (e.g. resolved translation posts
|
||||
// have content: '' and need their translation file content loaded).
|
||||
await Promise.all(filtered.map((post) => lazyLoadContent(post)));
|
||||
|
||||
return filtered;
|
||||
},
|
||||
|
||||
async getPublishedVersion(id: string): Promise<PostData | null> {
|
||||
const post = byId.get(id);
|
||||
if (!post) return null;
|
||||
|
||||
await lazyLoadContent(post);
|
||||
|
||||
return post;
|
||||
},
|
||||
|
||||
async getPost(id: string): Promise<PostData | null> {
|
||||
const post = byId.get(id) ?? null;
|
||||
if (post) {
|
||||
await lazyLoadContent(post);
|
||||
}
|
||||
return post;
|
||||
},
|
||||
|
||||
async hasPublishedVersion(id: string): Promise<boolean> {
|
||||
return byId.has(id);
|
||||
},
|
||||
|
||||
async findPublishedBySlug(slug: string, dateFilter?: { year: number; month: number }): Promise<PostData | null> {
|
||||
const candidates = bySlug.get(slug);
|
||||
if (!candidates || candidates.length === 0) return null;
|
||||
|
||||
if (!dateFilter) return candidates[0];
|
||||
|
||||
return candidates.find((p) =>
|
||||
p.createdAt.getFullYear() === dateFilter.year
|
||||
&& p.createdAt.getMonth() === dateFilter.month - 1,
|
||||
) ?? null;
|
||||
},
|
||||
|
||||
async getLinkedBy(postId: string): Promise<Array<{ id: string; title: string; slug: string }>> {
|
||||
return backlinksMap?.get(postId) ?? [];
|
||||
},
|
||||
|
||||
async getAllBacklinks(): Promise<Map<string, Array<{ id: string; title: string; slug: string }>>> {
|
||||
return backlinksMap ?? new Map();
|
||||
},
|
||||
|
||||
async getPostTranslation(_postId: string, _language: string): Promise<PostTranslationData | null> {
|
||||
// Translation variants are already included as separate route posts
|
||||
return null;
|
||||
},
|
||||
|
||||
async getPostTranslations(_postId: string): Promise<PostTranslationData[]> {
|
||||
return [];
|
||||
},
|
||||
|
||||
setProjectContext(_projectId: string, _dataDir?: string): void {
|
||||
// No-op — data is already loaded
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DataBackedMediaEngine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface DataBackedMediaEngineContract {
|
||||
getAllMedia: () => Promise<MediaData[]>;
|
||||
setProjectContext: (projectId: string, dataDir?: string, internalDir?: string) => void;
|
||||
}
|
||||
|
||||
export function createDataBackedMediaEngine(
|
||||
mediaItems: MediaData[],
|
||||
): DataBackedMediaEngineContract {
|
||||
return {
|
||||
async getAllMedia() {
|
||||
return mediaItems;
|
||||
},
|
||||
setProjectContext(_projectId: string, _dataDir?: string, _internalDir?: string): void {
|
||||
// No-op
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DataBackedPostMediaEngine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface DataBackedPostMediaEngineInit {
|
||||
mediaItems: MediaData[];
|
||||
postMediaLinks: Map<string, Array<{ mediaId: string; sortOrder: number }>>;
|
||||
}
|
||||
|
||||
export interface DataBackedPostMediaEngineContract {
|
||||
setProjectContext: (projectId: string) => void;
|
||||
getLinkedMediaForPost: (postId: string) => Promise<Array<{ mediaId: string; sortOrder: number }>>;
|
||||
getLinkedMediaDataForPost: (postId: string) => Promise<Array<{ media: MediaData }>>;
|
||||
}
|
||||
|
||||
export function createDataBackedPostMediaEngine(init: DataBackedPostMediaEngineInit): DataBackedPostMediaEngineContract {
|
||||
const { mediaItems, postMediaLinks } = init;
|
||||
const mediaById = new Map<string, MediaData>();
|
||||
for (const m of mediaItems) {
|
||||
mediaById.set(m.id, m);
|
||||
}
|
||||
|
||||
return {
|
||||
setProjectContext(_projectId: string): void {
|
||||
// No-op
|
||||
},
|
||||
async getLinkedMediaForPost(postId: string): Promise<Array<{ mediaId: string; sortOrder: number }>> {
|
||||
return postMediaLinks.get(postId) ?? [];
|
||||
},
|
||||
async getLinkedMediaDataForPost(postId: string): Promise<Array<{ media: MediaData }>> {
|
||||
const links = postMediaLinks.get(postId) ?? [];
|
||||
const result: Array<{ media: MediaData }> = [];
|
||||
for (const link of links) {
|
||||
const media = mediaById.get(link.mediaId);
|
||||
if (media) {
|
||||
result.push({ media });
|
||||
}
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import type { PostData } from './PostEngine';
|
||||
export interface GenerationSnapshotPostEngine {
|
||||
getPostsFiltered: (filter: { status?: 'draft' | 'published' | 'archived'; excludeCategories?: string[] }) => Promise<PostData[]>;
|
||||
getPublishedVersion: (id: string) => Promise<PostData | null>;
|
||||
getPublishedVersionsBulk?: (ids: string[]) => Promise<Map<string, PostData>>;
|
||||
}
|
||||
|
||||
export interface GenerationPublishedSets {
|
||||
@@ -10,57 +11,64 @@ export interface GenerationPublishedSets {
|
||||
publishedListPosts: PostData[];
|
||||
}
|
||||
|
||||
async function resolvePublishedVersions(
|
||||
postEngine: GenerationSnapshotPostEngine,
|
||||
ids: string[],
|
||||
): Promise<Map<string, PostData>> {
|
||||
if (ids.length === 0) return new Map();
|
||||
|
||||
if (postEngine.getPublishedVersionsBulk) {
|
||||
return postEngine.getPublishedVersionsBulk(ids);
|
||||
}
|
||||
|
||||
const result = new Map<string, PostData>();
|
||||
const entries = await Promise.all(
|
||||
ids.map(async (id) => {
|
||||
const version = await postEngine.getPublishedVersion(id);
|
||||
return { id, version };
|
||||
}),
|
||||
);
|
||||
for (const { id, version } of entries) {
|
||||
if (version) result.set(id, version);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function loadPublishedGenerationSets(
|
||||
postEngine: GenerationSnapshotPostEngine,
|
||||
listExcludedCategories: string[],
|
||||
): Promise<GenerationPublishedSets> {
|
||||
const publishedCandidates = await postEngine.getPostsFiltered({ status: 'published' });
|
||||
const draftCandidates = await postEngine.getPostsFiltered({ status: 'draft' });
|
||||
const publishedListCandidates = await postEngine.getPostsFiltered({
|
||||
status: 'published',
|
||||
excludeCategories: listExcludedCategories,
|
||||
});
|
||||
const draftListCandidates = await postEngine.getPostsFiltered({
|
||||
status: 'draft',
|
||||
excludeCategories: listExcludedCategories,
|
||||
});
|
||||
|
||||
const publishedSnapshots = await Promise.all(
|
||||
publishedCandidates.map(async (post) => {
|
||||
const snapshot = await postEngine.getPublishedVersion(post.id);
|
||||
return snapshot || post;
|
||||
}),
|
||||
);
|
||||
const draftPublishedSnapshots = await Promise.all(
|
||||
draftCandidates.map(async (post) => postEngine.getPublishedVersion(post.id)),
|
||||
);
|
||||
const publishedListSnapshots = await Promise.all(
|
||||
publishedListCandidates.map(async (post) => {
|
||||
const snapshot = await postEngine.getPublishedVersion(post.id);
|
||||
return snapshot || post;
|
||||
}),
|
||||
);
|
||||
const draftListPublishedSnapshots = await Promise.all(
|
||||
draftListCandidates.map(async (post) => postEngine.getPublishedVersion(post.id)),
|
||||
);
|
||||
const allIds = new Set<string>();
|
||||
for (const p of publishedCandidates) allIds.add(p.id);
|
||||
for (const p of draftCandidates) allIds.add(p.id);
|
||||
|
||||
const publishedVersions = await resolvePublishedVersions(postEngine, Array.from(allIds));
|
||||
|
||||
const excludedCategorySet = new Set(listExcludedCategories);
|
||||
const isListExcluded = (post: PostData) =>
|
||||
excludedCategorySet.size > 0 && post.categories.some((c) => excludedCategorySet.has(c));
|
||||
|
||||
const publishedPostById = new Map<string, PostData>();
|
||||
for (const post of publishedSnapshots) {
|
||||
publishedPostById.set(post.id, post);
|
||||
}
|
||||
for (const snapshot of draftPublishedSnapshots) {
|
||||
if (snapshot) {
|
||||
publishedPostById.set(snapshot.id, snapshot);
|
||||
const publishedListPostById = new Map<string, PostData>();
|
||||
|
||||
for (const post of publishedCandidates) {
|
||||
const snapshot = publishedVersions.get(post.id) || post;
|
||||
publishedPostById.set(post.id, snapshot);
|
||||
if (!isListExcluded(post)) {
|
||||
publishedListPostById.set(post.id, snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
const publishedListPostById = new Map<string, PostData>();
|
||||
for (const post of publishedListSnapshots) {
|
||||
publishedListPostById.set(post.id, post);
|
||||
}
|
||||
for (const snapshot of draftListPublishedSnapshots) {
|
||||
for (const post of draftCandidates) {
|
||||
const snapshot = publishedVersions.get(post.id);
|
||||
if (snapshot) {
|
||||
publishedListPostById.set(snapshot.id, snapshot);
|
||||
publishedPostById.set(post.id, snapshot);
|
||||
if (!isListExcluded(post)) {
|
||||
publishedListPostById.set(post.id, snapshot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import path from 'node:path';
|
||||
import type { CategoryRenderSettings } from './PageRenderer';
|
||||
import { buildCanonicalPostPath } from './PageRenderer';
|
||||
import type { CategoryRenderSettings, HtmlRewriteContext } from './PageRenderer';
|
||||
import { buildCanonicalPostPath, mapToRecord } from './PageRenderer';
|
||||
import type { MenuDocument } from './MenuEngine';
|
||||
import type { ProjectMetadata } from './MetaEngine';
|
||||
import type { PostData } from './PostEngine';
|
||||
@@ -8,6 +8,7 @@ import type { PicoThemeName } from '../shared/picoThemes';
|
||||
import type { CategoryMetadata } from './BlogGenerationEngine';
|
||||
import { PreviewServer } from './PreviewServer';
|
||||
import type { PostTranslationData } from './PostEngine';
|
||||
import { readPostTranslationFile } from './postTranslationFileUtils';
|
||||
|
||||
interface RenderContext {
|
||||
projectContext: {
|
||||
@@ -49,11 +50,14 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
|
||||
projectName: string;
|
||||
projectDescription?: string;
|
||||
language?: string;
|
||||
blogLanguages?: string[];
|
||||
picoTheme?: PicoThemeName;
|
||||
categoryMetadata?: Record<string, CategoryMetadata>;
|
||||
categorySettings?: Record<string, CategoryRenderSettings>;
|
||||
menu?: MenuDocument;
|
||||
};
|
||||
/** The project's actual main language (for href_prefix computation). Defaults to options.language. */
|
||||
projectMainLanguage?: string;
|
||||
maxPostsPerPage: number;
|
||||
publishedPostsForLookup: PostData[];
|
||||
languagePrefix?: string;
|
||||
@@ -66,6 +70,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
|
||||
getPostTranslation?: (postId: string, language: string) => Promise<PostTranslationData | null>;
|
||||
hasPublishedVersion: (postId: string) => Promise<boolean>;
|
||||
getLinkedBy?: (postId: string) => Promise<{ id: string; title: string; slug: string }[]>;
|
||||
getAllBacklinks?: () => Promise<Map<string, { id: string; title: string; slug: string }[]>>;
|
||||
setProjectContext: (projectId: string, dataDir?: string) => void;
|
||||
};
|
||||
mediaEngine: {
|
||||
@@ -79,10 +84,13 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
|
||||
};
|
||||
};
|
||||
}): (pathname: string) => Promise<string | null> {
|
||||
const projectMainLanguage = params.projectMainLanguage ?? params.options.language;
|
||||
|
||||
const metadata: ProjectMetadata = {
|
||||
name: params.options.projectName,
|
||||
description: params.options.projectDescription,
|
||||
mainLanguage: params.options.language,
|
||||
mainLanguage: projectMainLanguage,
|
||||
blogLanguages: params.options.blogLanguages,
|
||||
maxPostsPerPage: params.maxPostsPerPage,
|
||||
picoTheme: params.options.picoTheme,
|
||||
categoryMetadata: params.options.categoryMetadata,
|
||||
@@ -166,26 +174,53 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
|
||||
return null;
|
||||
}
|
||||
|
||||
let match: PostData | undefined;
|
||||
if (!dateFilter) {
|
||||
return candidates[0] ?? null;
|
||||
match = candidates[0];
|
||||
} else {
|
||||
match = candidates.find((candidate) => {
|
||||
const createdAt = candidate.createdAt;
|
||||
return createdAt.getFullYear() === dateFilter.year
|
||||
&& createdAt.getMonth() === dateFilter.month - 1;
|
||||
});
|
||||
}
|
||||
|
||||
const match = candidates.find((candidate) => {
|
||||
const createdAt = candidate.createdAt;
|
||||
return createdAt.getFullYear() === dateFilter.year
|
||||
&& createdAt.getMonth() === dateFilter.month - 1;
|
||||
});
|
||||
if (!match) return null;
|
||||
|
||||
return match ?? null;
|
||||
// Lazily resolve content from file when needed
|
||||
if (!match.content) {
|
||||
const variant = match as PostData & { translationFilePath?: string };
|
||||
if (variant.translationFilePath) {
|
||||
const fileData = await readPostTranslationFile(variant.translationFilePath);
|
||||
if (fileData) {
|
||||
match.content = fileData.content;
|
||||
}
|
||||
} else {
|
||||
const full = await cachedPostEngine.getPublishedVersion(match.id);
|
||||
if (full) {
|
||||
match.content = full.content;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return match;
|
||||
},
|
||||
getPost: (postId: string) => params.engines.postEngine.getPost(postId),
|
||||
getPostTranslation: params.engines.postEngine.getPostTranslation
|
||||
? (postId: string, language: string) => params.engines.postEngine.getPostTranslation!(postId, language)
|
||||
: undefined,
|
||||
hasPublishedVersion: (postId: string) => params.engines.postEngine.hasPublishedVersion(postId),
|
||||
getLinkedBy: params.engines.postEngine.getLinkedBy
|
||||
? (postId: string) => params.engines.postEngine.getLinkedBy!(postId)
|
||||
: undefined,
|
||||
getLinkedBy: params.engines.postEngine.getAllBacklinks
|
||||
? (() => {
|
||||
const backlinksCachePromise = params.engines.postEngine.getAllBacklinks!();
|
||||
return async (postId: string) => {
|
||||
const backlinksMap = await backlinksCachePromise;
|
||||
return backlinksMap.get(postId) ?? [];
|
||||
};
|
||||
})()
|
||||
: params.engines.postEngine.getLinkedBy
|
||||
? (postId: string) => params.engines.postEngine.getLinkedBy!(postId)
|
||||
: undefined,
|
||||
setProjectContext: (projectId: string, dataDir?: string) => {
|
||||
params.engines.postEngine.setProjectContext(projectId, dataDir);
|
||||
},
|
||||
@@ -224,7 +259,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
|
||||
userTemplatesDir: path.join(params.options.dataDir, 'templates'),
|
||||
});
|
||||
|
||||
const htmlRewriteContextPromise: Promise<{ canonicalPostPathBySlug: Map<string, string>; canonicalMediaPathBySourcePath: Map<string, string>; languagePrefix?: string }> = (async () => {
|
||||
const htmlRewriteContextPromise: Promise<HtmlRewriteContext> = (async () => {
|
||||
const canonicalPostPathBySlug = new Map<string, string>();
|
||||
for (const post of params.publishedPostsForLookup) {
|
||||
canonicalPostPathBySlug.set(post.slug, buildCanonicalPostPath(post));
|
||||
@@ -247,6 +282,8 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
|
||||
return {
|
||||
canonicalPostPathBySlug,
|
||||
canonicalMediaPathBySourcePath,
|
||||
canonicalPostPathBySlugRecord: mapToRecord(canonicalPostPathBySlug),
|
||||
canonicalMediaPathBySourcePathRecord: mapToRecord(canonicalMediaPathBySourcePath),
|
||||
languagePrefix: params.languagePrefix,
|
||||
};
|
||||
})();
|
||||
@@ -255,6 +292,7 @@ export function createPreviewBackedGenerationRouteRenderer(params: {
|
||||
renderWithContext: async (pathname, context) => previewServer.renderRouteForContext(pathname, {
|
||||
...context,
|
||||
htmlRewriteContext: await htmlRewriteContextPromise,
|
||||
preferredLanguage: params.options.language,
|
||||
}),
|
||||
context: {
|
||||
projectContext,
|
||||
|
||||
327
src/main/engine/GenerationWorkerData.ts
Normal file
327
src/main/engine/GenerationWorkerData.ts
Normal file
@@ -0,0 +1,327 @@
|
||||
/**
|
||||
* Serialization types and utilities for passing data to generation worker threads.
|
||||
*
|
||||
* Worker threads cannot receive non-serializable objects (Date, Map, etc.) via
|
||||
* the structured-clone algorithm used by workerData / postMessage. This module
|
||||
* provides a thin serialization layer that converts Dates to ISO strings and
|
||||
* Maps to arrays-of-tuples so the data survives the boundary.
|
||||
*/
|
||||
import type { PostData } from './PostEngine';
|
||||
import type { MediaData } from './MediaEngine';
|
||||
import type { CategoryMetadata, BlogGenerationOptions, BlogGenerationSection } from './BlogGenerationEngine';
|
||||
import type { CategoryRenderSettings } from './PageRenderer';
|
||||
import type { MenuDocument } from './MenuEngine';
|
||||
import type { PicoThemeName } from '../shared/picoThemes';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Serialized post (Dates → ISO strings)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SerializedPostData {
|
||||
id: string;
|
||||
projectId: string;
|
||||
title: string;
|
||||
slug: string;
|
||||
excerpt?: string;
|
||||
content: string;
|
||||
status: 'draft' | 'published' | 'archived';
|
||||
author?: string;
|
||||
language?: string;
|
||||
doNotTranslate?: boolean;
|
||||
templateSlug?: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
publishedAt?: string;
|
||||
tags: string[];
|
||||
categories: string[];
|
||||
availableLanguages: string[];
|
||||
translationSourceSlug?: string;
|
||||
translationCanonicalLanguage?: string;
|
||||
translationFilePath?: string;
|
||||
/** Absolute file path for canonical posts (for lazy content loading in workers). */
|
||||
filePath?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Serialized media item (only what generation needs)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SerializedMediaData {
|
||||
id: string;
|
||||
filename: string;
|
||||
originalName: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
title?: string;
|
||||
alt?: string;
|
||||
caption?: string;
|
||||
author?: string;
|
||||
language?: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
tags: string[];
|
||||
linkedPostIds?: string[];
|
||||
availableLanguages: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Serialized options (stripped to what the worker actually uses)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SerializedBlogGenerationOptions {
|
||||
projectId: string;
|
||||
projectName: string;
|
||||
projectDescription?: string;
|
||||
dataDir: string;
|
||||
baseUrl: string;
|
||||
language?: string;
|
||||
blogLanguages?: string[];
|
||||
picoTheme?: PicoThemeName;
|
||||
categoryMetadata?: Record<string, CategoryMetadata>;
|
||||
categorySettings?: Record<string, CategoryRenderSettings>;
|
||||
menu?: MenuDocument;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Worker task — everything a worker needs to process one section
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface GenerationWorkerTask {
|
||||
taskId: string;
|
||||
section: BlogGenerationSection;
|
||||
|
||||
/** Posts relevant to this task (single: chunk, others: all list posts). */
|
||||
posts: SerializedPostData[];
|
||||
|
||||
/**
|
||||
* All published route posts — used by the route renderer for slug lookups,
|
||||
* canonical path building, and backlink resolution.
|
||||
*/
|
||||
lookupPosts: SerializedPostData[];
|
||||
|
||||
/** Media items for canonical media path building. */
|
||||
mediaItems: SerializedMediaData[];
|
||||
|
||||
/** Pre-resolved backlinks map: postId → linked-by entries. */
|
||||
backlinksMap: Record<string, Array<{ id: string; title: string; slug: string }>>;
|
||||
|
||||
options: SerializedBlogGenerationOptions;
|
||||
maxPostsPerPage: number;
|
||||
htmlDir: string;
|
||||
|
||||
/** Pre-loaded hash map as [relativePath, contentHash] tuples. */
|
||||
hashMapEntries: Array<[string, string]>;
|
||||
|
||||
/** Post file paths for lazy content loading: [postId, absoluteFilePath] tuples. */
|
||||
postFilePathEntries: Array<[string, string]>;
|
||||
|
||||
/** Post-media links: [postId, [{mediaId, sortOrder}]] tuples for gallery/album macros. */
|
||||
postMediaLinksEntries: Array<[string, Array<{ mediaId: string; sortOrder: number }>]>;
|
||||
|
||||
/** Language prefix for subtree generation, e.g. "/fr". */
|
||||
languagePrefix?: string;
|
||||
|
||||
/** The project's actual main language (for href_prefix computation in subtree rendering). */
|
||||
mainLanguage?: string;
|
||||
|
||||
// -- Section-specific data (category / tag / date) -----------------------
|
||||
|
||||
allCategories?: string[];
|
||||
allTags?: string[];
|
||||
|
||||
/** Serialized Maps as arrays of tuples. */
|
||||
yearsEntries?: Array<[number, string]>;
|
||||
yearMonthsEntries?: Array<[string, string]>;
|
||||
yearMonthDaysEntries?: Array<[string, string]>;
|
||||
|
||||
/** Pre-built post index (avoids rebuilding in each worker). */
|
||||
postsByCategoryEntries?: Array<[string, SerializedPostData[]]>;
|
||||
postsByTagEntries?: Array<[string, SerializedPostData[]]>;
|
||||
postsByYearEntries?: Array<[number, SerializedPostData[]]>;
|
||||
postsByYearMonthEntries?: Array<[string, SerializedPostData[]]>;
|
||||
postsByYearMonthDayEntries?: Array<[string, SerializedPostData[]]>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Messages between main thread and worker
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface WorkerProgressMessage {
|
||||
type: 'progress';
|
||||
taskId: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface WorkerResultMessage {
|
||||
type: 'result';
|
||||
taskId: string;
|
||||
pagesGenerated: number;
|
||||
/** Hash updates accumulated during generation, to be written by the main thread. */
|
||||
hashUpdates: Array<{ relativePath: string; hash: string }>;
|
||||
}
|
||||
|
||||
export interface WorkerErrorMessage {
|
||||
type: 'error';
|
||||
taskId: string;
|
||||
error: string;
|
||||
}
|
||||
|
||||
export type WorkerOutboundMessage =
|
||||
| WorkerProgressMessage
|
||||
| WorkerResultMessage
|
||||
| WorkerErrorMessage;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Serialization helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function serializePostData(post: PostData): SerializedPostData {
|
||||
const serialized: SerializedPostData = {
|
||||
id: post.id,
|
||||
projectId: post.projectId,
|
||||
title: post.title,
|
||||
slug: post.slug,
|
||||
excerpt: post.excerpt,
|
||||
content: post.content ?? '',
|
||||
status: post.status,
|
||||
author: post.author,
|
||||
language: post.language,
|
||||
doNotTranslate: post.doNotTranslate,
|
||||
templateSlug: post.templateSlug,
|
||||
createdAt: post.createdAt instanceof Date ? post.createdAt.toISOString() : String(post.createdAt),
|
||||
updatedAt: post.updatedAt instanceof Date ? post.updatedAt.toISOString() : String(post.updatedAt),
|
||||
publishedAt: post.publishedAt instanceof Date ? post.publishedAt.toISOString() : post.publishedAt ? String(post.publishedAt) : undefined,
|
||||
tags: post.tags ?? [],
|
||||
categories: post.categories ?? [],
|
||||
availableLanguages: post.availableLanguages ?? [],
|
||||
};
|
||||
|
||||
// Preserve translation variant fields if present
|
||||
const variant = post as PostData & {
|
||||
translationSourceSlug?: string;
|
||||
translationCanonicalLanguage?: string;
|
||||
translationFilePath?: string;
|
||||
};
|
||||
if (variant.translationSourceSlug) serialized.translationSourceSlug = variant.translationSourceSlug;
|
||||
if (variant.translationCanonicalLanguage) serialized.translationCanonicalLanguage = variant.translationCanonicalLanguage;
|
||||
if (variant.translationFilePath) serialized.translationFilePath = variant.translationFilePath;
|
||||
|
||||
return serialized;
|
||||
}
|
||||
|
||||
export function deserializePostData(serialized: SerializedPostData): PostData {
|
||||
const post: PostData = {
|
||||
id: serialized.id,
|
||||
projectId: serialized.projectId,
|
||||
title: serialized.title,
|
||||
slug: serialized.slug,
|
||||
excerpt: serialized.excerpt,
|
||||
content: serialized.content ?? '',
|
||||
status: serialized.status,
|
||||
author: serialized.author,
|
||||
language: serialized.language,
|
||||
doNotTranslate: serialized.doNotTranslate,
|
||||
templateSlug: serialized.templateSlug,
|
||||
createdAt: new Date(serialized.createdAt),
|
||||
updatedAt: new Date(serialized.updatedAt),
|
||||
publishedAt: serialized.publishedAt ? new Date(serialized.publishedAt) : undefined,
|
||||
tags: serialized.tags ?? [],
|
||||
categories: serialized.categories ?? [],
|
||||
availableLanguages: serialized.availableLanguages ?? [],
|
||||
};
|
||||
|
||||
// Re-attach translation variant fields
|
||||
if (serialized.translationSourceSlug) {
|
||||
(post as any).translationSourceSlug = serialized.translationSourceSlug;
|
||||
}
|
||||
if (serialized.translationCanonicalLanguage) {
|
||||
(post as any).translationCanonicalLanguage = serialized.translationCanonicalLanguage;
|
||||
}
|
||||
if (serialized.translationFilePath) {
|
||||
(post as any).translationFilePath = serialized.translationFilePath;
|
||||
}
|
||||
|
||||
return post;
|
||||
}
|
||||
|
||||
export function serializeMediaItem(media: MediaData): SerializedMediaData {
|
||||
return {
|
||||
id: media.id,
|
||||
filename: media.filename,
|
||||
originalName: media.originalName,
|
||||
mimeType: media.mimeType,
|
||||
size: media.size,
|
||||
width: media.width,
|
||||
height: media.height,
|
||||
title: media.title,
|
||||
alt: media.alt,
|
||||
caption: media.caption,
|
||||
author: media.author,
|
||||
language: media.language,
|
||||
createdAt: media.createdAt instanceof Date ? media.createdAt.toISOString() : String(media.createdAt),
|
||||
updatedAt: media.updatedAt instanceof Date ? media.updatedAt.toISOString() : String(media.updatedAt),
|
||||
tags: media.tags ?? [],
|
||||
linkedPostIds: media.linkedPostIds,
|
||||
availableLanguages: media.availableLanguages ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
export function deserializeMediaItem(serialized: SerializedMediaData): MediaData {
|
||||
return {
|
||||
id: serialized.id,
|
||||
filename: serialized.filename,
|
||||
originalName: serialized.originalName,
|
||||
mimeType: serialized.mimeType,
|
||||
size: serialized.size,
|
||||
width: serialized.width,
|
||||
height: serialized.height,
|
||||
title: serialized.title,
|
||||
alt: serialized.alt,
|
||||
caption: serialized.caption,
|
||||
author: serialized.author,
|
||||
language: serialized.language,
|
||||
createdAt: new Date(serialized.createdAt),
|
||||
updatedAt: new Date(serialized.updatedAt),
|
||||
tags: serialized.tags ?? [],
|
||||
linkedPostIds: serialized.linkedPostIds,
|
||||
availableLanguages: serialized.availableLanguages ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeBlogGenerationOptions(options: BlogGenerationOptions): SerializedBlogGenerationOptions {
|
||||
return {
|
||||
projectId: options.projectId,
|
||||
projectName: options.projectName,
|
||||
projectDescription: options.projectDescription,
|
||||
dataDir: options.dataDir,
|
||||
baseUrl: options.baseUrl,
|
||||
language: options.language,
|
||||
blogLanguages: options.blogLanguages,
|
||||
picoTheme: options.picoTheme,
|
||||
categoryMetadata: options.categoryMetadata,
|
||||
categorySettings: options.categorySettings,
|
||||
menu: options.menu,
|
||||
};
|
||||
}
|
||||
|
||||
/** Serialize a Map<K, PostData[]> to an array of [K, SerializedPostData[]] tuples. */
|
||||
export function serializePostMap<K extends string | number>(map: Map<K, PostData[]>): Array<[K, SerializedPostData[]]> {
|
||||
return Array.from(map.entries()).map(([key, posts]) => [key, posts.map(serializePostData)]);
|
||||
}
|
||||
|
||||
/** Deserialize an array of [K, SerializedPostData[]] tuples to a Map<K, PostData[]>. */
|
||||
export function deserializePostMap<K extends string | number>(entries: Array<[K, SerializedPostData[]]>): Map<K, PostData[]> {
|
||||
return new Map(entries.map(([key, posts]) => [key, posts.map(deserializePostData)]));
|
||||
}
|
||||
|
||||
/** Serialize a Map<K, Date> to an array of [K, string] tuples. */
|
||||
export function serializeDateMap<K extends string | number>(map: Map<K, Date>): Array<[K, string]> {
|
||||
return Array.from(map.entries()).map(([key, date]) => [key, date.toISOString()]);
|
||||
}
|
||||
|
||||
/** Deserialize an array of [K, string] tuples to a Map<K, Date>. */
|
||||
export function deserializeDateMap<K extends string | number>(entries: Array<[K, string]>): Map<K, Date> {
|
||||
return new Map(entries.map(([key, iso]) => [key, new Date(iso)]));
|
||||
}
|
||||
139
src/main/engine/GenerationWorkerPool.ts
Normal file
139
src/main/engine/GenerationWorkerPool.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Worker pool for parallel blog generation.
|
||||
*
|
||||
* Manages a pool of worker threads that render HTML pages concurrently.
|
||||
* Each worker gets a self-contained GenerationWorkerTask and produces pages
|
||||
* independently. The pool limits concurrency to os.cpus().length - 1 (min 1).
|
||||
*/
|
||||
import { Worker } from 'worker_threads';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import type {
|
||||
GenerationWorkerTask,
|
||||
WorkerOutboundMessage,
|
||||
} from './GenerationWorkerData';
|
||||
|
||||
export interface WorkerPoolOptions {
|
||||
/** Max concurrent workers. Defaults to os.cpus().length - 1, min 1. */
|
||||
maxWorkers?: number;
|
||||
/** Override the worker script path (for testing). */
|
||||
workerPath?: string;
|
||||
}
|
||||
|
||||
export interface WorkerPoolResult {
|
||||
pagesGenerated: number;
|
||||
errors: Array<{ taskId: string; error: string }>;
|
||||
hashUpdates: Array<{ relativePath: string; hash: string }>;
|
||||
}
|
||||
|
||||
export type WorkerFactory = (workerPath: string, workerData: GenerationWorkerTask) => WorkerLike;
|
||||
|
||||
export interface WorkerLike {
|
||||
on(event: string, listener: (...args: unknown[]) => void): void;
|
||||
terminate(): Promise<number>;
|
||||
removeAllListeners(): void;
|
||||
}
|
||||
|
||||
export class GenerationWorkerPool {
|
||||
private readonly maxWorkers: number;
|
||||
private readonly workerPath: string;
|
||||
private readonly workerFactory: WorkerFactory;
|
||||
|
||||
constructor(options?: WorkerPoolOptions, workerFactory?: WorkerFactory) {
|
||||
this.maxWorkers = Math.max(1, options?.maxWorkers ?? (os.cpus().length - 1));
|
||||
this.workerPath = options?.workerPath ?? path.join(__dirname, 'generation.worker.js');
|
||||
this.workerFactory = workerFactory ?? ((wp, wd) => new Worker(wp, { workerData: wd }) as unknown as WorkerLike);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a set of generation tasks across the worker pool.
|
||||
*
|
||||
* Tasks are distributed to workers up to maxWorkers concurrency.
|
||||
* When a worker finishes, the next queued task is dispatched.
|
||||
*
|
||||
* @param tasks - Array of self-contained worker tasks
|
||||
* @param onProgress - Called for each page generated (for progress bar updates)
|
||||
* @returns Merged results from all workers
|
||||
*/
|
||||
async runTasks(
|
||||
tasks: GenerationWorkerTask[],
|
||||
onProgress: (message: string) => void,
|
||||
): Promise<WorkerPoolResult> {
|
||||
if (tasks.length === 0) {
|
||||
return { pagesGenerated: 0, errors: [], hashUpdates: [] };
|
||||
}
|
||||
|
||||
return new Promise<WorkerPoolResult>((resolve) => {
|
||||
let totalPages = 0;
|
||||
const errors: Array<{ taskId: string; error: string }> = [];
|
||||
const allHashUpdates: Array<{ relativePath: string; hash: string }> = [];
|
||||
let nextTaskIndex = 0;
|
||||
let activeWorkers = 0;
|
||||
|
||||
const startNextWorker = () => {
|
||||
if (nextTaskIndex >= tasks.length) {
|
||||
if (activeWorkers === 0) {
|
||||
resolve({ pagesGenerated: totalPages, errors, hashUpdates: allHashUpdates });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const task = tasks[nextTaskIndex++];
|
||||
activeWorkers++;
|
||||
|
||||
const worker = this.workerFactory(this.workerPath, task);
|
||||
|
||||
worker.on('message', (raw: unknown) => {
|
||||
const msg = raw as WorkerOutboundMessage;
|
||||
|
||||
switch (msg.type) {
|
||||
case 'progress':
|
||||
onProgress(msg.message);
|
||||
break;
|
||||
|
||||
case 'result':
|
||||
totalPages += msg.pagesGenerated;
|
||||
if (msg.hashUpdates) {
|
||||
allHashUpdates.push(...msg.hashUpdates);
|
||||
}
|
||||
activeWorkers--;
|
||||
void worker.terminate();
|
||||
startNextWorker();
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
errors.push({ taskId: msg.taskId, error: msg.error });
|
||||
activeWorkers--;
|
||||
void worker.terminate();
|
||||
startNextWorker();
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
worker.on('error', (err: unknown) => {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
errors.push({ taskId: task.taskId, error: errorMessage });
|
||||
activeWorkers--;
|
||||
startNextWorker();
|
||||
});
|
||||
|
||||
worker.on('exit', (code: unknown) => {
|
||||
// If the worker exited unexpectedly (no result/error message received),
|
||||
// we need to account for it. The 'error' handler above covers crashes.
|
||||
// Exit code 0 is normal (worker finished). Non-zero without error handler
|
||||
// means something unexpected happened.
|
||||
if (typeof code === 'number' && code !== 0) {
|
||||
// Only handle if we haven't already decremented (check via activeWorkers)
|
||||
// This is a safety net — most crashes are caught by the 'error' event.
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Start initial batch of workers
|
||||
const initialBatch = Math.min(this.maxWorkers, tasks.length);
|
||||
for (let i = 0; i < initialBatch; i++) {
|
||||
startNextWorker();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,10 @@ export interface HtmlRewriteContext {
|
||||
canonicalPostPathBySlug: Map<string, string>;
|
||||
canonicalMediaPathBySourcePath: Map<string, string>;
|
||||
languagePrefix?: string;
|
||||
/** Pre-computed Record version of canonicalPostPathBySlug (avoids repeated Map→Object conversion) */
|
||||
canonicalPostPathBySlugRecord?: Record<string, string>;
|
||||
/** Pre-computed Record version of canonicalMediaPathBySourcePath */
|
||||
canonicalMediaPathBySourcePathRecord?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface TemplatePostEntry {
|
||||
@@ -1445,8 +1449,8 @@ export class PageRenderer {
|
||||
has_next_page: hasNextPage,
|
||||
prev_page_href: prevPageHref,
|
||||
next_page_href: nextPageHref,
|
||||
canonical_post_path_by_slug: mapToRecord(rewriteContext.canonicalPostPathBySlug),
|
||||
canonical_media_path_by_source_path: mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
|
||||
canonical_post_path_by_slug: rewriteContext.canonicalPostPathBySlugRecord ?? mapToRecord(rewriteContext.canonicalPostPathBySlug),
|
||||
canonical_media_path_by_source_path: rewriteContext.canonicalMediaPathBySourcePathRecord ?? mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
|
||||
post_data_json_by_id: Object.fromEntries(
|
||||
posts.map((post) => [post.id, JSON.stringify(serializePostDataForMacro(post))]),
|
||||
),
|
||||
@@ -1572,7 +1576,7 @@ export class PageRenderer {
|
||||
? Array.from(new Set(renderablePost.tags.map((tag) => tag.trim()).filter((tag) => tag.length > 0)))
|
||||
: [];
|
||||
|
||||
const canonicalPostPathBySlug = mapToRecord(rewriteContext.canonicalPostPathBySlug);
|
||||
const canonicalPostPathBySlug = rewriteContext.canonicalPostPathBySlugRecord ?? mapToRecord(rewriteContext.canonicalPostPathBySlug);
|
||||
|
||||
// Per-post language overrides the page-level language when present
|
||||
const postLanguage = (renderablePost as { language?: string }).language;
|
||||
@@ -1598,7 +1602,7 @@ export class PageRenderer {
|
||||
calendar_initial_year: renderablePost.createdAt.getFullYear(),
|
||||
calendar_initial_month: renderablePost.createdAt.getMonth() + 1,
|
||||
canonical_post_path_by_slug: canonicalPostPathBySlug,
|
||||
canonical_media_path_by_source_path: mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
|
||||
canonical_media_path_by_source_path: rewriteContext.canonicalMediaPathBySourcePathRecord ?? mapToRecord(rewriteContext.canonicalMediaPathBySourcePath),
|
||||
post_data_json_by_id: {
|
||||
[renderablePost.id]: JSON.stringify(serializePostDataForMacro(renderablePost)),
|
||||
},
|
||||
|
||||
@@ -1077,6 +1077,40 @@ export class PostEngine extends EventEmitter {
|
||||
return result;
|
||||
}
|
||||
|
||||
async getPublishedTranslationsForRoutePosts(publishedPosts: PostData[]): Promise<Map<string, PostTranslationData[]>> {
|
||||
const allRows = await this.getAllTranslationRows();
|
||||
const postById = new Map(publishedPosts.map((p) => [p.id, p]));
|
||||
const result = new Map<string, PostTranslationData[]>();
|
||||
|
||||
for (const row of allRows) {
|
||||
if (row.status !== 'published') continue;
|
||||
const sourcePost = postById.get(row.translationFor);
|
||||
if (!sourcePost) continue;
|
||||
if (this.isCanonicalTranslationLanguage(sourcePost, row.language)) continue;
|
||||
|
||||
const translationData: PostTranslationData = {
|
||||
id: row.id,
|
||||
projectId: row.projectId,
|
||||
translationFor: row.translationFor,
|
||||
language: row.language,
|
||||
title: row.title,
|
||||
excerpt: row.excerpt || undefined,
|
||||
content: '',
|
||||
status: row.status as 'draft' | 'published' | 'archived',
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
publishedAt: row.publishedAt || undefined,
|
||||
filePath: row.filePath,
|
||||
};
|
||||
|
||||
const arr = result.get(row.translationFor) || [];
|
||||
arr.push(translationData);
|
||||
result.set(row.translationFor, arr);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async getPostTranslations(postId: string): Promise<PostTranslationData[]> {
|
||||
const sourcePost = await this.getPost(postId);
|
||||
const rows = this.filterCanonicalTranslationRows(sourcePost, await this.getTranslationRowsForPost(postId));
|
||||
@@ -2504,6 +2538,44 @@ export class PostEngine extends EventEmitter {
|
||||
};
|
||||
}
|
||||
|
||||
async getPublishedVersionsBulk(ids: string[]): Promise<Map<string, PostData>> {
|
||||
const result = new Map<string, PostData>();
|
||||
if (ids.length === 0) return result;
|
||||
|
||||
const db = getDatabase().getLocal();
|
||||
const idSet = new Set(ids);
|
||||
|
||||
const dbPosts = await db.select().from(posts)
|
||||
.where(eq(posts.projectId, this.currentProjectId))
|
||||
.all();
|
||||
|
||||
for (const dbPost of dbPosts) {
|
||||
if (!idSet.has(dbPost.id) || !dbPost.filePath) continue;
|
||||
result.set(dbPost.id, this.dbRowToPostData(dbPost, ''));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk-load file paths for all published posts in the current project.
|
||||
* Returns a Map from postId → absolute filePath.
|
||||
*/
|
||||
async getPublishedPostFilePaths(): Promise<Map<string, string>> {
|
||||
const db = getDatabase().getLocal();
|
||||
const dbPosts = await db.select().from(posts)
|
||||
.where(eq(posts.projectId, this.currentProjectId))
|
||||
.all();
|
||||
|
||||
const result = new Map<string, string>();
|
||||
for (const dbPost of dbPosts) {
|
||||
if (dbPost.filePath) {
|
||||
result.set(dbPost.id, dbPost.filePath);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild the FTS index for all posts in the current project.
|
||||
* Call this after changing the search language or after migration.
|
||||
@@ -3089,6 +3161,48 @@ export class PostEngine extends EventEmitter {
|
||||
return sourcePosts.filter(p => sourceIds.includes(p.id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk-load all backlinks for all posts in the current project.
|
||||
* Returns a Map from targetPostId → array of source posts that link to it.
|
||||
* Much more efficient than calling getLinkedBy per post during generation.
|
||||
*/
|
||||
async getAllBacklinks(): Promise<Map<string, { id: string; title: string; slug: string }[]>> {
|
||||
const db = getDatabase().getLocal();
|
||||
|
||||
const allLinks = await db
|
||||
.select({
|
||||
sourcePostId: postLinks.sourcePostId,
|
||||
targetPostId: postLinks.targetPostId,
|
||||
})
|
||||
.from(postLinks);
|
||||
|
||||
if (allLinks.length === 0) return new Map();
|
||||
|
||||
const sourceIds = new Set(allLinks.map(l => l.sourcePostId));
|
||||
const allSourcePosts = await db
|
||||
.select({ id: posts.id, title: posts.title, slug: posts.slug })
|
||||
.from(posts)
|
||||
.where(eq(posts.projectId, this.currentProjectId));
|
||||
|
||||
const sourcePostById = new Map(
|
||||
allSourcePosts.filter(p => sourceIds.has(p.id)).map(p => [p.id, p]),
|
||||
);
|
||||
|
||||
const result = new Map<string, { id: string; title: string; slug: string }[]>();
|
||||
for (const link of allLinks) {
|
||||
const sourcePost = sourcePostById.get(link.sourcePostId);
|
||||
if (!sourcePost) continue;
|
||||
const existing = result.get(link.targetPostId);
|
||||
if (existing) {
|
||||
existing.push(sourcePost);
|
||||
} else {
|
||||
result.set(link.targetPostId, [sourcePost]);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get posts that the specified post links TO ("links to")
|
||||
*/
|
||||
|
||||
@@ -394,6 +394,32 @@ export class PostMediaEngine extends EventEmitter {
|
||||
return link.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all post-media links for the current project, grouped by post ID.
|
||||
* Used to pre-load data for generation workers.
|
||||
*/
|
||||
async getAllPostMediaLinks(): Promise<Map<string, Array<{ mediaId: string; sortOrder: number }>>> {
|
||||
const db = this.getDb();
|
||||
|
||||
const links = await db
|
||||
.select()
|
||||
.from(postMedia)
|
||||
.where(eq(postMedia.projectId, this.currentProjectId))
|
||||
.orderBy(asc(postMedia.sortOrder));
|
||||
|
||||
const result = new Map<string, Array<{ mediaId: string; sortOrder: number }>>();
|
||||
for (const link of links) {
|
||||
const existing = result.get(link.postId);
|
||||
const entry = { mediaId: link.mediaId, sortOrder: link.sortOrder };
|
||||
if (existing) {
|
||||
existing.push(entry);
|
||||
} else {
|
||||
result.set(link.postId, [entry]);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map database row to PostMediaLinkData
|
||||
*/
|
||||
|
||||
@@ -72,18 +72,22 @@ export async function generateSinglePostPages(params: BaseParams & {
|
||||
posts: PostData[];
|
||||
}): Promise<number> {
|
||||
let count = 0;
|
||||
const BATCH_SIZE = 10;
|
||||
|
||||
for (const post of params.posts) {
|
||||
const createdAt = resolvePostCreatedAt(post);
|
||||
const year = createdAt.getFullYear();
|
||||
const month = String(createdAt.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(createdAt.getDate()).padStart(2, '0');
|
||||
for (let i = 0; i < params.posts.length; i += BATCH_SIZE) {
|
||||
const batch = params.posts.slice(i, i + BATCH_SIZE);
|
||||
const results = await Promise.all(batch.map(async (post) => {
|
||||
const createdAt = resolvePostCreatedAt(post);
|
||||
const year = createdAt.getFullYear();
|
||||
const month = String(createdAt.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(createdAt.getDate()).padStart(2, '0');
|
||||
|
||||
const urlPath = `${year}/${month}/${day}/${post.slug}`;
|
||||
const html = await renderRequiredRoute(params.renderRoute, `/${urlPath}`);
|
||||
await params.writePage(params.projectId, urlPath, html);
|
||||
count++;
|
||||
params.onPageGenerated(`Generated /${urlPath}`);
|
||||
const urlPath = `${year}/${month}/${day}/${post.slug}`;
|
||||
const html = await renderRequiredRoute(params.renderRoute, `/${urlPath}`);
|
||||
await params.writePage(params.projectId, urlPath, html);
|
||||
params.onPageGenerated(`Generated /${urlPath}`);
|
||||
}));
|
||||
count += results.length;
|
||||
}
|
||||
|
||||
return count;
|
||||
|
||||
@@ -109,7 +109,7 @@ export const CALENDAR_RUNTIME_JS = String.raw`(() => {
|
||||
}
|
||||
|
||||
async function loadCalendarData() {
|
||||
const response = await fetch(languagePrefix + '/calendar.json', { cache: 'no-store' });
|
||||
const response = await fetch('/calendar.json', { cache: 'no-store' });
|
||||
if (!response.ok) {
|
||||
throw new Error('calendar.json request failed');
|
||||
}
|
||||
|
||||
287
src/main/engine/generation.worker.ts
Normal file
287
src/main/engine/generation.worker.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Worker thread entry point for parallel blog generation.
|
||||
*
|
||||
* Each worker receives a GenerationWorkerTask via workerData, creates its own
|
||||
* rendering pipeline (Liquid, PageRenderer, PreviewServer, route renderer) and
|
||||
* renders the assigned pages, writing them to the filesystem.
|
||||
*
|
||||
* Workers do NOT open database connections. Hash reads come from a pre-loaded
|
||||
* map passed in task data. Hash writes are accumulated in memory and sent back
|
||||
* to the main thread in the result message for the main thread to persist.
|
||||
*/
|
||||
import { parentPort, workerData } from 'worker_threads';
|
||||
import type {
|
||||
GenerationWorkerTask,
|
||||
WorkerOutboundMessage,
|
||||
SerializedPostData,
|
||||
} from './GenerationWorkerData';
|
||||
import {
|
||||
deserializePostData,
|
||||
deserializeMediaItem,
|
||||
deserializePostMap,
|
||||
deserializeDateMap,
|
||||
} from './GenerationWorkerData';
|
||||
import {
|
||||
createDataBackedPostEngine,
|
||||
createDataBackedMediaEngine,
|
||||
createDataBackedPostMediaEngine,
|
||||
} from './DataBackedEngines';
|
||||
import { createPreviewBackedGenerationRouteRenderer } from './GenerationRouteRendererFactory';
|
||||
import {
|
||||
generateSinglePostPages,
|
||||
generateCategoryPages,
|
||||
generateTagPages,
|
||||
generateDateArchivePages,
|
||||
generateRootPages,
|
||||
generatePageRoutes,
|
||||
} from './RoutePageGenerationService';
|
||||
import { writeHtmlPage } from './BlogGenerationOutputService';
|
||||
import type { PostData } from './PostEngine';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// In-memory hash store (no DB access)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Creates a purely in-memory hash store.
|
||||
* Reads come from the pre-loaded hash map (passed from main thread).
|
||||
* Writes are accumulated in `pendingUpdates` and returned to the main thread
|
||||
* via the result message so it can persist them in a single connection.
|
||||
*/
|
||||
function createWorkerHashStore(hashCache: Map<string, string | null>) {
|
||||
const pendingUpdates: Array<{ relativePath: string; hash: string }> = [];
|
||||
|
||||
return {
|
||||
async get(_projectId: string, relativePath: string): Promise<string | null> {
|
||||
return hashCache.get(relativePath) ?? null;
|
||||
},
|
||||
|
||||
async set(_projectId: string, relativePath: string, hash: string): Promise<void> {
|
||||
pendingUpdates.push({ relativePath, hash });
|
||||
hashCache.set(relativePath, hash);
|
||||
},
|
||||
|
||||
pendingUpdates,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function send(message: WorkerOutboundMessage): void {
|
||||
parentPort?.postMessage(message);
|
||||
}
|
||||
|
||||
function deserializePostArray(serialized: SerializedPostData[]): PostData[] {
|
||||
return serialized.map(deserializePostData);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main worker logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function run(): Promise<void> {
|
||||
const task = workerData as GenerationWorkerTask;
|
||||
|
||||
try {
|
||||
// 1. Reconstruct hash cache from pre-loaded entries (no DB needed)
|
||||
const hashCache = new Map<string, string | null>();
|
||||
for (const [relativePath, hash] of task.hashMapEntries) {
|
||||
hashCache.set(relativePath, hash);
|
||||
}
|
||||
const hashStore = createWorkerHashStore(hashCache);
|
||||
|
||||
// 2. Deserialize post data
|
||||
const posts = deserializePostArray(task.posts);
|
||||
const lookupPosts = deserializePostArray(task.lookupPosts);
|
||||
const mediaItems = (task.mediaItems ?? []).map(deserializeMediaItem);
|
||||
|
||||
// 2b. Reconstruct post file paths for lazy content loading
|
||||
const postFilePaths = new Map<string, string>();
|
||||
if (task.postFilePathEntries) {
|
||||
for (const [postId, filePath] of task.postFilePathEntries) {
|
||||
postFilePaths.set(postId, filePath);
|
||||
}
|
||||
}
|
||||
|
||||
// 2c. Reconstruct post-media links for gallery/album macros
|
||||
const postMediaLinks = new Map<string, Array<{ mediaId: string; sortOrder: number }>>();
|
||||
if (task.postMediaLinksEntries) {
|
||||
for (const [postId, links] of task.postMediaLinksEntries) {
|
||||
postMediaLinks.set(postId, links);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Reconstruct backlinks Map
|
||||
const backlinksMap = new Map<string, Array<{ id: string; title: string; slug: string }>>();
|
||||
if (task.backlinksMap) {
|
||||
for (const [postId, links] of Object.entries(task.backlinksMap)) {
|
||||
backlinksMap.set(postId, links);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Create data-backed engines
|
||||
const postEngine = createDataBackedPostEngine({ allPosts: lookupPosts, backlinksMap, postFilePaths });
|
||||
const mediaEngine = createDataBackedMediaEngine(mediaItems);
|
||||
const postMediaEngine = createDataBackedPostMediaEngine({ mediaItems, postMediaLinks });
|
||||
|
||||
// 5. Create route renderer (same factory as main thread, but backed by data)
|
||||
const renderRoute = createPreviewBackedGenerationRouteRenderer({
|
||||
options: {
|
||||
...task.options,
|
||||
language: task.options.language,
|
||||
},
|
||||
projectMainLanguage: task.mainLanguage,
|
||||
maxPostsPerPage: task.maxPostsPerPage,
|
||||
publishedPostsForLookup: lookupPosts,
|
||||
languagePrefix: task.languagePrefix,
|
||||
engines: {
|
||||
postEngine: postEngine as any,
|
||||
mediaEngine: mediaEngine as any,
|
||||
postMediaEngine: postMediaEngine as any,
|
||||
},
|
||||
});
|
||||
|
||||
// 6. Build writePage function using in-memory hash store
|
||||
const knownDirectories = new Set<string>();
|
||||
|
||||
const writePage = (projectId: string, urlPath: string, content: string) => {
|
||||
const effectiveUrlPath = task.languagePrefix
|
||||
? `${task.languagePrefix.replace(/^\//, '')}/${urlPath}`
|
||||
: urlPath;
|
||||
|
||||
return writeHtmlPage({
|
||||
projectId,
|
||||
htmlDir: task.htmlDir,
|
||||
urlPath: effectiveUrlPath,
|
||||
content,
|
||||
knownDirectories,
|
||||
hashCache,
|
||||
getGeneratedFileHash: hashStore.get,
|
||||
setGeneratedFileHash: hashStore.set,
|
||||
refreshHashTimestampOnUnchanged: true,
|
||||
});
|
||||
};
|
||||
|
||||
const onPageGenerated = (message: string) => {
|
||||
send({ type: 'progress', taskId: task.taskId, message });
|
||||
};
|
||||
|
||||
// 7. Execute the assigned section
|
||||
let pagesGenerated = 0;
|
||||
const projectId = task.options.projectId;
|
||||
|
||||
switch (task.section) {
|
||||
case 'single': {
|
||||
pagesGenerated += await generateSinglePostPages({
|
||||
projectId,
|
||||
posts,
|
||||
renderRoute,
|
||||
writePage,
|
||||
onPageGenerated,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
case 'category': {
|
||||
const allCategories = new Set(task.allCategories ?? []);
|
||||
const postsByCategory = task.postsByCategoryEntries
|
||||
? deserializePostMap(task.postsByCategoryEntries)
|
||||
: undefined;
|
||||
|
||||
pagesGenerated += await generateCategoryPages({
|
||||
projectId,
|
||||
posts,
|
||||
allCategories,
|
||||
maxPostsPerPage: task.maxPostsPerPage,
|
||||
renderRoute,
|
||||
writePage,
|
||||
onPageGenerated,
|
||||
postsByCategory,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
case 'tag': {
|
||||
const allTags = new Set(task.allTags ?? []);
|
||||
const postsByTag = task.postsByTagEntries
|
||||
? deserializePostMap(task.postsByTagEntries)
|
||||
: undefined;
|
||||
|
||||
pagesGenerated += await generateTagPages({
|
||||
projectId,
|
||||
posts,
|
||||
allTags,
|
||||
maxPostsPerPage: task.maxPostsPerPage,
|
||||
renderRoute,
|
||||
writePage,
|
||||
onPageGenerated,
|
||||
postsByTag,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
case 'date': {
|
||||
const yearsMap = task.yearsEntries ? deserializeDateMap(task.yearsEntries) : new Map();
|
||||
const yearMonthsMap = task.yearMonthsEntries ? deserializeDateMap(task.yearMonthsEntries) : new Map();
|
||||
const yearMonthDaysMap = task.yearMonthDaysEntries ? deserializeDateMap(task.yearMonthDaysEntries) : new Map();
|
||||
const postsByYear = task.postsByYearEntries ? deserializePostMap(task.postsByYearEntries) : undefined;
|
||||
const postsByYearMonth = task.postsByYearMonthEntries ? deserializePostMap(task.postsByYearMonthEntries) : undefined;
|
||||
const postsByYearMonthDay = task.postsByYearMonthDayEntries ? deserializePostMap(task.postsByYearMonthDayEntries) : undefined;
|
||||
|
||||
pagesGenerated += await generateDateArchivePages({
|
||||
projectId,
|
||||
posts,
|
||||
yearsMap,
|
||||
yearMonthsMap,
|
||||
yearMonthDaysMap,
|
||||
maxPostsPerPage: task.maxPostsPerPage,
|
||||
renderRoute,
|
||||
writePage,
|
||||
onPageGenerated,
|
||||
postsByYear,
|
||||
postsByYearMonth,
|
||||
postsByYearMonthDay,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
case 'core': {
|
||||
// Core includes root pages and page routes (sitemap/feeds handled by main thread)
|
||||
pagesGenerated += await generateRootPages({
|
||||
projectId,
|
||||
posts,
|
||||
maxPostsPerPage: task.maxPostsPerPage,
|
||||
renderRoute,
|
||||
writePage,
|
||||
onPageGenerated,
|
||||
});
|
||||
pagesGenerated += await generatePageRoutes({
|
||||
projectId,
|
||||
posts: lookupPosts,
|
||||
renderRoute,
|
||||
writePage,
|
||||
onPageGenerated,
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 8. Report result with accumulated hash updates
|
||||
send({
|
||||
type: 'result',
|
||||
taskId: task.taskId,
|
||||
pagesGenerated,
|
||||
hashUpdates: hashStore.pendingUpdates,
|
||||
});
|
||||
} catch (err) {
|
||||
send({
|
||||
type: 'error',
|
||||
taskId: task.taskId,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
@@ -11,6 +11,7 @@ import type { EngineBundle } from '../engine/EngineBundle';
|
||||
import type { TranslationValidationReport } from '../shared/electronApi';
|
||||
import { autoTranslatePost, autoTranslateMediaMetadata } from './chatHandlers';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { getDatabase } from '../database/connection';
|
||||
|
||||
type SafeHandle = (channel: string, handler: (...args: any[]) => Promise<any>) => void;
|
||||
|
||||
@@ -84,6 +85,7 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl
|
||||
categoryMetadata: (metadata as any)?.categoryMetadata,
|
||||
categorySettings: (metadata as any)?.categorySettings,
|
||||
menu,
|
||||
dbPath: getDatabase().getDbPath(),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -91,6 +93,9 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl
|
||||
const blogGenerationEngine = bundle.blogGenerationEngine;
|
||||
const baseOptions = await resolveBlogGenerationBaseOptions();
|
||||
|
||||
// Pre-load post data ONCE before parallel tasks
|
||||
const preloadedData = await blogGenerationEngine.preloadGenerationData(baseOptions);
|
||||
|
||||
const taskTimestamp = Date.now();
|
||||
const taskGroupId = `site-render-${taskTimestamp}`;
|
||||
const taskGroupName = 'Render Site';
|
||||
@@ -109,6 +114,7 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl
|
||||
return blogGenerationEngine.generate({
|
||||
...baseOptions,
|
||||
sections: [section],
|
||||
preloadedData,
|
||||
}, (progress, message) => onProgress(progress, message || ''));
|
||||
},
|
||||
});
|
||||
|
||||
@@ -412,7 +412,7 @@ export const electronAPI: ElectronAPI = {
|
||||
translatePost: (postId: string, targetLanguage: string) => ipcRenderer.invoke('chat:translatePost', postId, targetLanguage),
|
||||
|
||||
// Media Language Detection
|
||||
detectMediaLanguage: (mediaId: string) => ipcRenderer.invoke('chat:detectMediaLanguage', mediaId),
|
||||
detectMediaLanguage: (title: string, alt: string, caption: string) => ipcRenderer.invoke('chat:detectMediaLanguage', title, alt, caption),
|
||||
|
||||
// Media Metadata Translation
|
||||
translateMediaMetadata: (mediaId: string, targetLanguage: string) => ipcRenderer.invoke('chat:translateMediaMetadata', mediaId, targetLanguage),
|
||||
|
||||
@@ -1070,7 +1070,7 @@ export interface ElectronAPI {
|
||||
translatePost: (postId: string, targetLanguage: string) => Promise<{ success: boolean; translation?: PostTranslationData; error?: string }>;
|
||||
|
||||
// Media Language Detection
|
||||
detectMediaLanguage: (mediaId: string) => Promise<{ success: boolean; language?: string; error?: string }>;
|
||||
detectMediaLanguage: (title: string, alt: string, caption: string) => Promise<{ success: boolean; language?: string; error?: string }>;
|
||||
|
||||
// Media Metadata Translation
|
||||
translateMediaMetadata: (mediaId: string, targetLanguage: string) => Promise<{ success: boolean; translation?: MediaTranslationData; error?: string }>;
|
||||
|
||||
@@ -39,7 +39,6 @@ const App: React.FC = () => {
|
||||
toggleSidebar,
|
||||
togglePanel,
|
||||
toggleAssistantSidebar,
|
||||
setActiveView,
|
||||
setSelectedPost,
|
||||
setActiveProject,
|
||||
setPicoTheme,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import React, { type ReactNode } from 'react';
|
||||
import React, { type ReactElement } from 'react';
|
||||
import Markdown from 'marked-react';
|
||||
import type { A2UIResolvedComponent, A2UIClientAction } from '../../../main/a2ui/types';
|
||||
|
||||
@@ -11,7 +11,7 @@ interface A2UIComponentProps {
|
||||
}
|
||||
|
||||
const safeRenderer = {
|
||||
image(src: string, alt: string): ReactNode {
|
||||
image(src: string, alt: string, _title?: string | null): ReactElement {
|
||||
if (/^https?:\/\//i.test(src)) {
|
||||
return <a href={src} key={src} title={alt}>{alt || src}</a>;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import React, { type ReactNode } from 'react';
|
||||
import React, { type ReactElement } from 'react';
|
||||
import Markdown from 'marked-react';
|
||||
import type { ChatMessage } from '../../types/electron';
|
||||
import type { ChatToolEvent } from '../../navigation/useChatSurfaceState';
|
||||
import type { A2UIResolvedComponent, A2UIClientAction } from '../../../main/a2ui/types';
|
||||
import type { A2UIClientAction } from '../../../main/a2ui/types';
|
||||
import { InlineSurface } from '../../a2ui/InlineSurface';
|
||||
import type { SurfaceEntry } from '../../a2ui/useA2UISurface';
|
||||
import { computeTurnIndex } from '../../a2ui/surfaceAssociation';
|
||||
@@ -51,7 +51,7 @@ export const ChatTranscript: React.FC<ChatTranscriptProps> = ({
|
||||
}) => {
|
||||
// Block external images — CSP only allows self/data/file/blob/bds-media/bds-thumb
|
||||
const safeRenderer = {
|
||||
image(src: string, alt: string): ReactNode {
|
||||
image(src: string, alt: string, _title?: string | null): ReactElement {
|
||||
if (/^https?:\/\//i.test(src)) {
|
||||
// Show alt text as a link instead of trying to load the image
|
||||
return <a href={src} key={src} title={alt}>{alt || src}</a>;
|
||||
|
||||
@@ -155,7 +155,7 @@ export const DocumentationView: React.FC<DocumentationViewProps> = ({
|
||||
headingSlugCounts.set(baseId, nextCount);
|
||||
const headingId = existingCount === 0 ? baseId : `${baseId}-${nextCount}`;
|
||||
|
||||
return React.createElement(`h${levelNumber}` as keyof JSX.IntrinsicElements, { id: headingId, key: getRendererKey('heading') }, children);
|
||||
return React.createElement(`h${levelNumber}` as 'h1', { id: headingId, key: getRendererKey('heading') }, children);
|
||||
},
|
||||
link(href: string, text: ReactNode) {
|
||||
if (!href.startsWith('#')) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import { AISuggestionsModal } from '../AISuggestionsModal/AISuggestionsModal';
|
||||
import { openEntityTab } from '../../navigation/tabPolicy';
|
||||
import { useI18n } from '../../i18n';
|
||||
import { SUPPORTED_POST_LANGUAGES, POST_LANGUAGE_FLAGS } from '../../../main/shared/i18n';
|
||||
import type { MediaData } from '../../../main/shared/electronApi';
|
||||
import { getMediaDisplayName } from './editorUtils';
|
||||
|
||||
export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
|
||||
@@ -71,7 +72,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
|
||||
try {
|
||||
const updated = await window.electronAPI?.media.update(item!.id, { language: newLanguage || undefined });
|
||||
if (updated) {
|
||||
updateMedia(item!.id, updated as Partial<typeof item>);
|
||||
updateMedia(item!.id, updated as Partial<MediaData>);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to update media language:', error);
|
||||
@@ -92,7 +93,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
|
||||
setMediaLanguage(result.language);
|
||||
const updated = await window.electronAPI?.media.update(item.id, { language: result.language });
|
||||
if (updated) {
|
||||
updateMedia(item.id, updated as Partial<typeof item>);
|
||||
updateMedia(item.id, updated as Partial<MediaData>);
|
||||
}
|
||||
showToast.success(tr('editor.media.toast.languageDetected', { language: tr(`language.${result.language}`) }));
|
||||
} else {
|
||||
@@ -249,7 +250,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
|
||||
// Close AI suggestions modal
|
||||
const handleCloseAISuggestionsModal = () => {
|
||||
setShowAISuggestionsModal(false);
|
||||
setAISuggestions(null);
|
||||
setAISuggestionFields([]);
|
||||
setAIError(undefined);
|
||||
};
|
||||
|
||||
@@ -364,7 +365,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
|
||||
tags: tags.split(',').map(t => t.trim()).filter(t => t.length > 0),
|
||||
});
|
||||
if (updated) {
|
||||
updateMedia(item.id, updated as Partial<typeof item>);
|
||||
updateMedia(item.id, updated as Partial<MediaData>);
|
||||
showToast.success(tr('editor.media.toast.updated'));
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -382,7 +383,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
|
||||
try {
|
||||
const updated = await window.electronAPI?.media.replaceFileDialog(item.id);
|
||||
if (updated) {
|
||||
updateMedia(item.id, updated as Partial<typeof item>);
|
||||
updateMedia(item.id, updated as Partial<MediaData>);
|
||||
showToast.success(tr('editor.media.toast.fileReplaced'));
|
||||
}
|
||||
// null means user cancelled or file unchanged - no action needed
|
||||
@@ -523,7 +524,7 @@ export const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
|
||||
{item.mimeType.startsWith('image/') ? (
|
||||
<div className="media-preview-image">
|
||||
<img
|
||||
src={`bds-media://${item.id}?t=${item.updatedAt instanceof Date ? item.updatedAt.getTime() : item.updatedAt}`}
|
||||
src={`bds-media://${item.id}?t=${item.updatedAt}`}
|
||||
alt={item.alt || item.originalName}
|
||||
onError={(e) => {
|
||||
// Fallback to placeholder if image fails to load
|
||||
|
||||
@@ -159,7 +159,6 @@ export const PostEditor: React.FC<PostEditorProps> = ({ postId }) => {
|
||||
showErrorModal,
|
||||
showConfirmDeleteModal,
|
||||
media,
|
||||
closeTab,
|
||||
} = useAppStore();
|
||||
|
||||
// Fetch full post data from backend
|
||||
@@ -194,7 +193,7 @@ export const PostEditor: React.FC<PostEditorProps> = ({ postId }) => {
|
||||
const [doNotTranslate, setDoNotTranslate] = useState(false);
|
||||
const [activeEditingLanguage, setActiveEditingLanguage] = useState('');
|
||||
const [canonicalDraft, setCanonicalDraft] = useState<EditableContentDraft>({ title: '', excerpt: '', content: '' });
|
||||
const [savedCanonicalDraft, setSavedCanonicalDraft] = useState<EditableContentDraft>({ title: '', excerpt: '', content: '' });
|
||||
const [, setSavedCanonicalDraft] = useState<EditableContentDraft>({ title: '', excerpt: '', content: '' });
|
||||
const [translationDrafts, setTranslationDrafts] = useState<Record<string, import('../../../main/shared/electronApi').PostTranslationData>>({});
|
||||
const [savedTranslationDrafts, setSavedTranslationDrafts] = useState<Record<string, import('../../../main/shared/electronApi').PostTranslationData>>({});
|
||||
const [availablePostTemplates, setAvailablePostTemplates] = useState<Array<{ slug: string; title: string }>>([]);
|
||||
|
||||
@@ -390,7 +390,7 @@ export const GitSidebar: React.FC = () => {
|
||||
recentCommitsToKeep: 2,
|
||||
});
|
||||
if (!result.success) {
|
||||
if (result.code === 'offline') {
|
||||
if ('code' in result && result.code === 'offline') {
|
||||
showErrorModal({ message: tr('gitSidebar.error.offlineMode') });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -208,23 +208,25 @@ export const ImportAnalysisView: React.FC<ImportAnalysisViewProps> = ({ definiti
|
||||
|
||||
// Subscribe to task completion events
|
||||
useEffect(() => {
|
||||
const unsubscribe = window.electronAPI?.on('task:completed', (task: { taskId: string }) => {
|
||||
const unsubscribe = window.electronAPI?.on('task:completed', ((...args: unknown[]) => {
|
||||
const task = args[0] as { taskId: string };
|
||||
setExecutionState(prev => {
|
||||
if (prev.taskId !== task.taskId) return prev;
|
||||
return { ...prev, isExecuting: false, completed: true };
|
||||
});
|
||||
});
|
||||
}) as (...args: unknown[]) => void);
|
||||
return () => unsubscribe?.();
|
||||
}, []);
|
||||
|
||||
// Subscribe to task failure events
|
||||
useEffect(() => {
|
||||
const unsubscribe = window.electronAPI?.on('task:failed', (task: { taskId: string; error: string }) => {
|
||||
const unsubscribe = window.electronAPI?.on('task:failed', ((...args: unknown[]) => {
|
||||
const task = args[0] as { taskId: string; error: string };
|
||||
setExecutionState(prev => {
|
||||
if (prev.taskId !== task.taskId) return prev;
|
||||
return { ...prev, isExecuting: false, error: task.error };
|
||||
});
|
||||
});
|
||||
}) as (...args: unknown[]) => void);
|
||||
return () => unsubscribe?.();
|
||||
}, []);
|
||||
|
||||
@@ -919,7 +921,7 @@ const DateDistributionCard: React.FC<{ distribution: DateDistribution }> = ({ di
|
||||
};
|
||||
|
||||
// Helper function to format post metadata for tooltip (new post from WXR)
|
||||
function formatPostTooltip(wxrPost: AnalyzedPostItem['wxrPost'], t: (key: string, params?: Record<string, unknown>) => string): string {
|
||||
function formatPostTooltip(wxrPost: AnalyzedPostItem['wxrPost'], t: (key: string, params?: Record<string, string | number>) => string): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`${t('importAnalysis.wordpressId')}: ${wxrPost.wpId}`);
|
||||
lines.push(`${t('importAnalysis.type')}: ${wxrPost.postType}`);
|
||||
@@ -1051,7 +1053,7 @@ function ExistingPostHoverCard({ children, className, postId }: {
|
||||
}
|
||||
|
||||
// Helper function to format media metadata for tooltip
|
||||
function formatMediaTooltip(wxrMedia: AnalyzedMediaItem['wxrMedia'], t: (key: string, params?: Record<string, unknown>) => string): string {
|
||||
function formatMediaTooltip(wxrMedia: AnalyzedMediaItem['wxrMedia'], t: (key: string, params?: Record<string, string | number>) => string): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`${t('importAnalysis.wordpressId')}: ${wxrMedia.wpId}`);
|
||||
lines.push(`${t('importAnalysis.mimeType')}: ${wxrMedia.mimeType || t('importAnalysis.unknown')}`);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import React, { useEffect, useRef, useCallback, useState } from 'react';
|
||||
import { Editor, defaultValueCtx, editorViewCtx, rootCtx, remarkStringifyOptionsCtx, remarkPluginsCtx } from '@milkdown/kit/core';
|
||||
import { commonmark, toggleStrongCommand, toggleEmphasisCommand, wrapInBlockquoteCommand, wrapInBulletListCommand, wrapInOrderedListCommand, insertHrCommand, toggleInlineCodeCommand, insertImageCommand, toggleLinkCommand } from '@milkdown/kit/preset/commonmark';
|
||||
import { commonmark, toggleStrongCommand, toggleEmphasisCommand, wrapInBlockquoteCommand, wrapInBulletListCommand, wrapInOrderedListCommand, insertHrCommand, toggleInlineCodeCommand, insertImageCommand } from '@milkdown/kit/preset/commonmark';
|
||||
import { gfm, toggleStrikethroughCommand } from '@milkdown/kit/preset/gfm';
|
||||
import { history, undoCommand, redoCommand } from '@milkdown/kit/plugin/history';
|
||||
import { listener, listenerCtx } from '@milkdown/kit/plugin/listener';
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import React, { useCallback, useRef, useState } from 'react';
|
||||
import MonacoEditor, { type Monaco } from '@monaco-editor/react';
|
||||
import type { ScriptData } from '../../../main/shared/electronApi';
|
||||
import { useAppStore } from '../../store';
|
||||
@@ -89,7 +89,6 @@ export const ScriptsView: React.FC<ScriptsViewProps> = ({ scriptId }) => {
|
||||
|
||||
// Refresh entrypoints asynchronously
|
||||
entrypointCancelRef.current = true; // cancel any pending refresh
|
||||
const cancelToken = {};
|
||||
entrypointCancelRef.current = false;
|
||||
const refreshEntrypoints = async () => {
|
||||
try {
|
||||
|
||||
@@ -28,7 +28,7 @@ export function SidebarEntityList<TItem>({
|
||||
renderItem,
|
||||
getItemKey,
|
||||
topContent,
|
||||
}: SidebarEntityListProps<TItem>): JSX.Element {
|
||||
}: SidebarEntityListProps<TItem>): React.JSX.Element {
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="chat-list">
|
||||
|
||||
@@ -346,7 +346,7 @@ export const WindowTitleBar: React.FC = () => {
|
||||
};
|
||||
}, [isMac, mnemonicByKey, showMnemonics]);
|
||||
|
||||
const handleMenuButtonClick = (event: React.MouseEvent<HTMLButtonElement>, label: string) => {
|
||||
const handleMenuButtonClick = (_event: React.MouseEvent<HTMLButtonElement>, label: string) => {
|
||||
const left = getMenuLeft(label);
|
||||
if (left === null) {
|
||||
return;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { PythonMacroInfo, PythonMacroResolver, PythonMacroRendererFn } from './types';
|
||||
import type { PythonMacroResolver, PythonMacroRendererFn } from './types';
|
||||
import { setPythonMacroResolver } from './registry';
|
||||
import { getPythonRuntimeManager } from '../python/runtimeManagerInstance';
|
||||
|
||||
|
||||
@@ -138,7 +138,7 @@ const tabsElementSchema: z.ZodTypeAny = z.lazy(() => z.object({
|
||||
).min(1),
|
||||
}));
|
||||
|
||||
assistantPanelElementSchemaRef = z.discriminatedUnion('type', [
|
||||
assistantPanelElementSchemaRef = z.union([
|
||||
textElementSchema,
|
||||
metricElementSchema,
|
||||
listElementSchema,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { openEntityTab } from './tabPolicy';
|
||||
import { openEntityTab, type CanonicalTabSpec } from './tabPolicy';
|
||||
import type { SidebarView } from './sidebarViewRegistry';
|
||||
|
||||
interface BlogmarkStateSnapshot {
|
||||
@@ -14,7 +14,7 @@ interface BlogmarkHandlers {
|
||||
setActiveView: (view: SidebarView) => void;
|
||||
toggleSidebar: () => void;
|
||||
setSelectedPost: (id: string) => void;
|
||||
openTab: (tab: { type: 'post'; id: string; isTransient: boolean }) => void;
|
||||
openTab: (tab: CanonicalTabSpec) => void;
|
||||
}
|
||||
|
||||
export function handleBlogmarkCreatedEvent(
|
||||
|
||||
@@ -83,13 +83,13 @@ export function parseBlogmarkCreatedEventPayload(payload: unknown): BlogmarkCrea
|
||||
|
||||
if (isRecord(payload.post)) {
|
||||
return {
|
||||
post: payload.post as PostData,
|
||||
post: payload.post as unknown as PostData,
|
||||
transform: parseTransformDebugInfo(payload.transform),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
post: payload as PostData,
|
||||
post: payload as unknown as PostData,
|
||||
transform: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createPythonRuntimeWorker } from './createPythonRuntimeWorker';
|
||||
import type { PythonWorkerMessage, PythonWorkerRequest } from './runtimeProtocol';
|
||||
import type { PythonSyntaxError } from './runtimeProtocol';
|
||||
import { parseMacroContextV1, parseMacroResultV1, type MacroContextV1, type MacroResultV1 } from './abiV1';
|
||||
import { parseMacroContextV1, parseMacroResultV1, type MacroResultV1 } from './abiV1';
|
||||
import { invokePythonApiMethodV1 } from './pythonApiInvokerV1';
|
||||
import { showToast } from '../components/Toast';
|
||||
|
||||
|
||||
4
src/renderer/types/highlightjs-cdn-assets.d.ts
vendored
Normal file
4
src/renderer/types/highlightjs-cdn-assets.d.ts
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
declare module '@highlightjs/cdn-assets/es/highlight.min.js' {
|
||||
import hljs from 'highlight.js';
|
||||
export default hljs;
|
||||
}
|
||||
Reference in New Issue
Block a user