Feat/webworker for incremental render (#51)

* feat: web worker for incremental render

* feat: optimizing incremental render for date archives

* feat: more work on web worker

* fix: blogmark process handled defaulting wrong

---------

Co-authored-by: hugo <hugoms@me.com>
This commit is contained in:
Georg Bauer
2026-03-13 13:27:45 +01:00
committed by GitHub
parent 101036e58e
commit 914af9831d
11 changed files with 1485 additions and 312 deletions

View File

@@ -0,0 +1,208 @@
/**
* Builds targeted worker tasks for the applyValidation flow.
*
* Takes a targeted validation plan and produces GenerationWorkerTask[]
* scoped to only the affected sections (categories, tags, dates, singles, core).
* This gives applyValidation the same worker-pool parallelism as full generation
* while limiting the volume to only what's needed.
*/
import type { PostData } from './PostEngine';
import type { GenerationPostIndex } from './GenerationPostIndexService';
import type { TargetedValidationPlan } from './ValidationApplyPlannerService';
import type {
GenerationWorkerTask,
SerializedPostData,
SerializedMediaData,
SerializedBlogGenerationOptions,
} from './GenerationWorkerData';
import {
serializePostData,
serializePostMap,
serializeDateMap,
} from './GenerationWorkerData';
export interface ApplyValidationWorkerParams {
options: SerializedBlogGenerationOptions;
maxPostsPerPage: number;
htmlDir: string;
mediaItems: SerializedMediaData[];
backlinksRecord: Record<string, Array<{ id: string; title: string; slug: string }>>;
hashMapEntries: Array<[string, string]>;
postFilePathEntries: Array<[string, string]>;
postMediaLinksEntries: Array<[string, Array<{ mediaId: string; sortOrder: number }>]>;
}
export interface ApplyValidationLanguageParams {
targetedPlan: TargetedValidationPlan;
publishedRoutePosts: PostData[];
publishedListPosts: PostData[];
generationPostIndex: GenerationPostIndex;
years: Map<number, Date>;
yearMonths: Map<string, Date>;
yearMonthDays: Map<string, Date>;
languagePrefix?: string;
mainLanguage?: string;
}
export function buildApplyValidationWorkerTasks(
base: ApplyValidationWorkerParams,
lang: ApplyValidationLanguageParams,
): GenerationWorkerTask[] {
const { targetedPlan, publishedRoutePosts, publishedListPosts, generationPostIndex } = lang;
const serializedRoutePosts = publishedRoutePosts.map(serializePostData);
const serializedListPosts = publishedListPosts.map(serializePostData);
const baseTaskData = {
lookupPosts: serializedRoutePosts,
mediaItems: base.mediaItems,
backlinksMap: base.backlinksRecord,
options: lang.languagePrefix
? { ...base.options, language: lang.languagePrefix.replace(/^\//, '') }
: base.options,
maxPostsPerPage: base.maxPostsPerPage,
htmlDir: base.htmlDir,
hashMapEntries: base.hashMapEntries,
postFilePathEntries: base.postFilePathEntries,
postMediaLinksEntries: base.postMediaLinksEntries,
languagePrefix: lang.languagePrefix,
mainLanguage: lang.mainLanguage,
};
const tasks: GenerationWorkerTask[] = [];
let taskCounter = 0;
const langSuffix = lang.languagePrefix ? `-${lang.languagePrefix.replace(/^\//, '')}` : '';
const nextTaskId = (section: string) => `apply-${section}${langSuffix}-${++taskCounter}`;
// Core (root + page routes)
if (targetedPlan.requestRootRoutes) {
tasks.push({
...baseTaskData,
taskId: nextTaskId('core'),
section: 'core',
posts: serializedListPosts,
});
}
// Single posts
if (targetedPlan.requestedPostIds.size > 0) {
const requestedSinglePosts = publishedRoutePosts
.filter((p) => targetedPlan.requestedPostIds.has(p.id))
.map(serializePostData);
if (requestedSinglePosts.length > 0) {
tasks.push({
...baseTaskData,
taskId: nextTaskId('single'),
section: 'single',
posts: requestedSinglePosts,
});
}
}
// Page-slug posts (separate from singles; they're resolved through core)
// Pages are rendered via the 'core' section's generatePageRoutes call,
// so they're covered by the requestRootRoutes task above.
// Categories
if (targetedPlan.requestedCategorySet.size > 0) {
// Only include the requested categories and their post-index entries
const filteredPostsByCategory = new Map<string, PostData[]>();
for (const category of targetedPlan.requestedCategorySet) {
const posts = generationPostIndex.postsByCategory.get(category);
if (posts) {
filteredPostsByCategory.set(category, posts);
}
}
tasks.push({
...baseTaskData,
taskId: nextTaskId('category'),
section: 'category',
posts: serializedListPosts,
allCategories: Array.from(targetedPlan.requestedCategorySet),
postsByCategoryEntries: serializePostMap(filteredPostsByCategory),
});
}
// Tags
if (targetedPlan.requestedTagSet.size > 0) {
const filteredPostsByTag = new Map<string, PostData[]>();
for (const tag of targetedPlan.requestedTagSet) {
const posts = generationPostIndex.postsByTag.get(tag);
if (posts) {
filteredPostsByTag.set(tag, posts);
}
}
tasks.push({
...baseTaskData,
taskId: nextTaskId('tag'),
section: 'tag',
posts: serializedListPosts,
allTags: Array.from(targetedPlan.requestedTagSet),
postsByTagEntries: serializePostMap(filteredPostsByTag),
});
}
// Date archives
const { requestedYears, requestedYearMonths, requestedYearMonthDays } = targetedPlan;
const hasDateRequests = requestedYears.size > 0
|| requestedYearMonths.size > 0
|| requestedYearMonthDays.size > 0;
if (hasDateRequests) {
// Filter archive maps to only the requested keys
const filteredYears = new Map<number, Date>();
for (const year of requestedYears) {
const lastmod = lang.years.get(year);
if (lastmod) filteredYears.set(year, lastmod);
}
const filteredYearMonths = new Map<string, Date>();
for (const ym of requestedYearMonths) {
const lastmod = lang.yearMonths.get(ym);
if (lastmod) filteredYearMonths.set(ym, lastmod);
}
const filteredYearMonthDays = new Map<string, Date>();
for (const ymd of requestedYearMonthDays) {
const lastmod = lang.yearMonthDays.get(ymd);
if (lastmod) filteredYearMonthDays.set(ymd, lastmod);
}
// Filter post-index entries to only the requested date keys
const filteredPostsByYear = new Map<number, PostData[]>();
for (const year of requestedYears) {
const posts = generationPostIndex.postsByYear.get(year);
if (posts) filteredPostsByYear.set(year, posts);
}
const filteredPostsByYearMonth = new Map<string, PostData[]>();
for (const ym of requestedYearMonths) {
const posts = generationPostIndex.postsByYearMonth.get(ym);
if (posts) filteredPostsByYearMonth.set(ym, posts);
}
const filteredPostsByYearMonthDay = new Map<string, PostData[]>();
for (const ymd of requestedYearMonthDays) {
const posts = generationPostIndex.postsByYearMonthDay.get(ymd);
if (posts) filteredPostsByYearMonthDay.set(ymd, posts);
}
tasks.push({
...baseTaskData,
taskId: nextTaskId('date'),
section: 'date',
posts: serializedListPosts,
yearsEntries: serializeDateMap(filteredYears),
yearMonthsEntries: serializeDateMap(filteredYearMonths),
yearMonthDaysEntries: serializeDateMap(filteredYearMonthDays),
postsByYearEntries: serializePostMap(filteredPostsByYear),
postsByYearMonthEntries: serializePostMap(filteredPostsByYearMonth),
postsByYearMonthDayEntries: serializePostMap(filteredPostsByYearMonthDay),
});
}
return tasks;
}

View File

@@ -44,9 +44,10 @@ import {
buildRequestedArchiveMaps,
selectRequestedPosts,
} from './ApplyValidationDataService';
import { buildApplyValidationWorkerTasks } from './ApplyValidationWorkerService';
import { getGeneratedFileHashRecord } from '../database/generatedFileHashStore';
import { getAllGeneratedFileHashes, setGeneratedFileHash } from '../database/generatedFileHashStore';
import { GenerationWorkerPool, type WorkerPoolResult } from './GenerationWorkerPool';
import { GenerationWorkerPool } from './GenerationWorkerPool';
import {
serializePostData,
serializeMediaItem,
@@ -54,7 +55,6 @@ import {
serializePostMap,
serializeDateMap,
type GenerationWorkerTask,
type SerializedPostData,
} from './GenerationWorkerData';
import { readPostTranslationFile } from './postTranslationFileUtils';
@@ -130,6 +130,34 @@ export interface SiteValidationApplyResult {
removedEmptyDirCount: number;
}
export interface ApplyValidationPreparation {
deletedUrlCount: number;
removedEmptyDirCount: number;
requiresFallbackSectionRender: boolean;
sectionsToRender: BlogGenerationSection[];
/** Pre-built worker tasks partitioned by section (worker mode only). */
workerTasksBySection: Map<BlogGenerationSection, GenerationWorkerTask[]>;
/** Data for main-thread fallback rendering (non-worker mode only). */
targetedRenderData?: ApplyValidationTargetedRenderData;
}
interface ApplyValidationTargetedRenderData {
options: BlogGenerationOptions;
maxPostsPerPage: number;
htmlDir: string;
publishedPosts: PostData[];
publishedListPosts: PostData[];
publishedRoutePosts: PostData[];
generationPostIndex: GenerationPostIndex;
targetedPlan: import('./ValidationApplyPlannerService').TargetedValidationPlan;
missingPathPlan: import('./ValidationApplyPlannerService').MissingPathPlan;
mainLanguage: string;
additionalLanguages: string[];
years: Map<number, Date>;
yearMonths: Map<string, Date>;
yearMonthDays: Map<string, Date>;
}
export interface CalendarRegenerationResult {
calendarPath: string;
changed: boolean;
@@ -1543,15 +1571,55 @@ export class BlogGenerationEngine {
report: SiteValidationReport,
onProgress: (progress: number, message?: string) => void,
): Promise<SiteValidationApplyResult> {
onProgress(0, 'Applying validation changes...');
const preparation = await this.prepareApplyValidation(options, report, (progress, message) => {
const mapped = Math.floor((progress / 100) * 45);
onProgress(mapped, message);
});
let renderedUrlCount = 0;
const sections = preparation.sectionsToRender;
for (let i = 0; i < sections.length; i++) {
const section = sections[i];
const sectionCount = await this.applyValidationForSection(options, preparation, section, (progress, message) => {
const base = 45 + Math.floor((i / Math.max(1, sections.length)) * 45);
const span = Math.max(1, Math.floor(45 / Math.max(1, sections.length)));
const mapped = base + Math.floor((progress / 100) * span);
onProgress(Math.min(90, mapped), message);
});
renderedUrlCount += sectionCount;
}
if (renderedUrlCount > 0 || preparation.deletedUrlCount > 0) {
onProgress(90, 'Regenerating calendar data...');
await this.regenerateCalendar(options, (progress, message) => {
const mappedProgress = 90 + Math.floor((progress / 100) * 9);
onProgress(Math.min(99, mappedProgress), message || 'Regenerating calendar data...');
});
}
onProgress(100, `Apply complete (${preparation.deletedUrlCount} deleted, ${renderedUrlCount} rendered)`);
return {
renderedUrlCount,
deletedUrlCount: preparation.deletedUrlCount,
removedEmptyDirCount: preparation.removedEmptyDirCount,
};
}
// ── Multi-task apply validation: preparation + per-section rendering ───
async prepareApplyValidation(
options: BlogGenerationOptions,
report: SiteValidationReport,
onProgress: (progress: number, message?: string) => void,
): Promise<ApplyValidationPreparation> {
onProgress(0, 'Planning validation apply steps...');
const missingPaths = Array.isArray(report.missingUrlPaths) ? report.missingUrlPaths : [];
const updatedPostPaths = Array.isArray(report.updatedPostUrlPaths) ? report.updatedPostUrlPaths : [];
const rerenderPaths = Array.from(new Set([...missingPaths, ...updatedPostPaths]));
const extraPaths = Array.isArray(report.extraUrlPaths) ? report.extraUrlPaths : [];
onProgress(10, 'Planning validation apply steps...');
const mainLanguage = (options.language ?? 'en').trim().toLowerCase();
const additionalLanguages = (options.blogLanguages ?? [])
.map((lang) => lang.trim().toLowerCase())
@@ -1567,7 +1635,6 @@ export class BlogGenerationEngine {
const pruneEmptyParents = async (startDir: string): Promise<void> => {
let currentDir = startDir;
while (path.resolve(currentDir) !== path.resolve(htmlDir)) {
let entries: string[];
try {
@@ -1575,11 +1642,7 @@ export class BlogGenerationEngine {
} catch {
break;
}
if (entries.length > 0) {
break;
}
if (entries.length > 0) break;
await fs.rm(currentDir, { recursive: true, force: true });
removedEmptyDirCount += 1;
currentDir = path.dirname(currentDir);
@@ -1596,182 +1659,122 @@ export class BlogGenerationEngine {
} catch {
// ignore missing files and continue
}
if (extraPaths.length > 0) {
const deleteProgress = 20 + Math.floor(((index + 1) / extraPaths.length) * 25);
onProgress(Math.min(45, deleteProgress), `Deleted ${index + 1}/${extraPaths.length} extra URLs`);
onProgress(Math.min(50, deleteProgress), `Deleted ${index + 1}/${extraPaths.length} extra URLs`);
}
}
let renderedUrlCount = 0;
if (missingPathPlan.requiresFallbackSectionRender) {
onProgress(50, 'Rendering missing routes (fallback section mode)...');
const sectionExecutionOrder: BlogGenerationSection[] = ['category', 'tag', 'date', 'core', 'single'];
for (let index = 0; index < sectionExecutionOrder.length; index += 1) {
const section = sectionExecutionOrder[index];
const generationResult = await this.generate({
...options,
maxPostsPerPage: options.maxPostsPerPage,
sections: [section],
}, (progress, message) => {
const base = 50 + Math.floor((index / sectionExecutionOrder.length) * 40);
const span = Math.max(1, Math.floor(40 / sectionExecutionOrder.length));
const mapped = base + Math.floor((progress / 100) * span);
onProgress(Math.min(90, mapped), message || `Rendering ${section} routes...`);
});
onProgress(100, 'Preparation complete (fallback section mode)');
return {
deletedUrlCount,
removedEmptyDirCount,
requiresFallbackSectionRender: true,
sectionsToRender: ['category', 'tag', 'date', 'core', 'single'],
workerTasksBySection: new Map(),
};
}
renderedUrlCount += generationResult.pagesGenerated;
onProgress(55, 'Loading post data...');
const categorySettings = resolveCategorySettings(options.categoryMetadata, options.categorySettings);
const listExcludedCategories = Object.entries(categorySettings)
.filter(([, settings]) => settings.renderInLists === false)
.map(([category]) => category);
const maxPostsPerPage = clampMaxPostsPerPage(options.maxPostsPerPage);
const { publishedPosts, publishedListPosts } = await loadPublishedGenerationSets(this.postEngine, listExcludedCategories);
const { routePosts: publishedRoutePosts, translationsByPost } = await this.buildPublishedRoutePosts(publishedPosts);
const generationPostIndex = buildGenerationPostIndex(publishedListPosts);
const { allCategories, allTags, years, yearMonths, yearMonthDays } = buildApplyValidationArchives(publishedListPosts);
const targetedPlan = buildTargetedValidationPlan({
initialPlan: missingPathPlan,
publishedPosts: publishedRoutePosts,
allCategories,
allTags,
availableYearMonths: yearMonths.keys(),
availableYearMonthDays: yearMonthDays.keys(),
});
await fs.mkdir(htmlDir, { recursive: true });
const useWorkers = !!options.dbPath;
// Build worker tasks partitioned by section (worker mode only)
const workerTasksBySection = new Map<BlogGenerationSection, GenerationWorkerTask[]>();
if (useWorkers) {
onProgress(65, 'Loading media and serializing worker data...');
const rawMedia = await this.mediaEngine.getAllMedia();
const mediaItems = rawMedia.map(serializeMediaItem);
let backlinksRecord: Record<string, Array<{ id: string; title: string; slug: string }>> = {};
if (typeof this.postEngine.getAllBacklinks === 'function') {
const blMap = await this.postEngine.getAllBacklinks();
for (const [postId, links] of blMap) {
backlinksRecord[postId] = links;
}
}
} else {
const categorySettings = resolveCategorySettings(options.categoryMetadata, options.categorySettings);
const listExcludedCategories = Object.entries(categorySettings)
.filter(([, settings]) => settings.renderInLists === false)
.map(([category]) => category);
const maxPostsPerPage = clampMaxPostsPerPage(options.maxPostsPerPage);
const { publishedPosts, publishedListPosts } = await loadPublishedGenerationSets(this.postEngine, listExcludedCategories);
const { routePosts: publishedRoutePosts } = await this.buildPublishedRoutePosts(publishedPosts);
const generationPostIndex = buildGenerationPostIndex(publishedListPosts);
const serializedOptions = serializeBlogGenerationOptions(options);
const { allCategories, allTags, years, yearMonths, yearMonthDays } = buildApplyValidationArchives(publishedListPosts);
let postFilePathEntries: Array<[string, string]> = [];
if (typeof this.postEngine.getPublishedPostFilePaths === 'function') {
const filePathMap = await this.postEngine.getPublishedPostFilePaths();
postFilePathEntries = Array.from(filePathMap);
}
const targetedPlan = buildTargetedValidationPlan({
initialPlan: missingPathPlan,
publishedPosts: publishedRoutePosts,
allCategories,
allTags,
availableYearMonths: yearMonths.keys(),
availableYearMonthDays: yearMonthDays.keys(),
});
let postMediaLinksEntries: Array<[string, Array<{ mediaId: string; sortOrder: number }>]> = [];
if (typeof this.postMediaEngine.getAllPostMediaLinks === 'function') {
const linksMap = await this.postMediaEngine.getAllPostMediaLinks();
postMediaLinksEntries = Array.from(linksMap);
}
const htmlDir = path.join(options.dataDir, 'html');
await fs.mkdir(htmlDir, { recursive: true });
const generatedHashCache = new Map<string, string | null>();
const existingHashes = await getAllGeneratedFileHashes(options.projectId);
for (const [relativePath, hash] of existingHashes) {
generatedHashCache.set(relativePath, hash);
}
const hashMapEntries: Array<[string, string]> = [];
for (const [relativePath, hash] of generatedHashCache) {
if (hash !== null) hashMapEntries.push([relativePath, hash]);
}
const renderRoute = createPreviewBackedGenerationRouteRenderer({
options,
const mainLangRoutePosts = this.resolvePostsForLanguage(publishedRoutePosts, mainLanguage, translationsByPost, mainLanguage);
const mainLangListPosts = this.resolvePostsForLanguage(publishedListPosts, mainLanguage, translationsByPost, mainLanguage);
const mainLangPostIndex = buildGenerationPostIndex(mainLangListPosts);
const baseWorkerParams = {
options: serializedOptions,
maxPostsPerPage,
publishedPostsForLookup: publishedRoutePosts,
engines: {
postEngine: this.postEngine,
mediaEngine: this.mediaEngine,
postMediaEngine: this.postMediaEngine,
},
});
const writePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({
projectId,
htmlDir,
urlPath,
content,
refreshHashTimestampOnUnchanged: true,
});
const onPageGenerated = (_message: string) => {
// no-op for applyValidation
mediaItems,
backlinksRecord,
hashMapEntries,
postFilePathEntries,
postMediaLinksEntries,
};
const { requestedSinglePosts, requestedPagePosts } = selectRequestedPosts({
publishedPosts: publishedRoutePosts,
requestedPostIds: targetedPlan.requestedPostIds,
requestedPageSlugs: targetedPlan.requestedPageSlugs,
});
onProgress(80, 'Building targeted worker tasks...');
const { requestedYearsMap, requestedYearMonthsMap, requestedYearMonthDaysMap } = buildRequestedArchiveMaps({
requestedYears: targetedPlan.requestedYears,
requestedYearMonths: targetedPlan.requestedYearMonths,
requestedYearMonthDays: targetedPlan.requestedYearMonthDays,
const allTasks: GenerationWorkerTask[] = buildApplyValidationWorkerTasks(baseWorkerParams, {
targetedPlan,
publishedRoutePosts: mainLangRoutePosts,
publishedListPosts: mainLangListPosts,
generationPostIndex: mainLangPostIndex,
years,
yearMonths,
yearMonthDays,
});
onProgress(
48,
`Targeted rerender plan: singles=${requestedSinglePosts.length}, categories=${targetedPlan.requestedCategorySet.size}, tags=${targetedPlan.requestedTagSet.size}, years=${requestedYearsMap.size}, months=${requestedYearMonthsMap.size}, days=${requestedYearMonthDaysMap.size}, root=${targetedPlan.requestRootRoutes ? 1 : 0}, pages=${requestedPagePosts.length}`,
);
onProgress(50, 'Rendering targeted missing routes...');
if (targetedPlan.requestRootRoutes) {
renderedUrlCount += await generateRootPages({
projectId: options.projectId,
posts: publishedListPosts,
maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
});
}
if (requestedPagePosts.length > 0) {
renderedUrlCount += await generatePageRoutes({
projectId: options.projectId,
posts: requestedPagePosts,
renderRoute,
writePage,
onPageGenerated,
});
}
if (targetedPlan.requestedCategorySet.size > 0) {
renderedUrlCount += await generateCategoryPages({
projectId: options.projectId,
posts: publishedListPosts,
allCategories: targetedPlan.requestedCategorySet,
maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
postsByCategory: generationPostIndex.postsByCategory,
});
}
if (targetedPlan.requestedTagSet.size > 0) {
renderedUrlCount += await generateTagPages({
projectId: options.projectId,
posts: publishedListPosts,
allTags: targetedPlan.requestedTagSet,
maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
postsByTag: generationPostIndex.postsByTag,
});
}
if (requestedSinglePosts.length > 0) {
renderedUrlCount += await generateSinglePostPages({
projectId: options.projectId,
posts: requestedSinglePosts,
renderRoute,
writePage,
onPageGenerated,
});
}
if (requestedYearsMap.size > 0 || requestedYearMonthsMap.size > 0 || requestedYearMonthDaysMap.size > 0) {
renderedUrlCount += await generateDateArchivePages({
projectId: options.projectId,
posts: publishedListPosts,
yearsMap: requestedYearsMap,
yearMonthsMap: requestedYearMonthsMap,
yearMonthDaysMap: requestedYearMonthDaysMap,
maxPostsPerPage,
renderRoute,
writePage,
onPageGenerated,
postsByYear: generationPostIndex.postsByYear,
postsByYearMonth: generationPostIndex.postsByYearMonth,
postsByYearMonthDay: generationPostIndex.postsByYearMonthDay,
});
}
// --- Render missing per-language subtree pages ---
for (const [lang, langMissingPlan] of missingPathPlan.languagePlans) {
const langPosts = publishedPosts.filter((p) => !(p as PostData & { doNotTranslate?: boolean }).doNotTranslate);
const langListPosts = publishedListPosts.filter((p) => !(p as PostData & { doNotTranslate?: boolean }).doNotTranslate);
const langPostIndex = buildGenerationPostIndex(langListPosts);
const langPosts = publishedPosts.filter((p) => !p.doNotTranslate);
const langListPosts = publishedListPosts.filter((p) => !p.doNotTranslate);
const langArchives = buildApplyValidationArchives(langListPosts);
const langTargetedPlan = buildTargetedValidationPlan({
initialPlan: langMissingPlan,
publishedPosts: langPosts,
@@ -1781,138 +1784,386 @@ export class BlogGenerationEngine {
availableYearMonthDays: langArchives.yearMonthDays.keys(),
});
const langRenderRoute = createPreviewBackedGenerationRouteRenderer({
options: { ...options, language: lang },
maxPostsPerPage,
publishedPostsForLookup: langPosts,
languagePrefix: `/${lang}`,
engines: {
postEngine: this.postEngine,
mediaEngine: this.mediaEngine,
postMediaEngine: this.postMediaEngine,
const resolvedLangPosts = this.resolvePostsForLanguage(langPosts, lang, translationsByPost, mainLanguage);
const resolvedLangListPosts = this.resolvePostsForLanguage(langListPosts, lang, translationsByPost, mainLanguage);
const resolvedLangPostIndex = buildGenerationPostIndex(resolvedLangListPosts);
const langTasks = buildApplyValidationWorkerTasks(
{ ...baseWorkerParams, options: { ...serializedOptions, language: lang } },
{
targetedPlan: langTargetedPlan,
publishedRoutePosts: resolvedLangPosts,
publishedListPosts: resolvedLangListPosts,
generationPostIndex: resolvedLangPostIndex,
years: langArchives.years,
yearMonths: langArchives.yearMonths,
yearMonthDays: langArchives.yearMonthDays,
languagePrefix: `/${lang}`,
mainLanguage,
},
});
);
allTasks.push(...langTasks);
}
const langWritePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({
projectId,
htmlDir,
urlPath: `${lang}/${urlPath}`,
content,
refreshHashTimestampOnUnchanged: true,
});
for (const task of allTasks) {
const sectionTasks = workerTasksBySection.get(task.section) ?? [];
sectionTasks.push(task);
workerTasksBySection.set(task.section, sectionTasks);
}
}
if (langTargetedPlan.requestRootRoutes) {
// Determine which sections have work
const sectionsToRender: BlogGenerationSection[] = useWorkers
? Array.from(workerTasksBySection.keys())
: this.computeTargetedSectionsToRender(targetedPlan, missingPathPlan, publishedPosts, publishedListPosts);
onProgress(100, 'Preparation complete');
return {
deletedUrlCount,
removedEmptyDirCount,
requiresFallbackSectionRender: false,
sectionsToRender,
workerTasksBySection,
targetedRenderData: !useWorkers ? {
options,
maxPostsPerPage,
htmlDir,
publishedPosts,
publishedListPosts,
publishedRoutePosts,
generationPostIndex,
targetedPlan,
missingPathPlan,
mainLanguage,
additionalLanguages,
years,
yearMonths,
yearMonthDays,
} : undefined,
};
}
private computeTargetedSectionsToRender(
targetedPlan: import('./ValidationApplyPlannerService').TargetedValidationPlan,
missingPathPlan: import('./ValidationApplyPlannerService').MissingPathPlan,
publishedPosts: PostData[],
publishedListPosts: PostData[],
): BlogGenerationSection[] {
const sections = new Set<BlogGenerationSection>();
const checkPlan = (plan: import('./ValidationApplyPlannerService').TargetedValidationPlan) => {
if (plan.requestRootRoutes || plan.requestedPageSlugs.size > 0) sections.add('core');
if (plan.requestedPostIds.size > 0) sections.add('single');
if (plan.requestedCategorySet.size > 0) sections.add('category');
if (plan.requestedTagSet.size > 0) sections.add('tag');
if (plan.requestedYears.size > 0 || plan.requestedYearMonths.size > 0 || plan.requestedYearMonthDays.size > 0) sections.add('date');
};
checkPlan(targetedPlan);
for (const [, langMissingPlan] of missingPathPlan.languagePlans) {
const langPosts = publishedPosts.filter((p) => !p.doNotTranslate);
const langListPosts = publishedListPosts.filter((p) => !p.doNotTranslate);
const langArchives = buildApplyValidationArchives(langListPosts);
const langTargetedPlan = buildTargetedValidationPlan({
initialPlan: langMissingPlan,
publishedPosts: langPosts,
allCategories: langArchives.allCategories,
allTags: langArchives.allTags,
availableYearMonths: langArchives.yearMonths.keys(),
availableYearMonthDays: langArchives.yearMonthDays.keys(),
});
checkPlan(langTargetedPlan);
}
return Array.from(sections);
}
async applyValidationForSection(
options: BlogGenerationOptions,
preparation: ApplyValidationPreparation,
section: BlogGenerationSection,
onProgress: (progress: number, message?: string) => void,
): Promise<number> {
if (preparation.requiresFallbackSectionRender) {
const result = await this.generate({
...options,
maxPostsPerPage: options.maxPostsPerPage,
sections: [section],
}, (progress, message) => {
onProgress(progress, message || `Rendering ${section} routes...`);
});
return result.pagesGenerated;
}
// Worker-based targeted rendering for this section
const sectionTasks = preparation.workerTasksBySection.get(section);
if (sectionTasks && sectionTasks.length > 0) {
onProgress(0, `Dispatching ${sectionTasks.length} targeted ${section} tasks to worker pool...`);
const pool = new GenerationWorkerPool();
const result = await pool.runTasks(sectionTasks, (message) => {
onProgress(50, message);
});
if (result.errors.length > 0) {
console.error(`[ApplyValidation/${section}] ${result.errors.length} task(s) failed:`);
for (const err of result.errors) {
console.error(` [${err.taskId}] ${err.error}`);
}
}
if (result.hashUpdates.length > 0) {
for (const update of result.hashUpdates) {
await setGeneratedFileHash(options.projectId, update.relativePath, update.hash);
}
}
onProgress(100, `${section} rendering complete`);
return result.pagesGenerated;
}
// Main-thread fallback for this section
if (preparation.targetedRenderData) {
return this.applyValidationSectionOnMainThread(preparation.targetedRenderData, section, onProgress);
}
return 0;
}
// ── Per-section main-thread targeted apply validation ────────────────
private async applyValidationSectionOnMainThread(
data: ApplyValidationTargetedRenderData,
section: BlogGenerationSection,
onProgress: (progress: number, message?: string) => void,
): Promise<number> {
const {
options, maxPostsPerPage, htmlDir,
publishedPosts, publishedListPosts, publishedRoutePosts,
generationPostIndex, targetedPlan, missingPathPlan,
years, yearMonths, yearMonthDays,
} = data;
let renderedUrlCount = 0;
const renderRoute = createPreviewBackedGenerationRouteRenderer({
options,
maxPostsPerPage,
publishedPostsForLookup: publishedRoutePosts,
engines: {
postEngine: this.postEngine,
mediaEngine: this.mediaEngine,
postMediaEngine: this.postMediaEngine,
},
});
const writePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({
projectId,
htmlDir,
urlPath,
content,
refreshHashTimestampOnUnchanged: true,
});
const onPageGenerated = (_message: string) => {
// no-op for applyValidation
};
const { requestedSinglePosts, requestedPagePosts } = selectRequestedPosts({
publishedPosts: publishedRoutePosts,
requestedPostIds: targetedPlan.requestedPostIds,
requestedPageSlugs: targetedPlan.requestedPageSlugs,
});
const { requestedYearsMap, requestedYearMonthsMap, requestedYearMonthDaysMap } = buildRequestedArchiveMaps({
requestedYears: targetedPlan.requestedYears,
requestedYearMonths: targetedPlan.requestedYearMonths,
requestedYearMonthDays: targetedPlan.requestedYearMonthDays,
years,
yearMonths,
yearMonthDays,
});
onProgress(10, `Rendering ${section} section...`);
// Main language rendering for this section
switch (section) {
case 'core':
if (targetedPlan.requestRootRoutes) {
renderedUrlCount += await generateRootPages({
projectId: options.projectId,
posts: langListPosts,
maxPostsPerPage,
renderRoute: langRenderRoute,
writePage: langWritePage,
onPageGenerated,
projectId: options.projectId, posts: publishedListPosts, maxPostsPerPage,
renderRoute, writePage, onPageGenerated,
});
const langRequestedPagePosts = selectRequestedPosts({
publishedPosts: langPosts,
requestedPostIds: new Set(),
requestedPageSlugs: langTargetedPlan.requestedPageSlugs,
}).requestedPagePosts;
if (langRequestedPagePosts.length > 0) {
renderedUrlCount += await generatePageRoutes({
projectId: options.projectId,
posts: langRequestedPagePosts,
renderRoute: langRenderRoute,
writePage: langWritePage,
onPageGenerated,
}
if (requestedPagePosts.length > 0) {
renderedUrlCount += await generatePageRoutes({
projectId: options.projectId, posts: requestedPagePosts,
renderRoute, writePage, onPageGenerated,
});
}
break;
case 'single':
if (requestedSinglePosts.length > 0) {
renderedUrlCount += await generateSinglePostPages({
projectId: options.projectId, posts: requestedSinglePosts,
renderRoute, writePage, onPageGenerated,
});
}
break;
case 'category':
if (targetedPlan.requestedCategorySet.size > 0) {
renderedUrlCount += await generateCategoryPages({
projectId: options.projectId, posts: publishedListPosts,
allCategories: targetedPlan.requestedCategorySet, maxPostsPerPage,
renderRoute, writePage, onPageGenerated,
postsByCategory: generationPostIndex.postsByCategory,
});
}
break;
case 'tag':
if (targetedPlan.requestedTagSet.size > 0) {
renderedUrlCount += await generateTagPages({
projectId: options.projectId, posts: publishedListPosts,
allTags: targetedPlan.requestedTagSet, maxPostsPerPage,
renderRoute, writePage, onPageGenerated,
postsByTag: generationPostIndex.postsByTag,
});
}
break;
case 'date':
if (requestedYearsMap.size > 0 || requestedYearMonthsMap.size > 0 || requestedYearMonthDaysMap.size > 0) {
renderedUrlCount += await generateDateArchivePages({
projectId: options.projectId, posts: publishedListPosts,
yearsMap: requestedYearsMap, yearMonthsMap: requestedYearMonthsMap,
yearMonthDaysMap: requestedYearMonthDaysMap, maxPostsPerPage,
renderRoute, writePage, onPageGenerated,
postsByYear: generationPostIndex.postsByYear,
postsByYearMonth: generationPostIndex.postsByYearMonth,
postsByYearMonthDay: generationPostIndex.postsByYearMonthDay,
});
}
break;
}
// Language subtrees for this section
for (const [lang, langMissingPlan] of missingPathPlan.languagePlans) {
const langPosts = publishedPosts.filter((p) => !(p as PostData & { doNotTranslate?: boolean }).doNotTranslate);
const langListPosts = publishedListPosts.filter((p) => !(p as PostData & { doNotTranslate?: boolean }).doNotTranslate);
const langPostIndex = buildGenerationPostIndex(langListPosts);
const langArchives = buildApplyValidationArchives(langListPosts);
const langTargetedPlan = buildTargetedValidationPlan({
initialPlan: langMissingPlan,
publishedPosts: langPosts,
allCategories: langArchives.allCategories,
allTags: langArchives.allTags,
availableYearMonths: langArchives.yearMonths.keys(),
availableYearMonthDays: langArchives.yearMonthDays.keys(),
});
const langRenderRoute = createPreviewBackedGenerationRouteRenderer({
options: { ...options, language: lang },
maxPostsPerPage,
publishedPostsForLookup: langPosts,
languagePrefix: `/${lang}`,
engines: {
postEngine: this.postEngine,
mediaEngine: this.mediaEngine,
postMediaEngine: this.postMediaEngine,
},
});
const langWritePage = (projectId: string, urlPath: string, content: string) => writeHtmlPage({
projectId,
htmlDir,
urlPath: `${lang}/${urlPath}`,
content,
refreshHashTimestampOnUnchanged: true,
});
switch (section) {
case 'core':
if (langTargetedPlan.requestRootRoutes) {
renderedUrlCount += await generateRootPages({
projectId: options.projectId, posts: langListPosts, maxPostsPerPage,
renderRoute: langRenderRoute, writePage: langWritePage, onPageGenerated,
});
const langPagePosts = selectRequestedPosts({
publishedPosts: langPosts, requestedPostIds: new Set(), requestedPageSlugs: langTargetedPlan.requestedPageSlugs,
}).requestedPagePosts;
if (langPagePosts.length > 0) {
renderedUrlCount += await generatePageRoutes({
projectId: options.projectId, posts: langPagePosts,
renderRoute: langRenderRoute, writePage: langWritePage, onPageGenerated,
});
}
}
break;
case 'category':
if (langTargetedPlan.requestedCategorySet.size > 0) {
renderedUrlCount += await generateCategoryPages({
projectId: options.projectId, posts: langListPosts,
allCategories: langTargetedPlan.requestedCategorySet, maxPostsPerPage,
renderRoute: langRenderRoute, writePage: langWritePage, onPageGenerated,
postsByCategory: langPostIndex.postsByCategory,
});
}
break;
case 'tag':
if (langTargetedPlan.requestedTagSet.size > 0) {
renderedUrlCount += await generateTagPages({
projectId: options.projectId, posts: langListPosts,
allTags: langTargetedPlan.requestedTagSet, maxPostsPerPage,
renderRoute: langRenderRoute, writePage: langWritePage, onPageGenerated,
postsByTag: langPostIndex.postsByTag,
});
}
break;
case 'single': {
const langSinglePosts = selectRequestedPosts({
publishedPosts: langPosts, requestedPostIds: langTargetedPlan.requestedPostIds, requestedPageSlugs: new Set(),
}).requestedSinglePosts;
if (langSinglePosts.length > 0) {
renderedUrlCount += await generateSinglePostPages({
projectId: options.projectId, posts: langSinglePosts,
renderRoute: langRenderRoute, writePage: langWritePage, onPageGenerated,
});
}
break;
}
if (langTargetedPlan.requestedCategorySet.size > 0) {
renderedUrlCount += await generateCategoryPages({
projectId: options.projectId,
posts: langListPosts,
allCategories: langTargetedPlan.requestedCategorySet,
maxPostsPerPage,
renderRoute: langRenderRoute,
writePage: langWritePage,
onPageGenerated,
postsByCategory: langPostIndex.postsByCategory,
});
}
if (langTargetedPlan.requestedTagSet.size > 0) {
renderedUrlCount += await generateTagPages({
projectId: options.projectId,
posts: langListPosts,
allTags: langTargetedPlan.requestedTagSet,
maxPostsPerPage,
renderRoute: langRenderRoute,
writePage: langWritePage,
onPageGenerated,
postsByTag: langPostIndex.postsByTag,
});
}
const langRequestedSinglePosts = selectRequestedPosts({
publishedPosts: langPosts,
requestedPostIds: langTargetedPlan.requestedPostIds,
requestedPageSlugs: new Set(),
}).requestedSinglePosts;
if (langRequestedSinglePosts.length > 0) {
renderedUrlCount += await generateSinglePostPages({
projectId: options.projectId,
posts: langRequestedSinglePosts,
renderRoute: langRenderRoute,
writePage: langWritePage,
onPageGenerated,
});
}
const langRequestedArchives = buildRequestedArchiveMaps({
requestedYears: langTargetedPlan.requestedYears,
requestedYearMonths: langTargetedPlan.requestedYearMonths,
requestedYearMonthDays: langTargetedPlan.requestedYearMonthDays,
years: langArchives.years,
yearMonths: langArchives.yearMonths,
yearMonthDays: langArchives.yearMonthDays,
});
if (langRequestedArchives.requestedYearsMap.size > 0 || langRequestedArchives.requestedYearMonthsMap.size > 0 || langRequestedArchives.requestedYearMonthDaysMap.size > 0) {
renderedUrlCount += await generateDateArchivePages({
projectId: options.projectId,
posts: langListPosts,
yearsMap: langRequestedArchives.requestedYearsMap,
yearMonthsMap: langRequestedArchives.requestedYearMonthsMap,
yearMonthDaysMap: langRequestedArchives.requestedYearMonthDaysMap,
maxPostsPerPage,
renderRoute: langRenderRoute,
writePage: langWritePage,
onPageGenerated,
postsByYear: langPostIndex.postsByYear,
postsByYearMonth: langPostIndex.postsByYearMonth,
postsByYearMonthDay: langPostIndex.postsByYearMonthDay,
case 'date': {
const langArchiveMaps = buildRequestedArchiveMaps({
requestedYears: langTargetedPlan.requestedYears,
requestedYearMonths: langTargetedPlan.requestedYearMonths,
requestedYearMonthDays: langTargetedPlan.requestedYearMonthDays,
years: langArchives.years, yearMonths: langArchives.yearMonths, yearMonthDays: langArchives.yearMonthDays,
});
if (langArchiveMaps.requestedYearsMap.size > 0 || langArchiveMaps.requestedYearMonthsMap.size > 0 || langArchiveMaps.requestedYearMonthDaysMap.size > 0) {
renderedUrlCount += await generateDateArchivePages({
projectId: options.projectId, posts: langListPosts,
yearsMap: langArchiveMaps.requestedYearsMap,
yearMonthsMap: langArchiveMaps.requestedYearMonthsMap,
yearMonthDaysMap: langArchiveMaps.requestedYearMonthDaysMap,
maxPostsPerPage, renderRoute: langRenderRoute, writePage: langWritePage, onPageGenerated,
postsByYear: langPostIndex.postsByYear, postsByYearMonth: langPostIndex.postsByYearMonth,
postsByYearMonthDay: langPostIndex.postsByYearMonthDay,
});
}
break;
}
}
}
if (renderedUrlCount > 0 || deletedUrlCount > 0) {
onProgress(90, 'Regenerating calendar data...');
await this.regenerateCalendar(options, (progress, message) => {
const mappedProgress = 90 + Math.floor((progress / 100) * 9);
onProgress(Math.min(99, mappedProgress), message || 'Regenerating calendar data...');
});
}
onProgress(100, `Apply complete (${deletedUrlCount} deleted, ${renderedUrlCount} rendered)`);
return {
renderedUrlCount,
deletedUrlCount,
removedEmptyDirCount,
};
onProgress(100, `${section} rendering complete`);
return renderedUrlCount;
}
}

View File

@@ -282,31 +282,18 @@ export function buildTargetedValidationPlan(params: BuildTargetedValidationPlanP
}
}
for (const year of Array.from(requestedYears.values())) {
for (const ym of availableYearMonths) {
if (ym.startsWith(`${year}/`)) {
requestedYearMonths.add(ym);
}
}
}
for (const ym of Array.from(requestedYearMonths.values())) {
for (const ymd of availableYearMonthDays) {
if (ymd.startsWith(`${ym}/`)) {
requestedYearMonthDays.add(ymd);
}
}
const [yearStr] = ym.split('/');
requestedYears.add(Number(yearStr));
}
// Upward cascade only: day → month → year
for (const ymd of Array.from(requestedYearMonthDays.values())) {
const [yearStr, monthStr] = ymd.split('/');
requestedYears.add(Number(yearStr));
requestedYearMonths.add(`${yearStr}/${monthStr}`);
}
for (const ym of Array.from(requestedYearMonths.values())) {
const [yearStr] = ym.split('/');
requestedYears.add(Number(yearStr));
}
return {
requestedPostIds,
requestedCategorySet: new Set(

View File

@@ -5,6 +5,7 @@ import {
type BlogGenerationSection,
type BlogGenerationOptions,
type SiteValidationReport,
type ApplyValidationPreparation,
} from '../engine/BlogGenerationEngine';
import { resolvePageTitle } from '../engine/PageRenderer';
import type { EngineBundle } from '../engine/EngineBundle';
@@ -328,14 +329,71 @@ export function registerBlogHandlers(safeHandle: SafeHandle, bundle: EngineBundl
const baseOptions = await resolveBlogGenerationBaseOptions();
const taskTimestamp = Date.now();
return bundle.taskManager.runTask({
id: `site-validate-apply-${taskTimestamp}`,
name: 'Apply Site Validation',
const taskGroupId = `site-validate-apply-${taskTimestamp}`;
const taskGroupName = 'Apply Site Validation';
const sectionDisplayNames: Record<BlogGenerationSection, string> = {
core: 'Render Site Core',
single: 'Render Single Posts',
category: 'Render Category Archives',
tag: 'Render Tag Archives',
date: 'Render Date Archives',
};
// Phase 1: Prepare — plan & delete extra URLs
const preparation = await bundle.taskManager.runTask({
id: `site-validate-prepare-${taskTimestamp}`,
name: 'Prepare Validation Apply',
groupId: taskGroupId,
groupName: taskGroupName,
execute: async (onProgress) => {
return blogGenerationEngine.applyValidation(baseOptions, report, (progress, message) => {
onProgress(progress, message || 'Applying site validation...');
return blogGenerationEngine.prepareApplyValidation(baseOptions, report, (progress, message) => {
onProgress(progress, message || 'Preparing validation apply...');
});
},
});
// Phase 2: Render sections in parallel (grouped tasks)
let renderedUrlCount = 0;
if (preparation.sectionsToRender.length > 0) {
const sectionResults = await Promise.all(
preparation.sectionsToRender.map((section) =>
bundle.taskManager.runTask({
id: `site-validate-render-${section}-${taskTimestamp}`,
name: sectionDisplayNames[section],
groupId: taskGroupId,
groupName: taskGroupName,
execute: async (onProgress) => {
return blogGenerationEngine.applyValidationForSection(baseOptions, preparation, section, (progress, message) => {
onProgress(progress, message || `Rendering ${section} routes...`);
});
},
}),
),
);
renderedUrlCount = sectionResults.reduce((sum, count) => sum + count, 0);
}
// Phase 3: Regenerate calendar (if anything changed)
if (renderedUrlCount > 0 || preparation.deletedUrlCount > 0) {
await bundle.taskManager.runTask({
id: `site-validate-calendar-${taskTimestamp}`,
name: 'Regenerate Calendar',
groupId: taskGroupId,
groupName: taskGroupName,
execute: async (onProgress) => {
return blogGenerationEngine.regenerateCalendar(baseOptions, (progress, message) => {
onProgress(progress, message || 'Regenerating calendar...');
});
},
});
}
return {
renderedUrlCount,
deletedUrlCount: preparation.deletedUrlCount,
removedEmptyDirCount: preparation.removedEmptyDirCount,
};
});
}

View File

@@ -449,6 +449,8 @@ async function processBlogmarkDeepLink(rawDeepLink: string): Promise<void> {
content: transformResult.post.content,
tags: transformResult.post.tags,
categories: transformResult.post.categories,
language: (metadata as { mainLanguage?: string } | null)?.mainLanguage || undefined,
author: (metadata as { defaultAuthor?: string } | null)?.defaultAuthor || undefined,
});
const blogmarkCreatedPayload = {

View File

@@ -235,6 +235,17 @@ export const PostEditor: React.FC<PostEditorProps> = ({ postId }) => {
const canonicalLanguage = postLanguage || post?.language || projectLanguage;
const fieldIdPrefix = `post-editor-${postId}`;
// Keep activeEditingLanguage in sync when canonicalLanguage changes
// (e.g. due to async projectLanguage loading after post init)
const prevCanonicalLanguageRef = useRef(canonicalLanguage);
useEffect(() => {
const prev = prevCanonicalLanguageRef.current;
prevCanonicalLanguageRef.current = canonicalLanguage;
if (prev !== canonicalLanguage && activeEditingLanguage === prev) {
setActiveEditingLanguage(canonicalLanguage);
}
}, [canonicalLanguage, activeEditingLanguage]);
const loadTranslations = useCallback(async () => {
const result = await window.electronAPI?.posts.getTranslations?.(postId);
const items = result || [];