From 5efbcfe03a9662a9171268a78ebc5c32a86ed194 Mon Sep 17 00:00:00 2001 From: hugo Date: Tue, 24 Feb 2026 23:02:12 +0100 Subject: [PATCH] feat: more on incremental rendering --- src/main/database/generatedFileHashStore.ts | 32 +++++ src/main/engine/BlogGenerationEngine.ts | 31 ++++- .../engine/BlogGenerationOutputService.ts | 7 ++ src/main/engine/SiteValidationDiffService.ts | 45 +++++++ src/main/shared/electronApi.ts | 1 + .../SiteValidationView/SiteValidationView.tsx | 20 ++- src/renderer/i18n/locales/de.json | 4 +- src/renderer/i18n/locales/en.json | 4 +- src/renderer/i18n/locales/es.json | 4 +- src/renderer/i18n/locales/fr.json | 4 +- src/renderer/i18n/locales/it.json | 4 +- tests/engine/BlogGenerationEngine.test.ts | 117 ++++++++++++++++++ .../engine/SiteValidationDiffService.test.ts | 38 ++++++ 13 files changed, 303 insertions(+), 8 deletions(-) diff --git a/src/main/database/generatedFileHashStore.ts b/src/main/database/generatedFileHashStore.ts index 1d41e67..cca71c5 100644 --- a/src/main/database/generatedFileHashStore.ts +++ b/src/main/database/generatedFileHashStore.ts @@ -1,5 +1,10 @@ import { getDatabase } from './connection'; +export interface GeneratedFileHashRecord { + contentHash: string; + updatedAt: number; +} + export async function getGeneratedFileHash(projectId: string, relativePath: string): Promise { const client = getDatabase().getLocalClient(); if (!client) { @@ -18,6 +23,33 @@ export async function getGeneratedFileHash(projectId: string, relativePath: stri return result.rows[0].content_hash; } +export async function getGeneratedFileHashRecord(projectId: string, relativePath: string): Promise { + const client = getDatabase().getLocalClient(); + if (!client) { + throw new Error('Database client not available'); + } + + const result = await client.execute({ + sql: 'SELECT content_hash, updated_at FROM generated_file_hashes WHERE project_id = ? AND relative_path = ? LIMIT 1', + args: [projectId, relativePath], + }); + + const row = result.rows[0]; + if (!row || typeof row.content_hash !== 'string') { + return null; + } + + const rawUpdatedAt = row.updated_at; + const updatedAt = typeof rawUpdatedAt === 'number' + ? rawUpdatedAt + : Number(rawUpdatedAt); + + return { + contentHash: row.content_hash, + updatedAt: Number.isFinite(updatedAt) ? updatedAt : 0, + }; +} + export async function setGeneratedFileHash(projectId: string, relativePath: string, hash: string): Promise { const client = getDatabase().getLocalClient(); if (!client) { diff --git a/src/main/engine/BlogGenerationEngine.ts b/src/main/engine/BlogGenerationEngine.ts index 3209a16..624ff9d 100644 --- a/src/main/engine/BlogGenerationEngine.ts +++ b/src/main/engine/BlogGenerationEngine.ts @@ -44,6 +44,7 @@ import { buildRequestedArchiveMaps, selectRequestedPosts, } from './ApplyValidationDataService'; +import { getGeneratedFileHashRecord } from '../database/generatedFileHashStore'; const DEFAULT_MAX_POSTS_PER_PAGE = 50; const MIN_MAX_POSTS_PER_PAGE = 1; @@ -96,6 +97,7 @@ export interface SiteValidationReport { sitemapChanged: boolean; missingUrlPaths: string[]; extraUrlPaths: string[]; + updatedPostUrlPaths: string[]; expectedUrlCount: number; existingHtmlUrlCount: number; } @@ -375,6 +377,7 @@ export class BlogGenerationEngine { content, knownDirectories: knownOutputDirectories, hashCache: generatedHashCache, + refreshHashTimestampOnUnchanged: true, }); let pagesGenerated = 0; @@ -551,19 +554,40 @@ export class BlogGenerationEngine { onProgress(50, 'Comparing sitemap to html pages...'); + const postTimestampChecks = await Promise.all(publishedPosts.map(async (post) => { + const createdAt = resolvePostCreatedAt(post); + const year = String(createdAt.getFullYear()); + const month = String(createdAt.getMonth() + 1).padStart(2, '0'); + const postFilePath = path.join(options.dataDir, 'posts', year, month, `${post.slug}.md`); + const postUrlPath = buildCanonicalPostPath(post); + const relativePath = `${postUrlPath.replace(/^\//, '')}/index.html`; + const generatedRecord = await getGeneratedFileHashRecord(options.projectId, relativePath); + + return { + postUrlPath, + postFilePath, + generatedUpdatedAtMs: generatedRecord?.updatedAt, + }; + })); + const diffResult = await compareSitemapToHtml({ sitemapXml, baseUrl: options.baseUrl, htmlDir, + postTimestampChecks, }); - onProgress(100, `Validation complete (${diffResult.missingUrlPaths.length} missing, ${diffResult.extraUrlPaths.length} extra)`); + onProgress( + 100, + `Validation complete (${diffResult.missingUrlPaths.length} missing, ${diffResult.extraUrlPaths.length} extra, ${diffResult.updatedPostUrlPaths.length} updated)` + ); return { sitemapPath, sitemapChanged, missingUrlPaths: diffResult.missingUrlPaths, extraUrlPaths: diffResult.extraUrlPaths, + updatedPostUrlPaths: diffResult.updatedPostUrlPaths, expectedUrlCount: diffResult.expectedUrlCount, existingHtmlUrlCount: diffResult.existingHtmlUrlCount, }; @@ -577,11 +601,13 @@ export class BlogGenerationEngine { onProgress(0, 'Applying validation changes...'); const missingPaths = Array.isArray(report.missingUrlPaths) ? report.missingUrlPaths : []; + const updatedPostPaths = Array.isArray(report.updatedPostUrlPaths) ? report.updatedPostUrlPaths : []; + const rerenderPaths = Array.from(new Set([...missingPaths, ...updatedPostPaths])); const extraPaths = Array.isArray(report.extraUrlPaths) ? report.extraUrlPaths : []; onProgress(10, 'Planning validation apply steps...'); - const missingPathPlan = planMissingValidationPaths(missingPaths); + const missingPathPlan = planMissingValidationPaths(rerenderPaths); onProgress(20, 'Deleting extra URLs...'); @@ -686,6 +712,7 @@ export class BlogGenerationEngine { htmlDir, urlPath, content, + refreshHashTimestampOnUnchanged: true, }); const onPageGenerated = (_message: string) => { // no-op for applyValidation diff --git a/src/main/engine/BlogGenerationOutputService.ts b/src/main/engine/BlogGenerationOutputService.ts index 344cf2f..12314b1 100644 --- a/src/main/engine/BlogGenerationOutputService.ts +++ b/src/main/engine/BlogGenerationOutputService.ts @@ -60,6 +60,7 @@ export async function writeFileIfHashChanged(params: { getGeneratedFileHash?: (projectId: string, relativePath: string) => Promise; setGeneratedFileHash?: (projectId: string, relativePath: string, hash: string) => Promise; computeHash?: (content: string) => string; + refreshHashTimestampOnUnchanged?: boolean; }): Promise { const getHash = params.getGeneratedFileHash ?? getGeneratedFileHash; const setHash = params.setGeneratedFileHash ?? setGeneratedFileHash; @@ -75,6 +76,10 @@ export async function writeFileIfHashChanged(params: { } if (previousHash === hash) { + if (params.refreshHashTimestampOnUnchanged) { + await setHash(params.projectId, params.relativePath, hash); + params.hashCache?.set(params.relativePath, hash); + } return false; } @@ -95,6 +100,7 @@ export async function writeHtmlPage(params: { getGeneratedFileHash?: (projectId: string, relativePath: string) => Promise; setGeneratedFileHash?: (projectId: string, relativePath: string, hash: string) => Promise; computeHash?: (content: string) => string; + refreshHashTimestampOnUnchanged?: boolean; }): Promise { const normalizedPath = params.urlPath.replace(/^\//, ''); const filePath = normalizedPath @@ -124,6 +130,7 @@ export async function writeHtmlPage(params: { getGeneratedFileHash: params.getGeneratedFileHash, setGeneratedFileHash: params.setGeneratedFileHash, computeHash: params.computeHash, + refreshHashTimestampOnUnchanged: params.refreshHashTimestampOnUnchanged, }); } diff --git a/src/main/engine/SiteValidationDiffService.ts b/src/main/engine/SiteValidationDiffService.ts index 90d43bc..1e4eaa6 100644 --- a/src/main/engine/SiteValidationDiffService.ts +++ b/src/main/engine/SiteValidationDiffService.ts @@ -4,14 +4,22 @@ import * as path from 'node:path'; export interface SiteValidationDiffResult { missingUrlPaths: string[]; extraUrlPaths: string[]; + updatedPostUrlPaths: string[]; expectedUrlCount: number; existingHtmlUrlCount: number; } +export interface PostTimestampCheck { + postUrlPath: string; + postFilePath: string; + generatedUpdatedAtMs?: number; +} + interface CompareSitemapToHtmlParams { sitemapXml: string; baseUrl: string; htmlDir: string; + postTimestampChecks?: PostTimestampCheck[]; } function normalizeUrlPath(urlPath: string): string { @@ -127,9 +135,46 @@ export async function compareSitemapToHtml(params: CompareSitemapToHtmlParams): .filter((value, index, array) => array.indexOf(value) === index) .sort(); + const updatedPostPathSet = new Set(); + const postTimestampChecks = Array.isArray(params.postTimestampChecks) ? params.postTimestampChecks : []; + for (const check of postTimestampChecks) { + const normalizedPostUrlPath = normalizeUrlPath(check.postUrlPath); + if (!expectedPathSet.has(normalizedPostUrlPath)) { + continue; + } + + if (missingUrlPaths.includes(normalizedPostUrlPath)) { + continue; + } + + const htmlPath = path.join(params.htmlDir, normalizedPostUrlPath === '/' ? 'index.html' : normalizedPostUrlPath.slice(1), 'index.html'); + + let htmlStat: Awaited>; + let postStat: Awaited>; + + try { + htmlStat = await fs.stat(htmlPath); + postStat = await fs.stat(check.postFilePath); + } catch { + continue; + } + + const generatedUpdatedAtMs = typeof check.generatedUpdatedAtMs === 'number' + ? check.generatedUpdatedAtMs + : 0; + const effectiveGeneratedAtMs = Math.max(htmlStat.mtimeMs, generatedUpdatedAtMs); + + if (postStat.mtimeMs > effectiveGeneratedAtMs) { + updatedPostPathSet.add(normalizedPostUrlPath); + } + } + + const updatedPostUrlPaths = Array.from(updatedPostPathSet.values()).sort(); + return { missingUrlPaths, extraUrlPaths, + updatedPostUrlPaths, expectedUrlCount: expectedPathSet.size, existingHtmlUrlCount: existingHtmlPathSet.size, }; diff --git a/src/main/shared/electronApi.ts b/src/main/shared/electronApi.ts index 437bcbf..f29b855 100644 --- a/src/main/shared/electronApi.ts +++ b/src/main/shared/electronApi.ts @@ -436,6 +436,7 @@ export interface SiteValidationReport { sitemapChanged: boolean; missingUrlPaths: string[]; extraUrlPaths: string[]; + updatedPostUrlPaths: string[]; expectedUrlCount: number; existingHtmlUrlCount: number; } diff --git a/src/renderer/components/SiteValidationView/SiteValidationView.tsx b/src/renderer/components/SiteValidationView/SiteValidationView.tsx index f3e100a..d62c910 100644 --- a/src/renderer/components/SiteValidationView/SiteValidationView.tsx +++ b/src/renderer/components/SiteValidationView/SiteValidationView.tsx @@ -10,6 +10,7 @@ type SiteValidationReport = { sitemapChanged: boolean; missingUrlPaths: string[]; extraUrlPaths: string[]; + updatedPostUrlPaths: string[]; expectedUrlCount: number; existingHtmlUrlCount: number; }; @@ -62,7 +63,8 @@ export const SiteValidationView: React.FC = () => { const canApply = useMemo(() => { if (!report) return false; - return report.missingUrlPaths.length > 0 || report.extraUrlPaths.length > 0; + const updatedPostUrlPaths = Array.isArray(report.updatedPostUrlPaths) ? report.updatedPostUrlPaths : []; + return report.missingUrlPaths.length > 0 || report.extraUrlPaths.length > 0 || updatedPostUrlPaths.length > 0; }, [report]); const handleApply = async () => { @@ -101,6 +103,8 @@ export const SiteValidationView: React.FC = () => { ); } + const updatedPostUrlPaths = Array.isArray(report.updatedPostUrlPaths) ? report.updatedPostUrlPaths : []; + return (
@@ -110,6 +114,7 @@ export const SiteValidationView: React.FC = () => { existing: report.existingHtmlUrlCount, missing: report.missingUrlPaths.length, extra: report.extraUrlPaths.length, + updated: updatedPostUrlPaths.length, })}

@@ -139,6 +144,19 @@ export const SiteValidationView: React.FC = () => { )} +
+

{tr('siteValidation.updatedTitle')}

+ {updatedPostUrlPaths.length === 0 ? ( +

{tr('siteValidation.noneUpdated')}

+ ) : ( +
    + {updatedPostUrlPaths.map((urlPath) => ( +
  • {urlPath}
    • + ))} +
+ )} +
+