Add Python macro worker runtime, ScriptEngine resolution, and PageRenderer/registry integration

Co-authored-by: rfc1437 <774975+rfc1437@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-02-26 21:41:26 +00:00
parent dd5a2e3377
commit b34cb4a110
8 changed files with 679 additions and 23 deletions

View File

@@ -11,6 +11,25 @@ import { CALENDAR_RUNTIME_JS } from './assets/calendarRuntime';
import { TAG_CLOUD_RUNTIME_JS } from './assets/tagCloudRuntime';
import { resolveRenderLanguageFromProjectPreferences, translateRender } from '../shared/i18n';
export interface PythonMacroScript {
id: string;
slug: string;
entrypoint: string;
content: string;
version: number;
}
export interface PythonMacroRendererContract {
getEnabledMacroScripts(): Promise<PythonMacroScript[]>;
renderMacro(params: {
scriptContent: string;
entrypoint: string;
contextJson: string;
cacheKey?: string;
timeoutMs?: number;
}): Promise<{ html: string; data?: Record<string, unknown>; warnings?: string[] }>;
}
export interface HtmlRewriteContext {
canonicalPostPathBySlug: Map<string, string>;
canonicalMediaPathBySourcePath: Map<string, string>;
@@ -801,6 +820,107 @@ export function renderMacro(
return '';
}
const JS_BUILTIN_MACROS = new Set(['youtube', 'vimeo', 'gallery', 'photo_archive', 'photo_album', 'tag_cloud']);
export function isBuiltInMacro(name: string): boolean {
return JS_BUILTIN_MACROS.has(normalizeMacroName(name));
}
export async function replaceAllMacrosAsync(
content: string,
postId: string,
mediaItems: MediaData[],
linkedMediaIds: Set<string> | null,
tagUsage: TagUsageEntry[],
renderLanguage: string,
pythonMacroRenderer?: PythonMacroRendererContract | null,
): Promise<string> {
const macroRegex = /\[\[(\w+)(?:\s+([^\]]+))?\]\]/g;
const matches: Array<{ fullMatch: string; name: string; rawParams: string | undefined; start: number; end: number }> = [];
let match: RegExpExecArray | null = null;
while ((match = macroRegex.exec(content)) !== null) {
matches.push({
fullMatch: match[0],
name: match[1].toLowerCase(),
rawParams: match[2],
start: match.index,
end: match.index + match[0].length,
});
}
if (matches.length === 0) {
return content;
}
let pythonScripts: PythonMacroScript[] | null = null;
const hasUnknownMacros = matches.some((m) => !isBuiltInMacro(m.name));
if (hasUnknownMacros && pythonMacroRenderer) {
try {
pythonScripts = await pythonMacroRenderer.getEnabledMacroScripts();
} catch {
pythonScripts = [];
}
}
const scriptsBySlug = new Map<string, PythonMacroScript>();
if (pythonScripts) {
for (const script of pythonScripts) {
scriptsBySlug.set(script.slug.toLowerCase(), script);
}
}
const rendered: string[] = [];
for (const m of matches) {
const params = parseMacroParams(m.rawParams);
const builtInResult = renderMacro(m.name, params, postId, mediaItems, linkedMediaIds, tagUsage, renderLanguage);
if (builtInResult || isBuiltInMacro(m.name)) {
rendered.push(builtInResult);
continue;
}
const pythonScript = scriptsBySlug.get(normalizeMacroName(m.name));
if (pythonScript && pythonMacroRenderer) {
try {
const context = {
env: {
isPreview: false,
mainLanguage: renderLanguage,
hook: m.name,
source: { kind: 'macro', id: pythonScript.id },
},
params: params as Record<string, unknown>,
};
const result = await pythonMacroRenderer.renderMacro({
scriptContent: pythonScript.content,
entrypoint: pythonScript.entrypoint,
contextJson: JSON.stringify(context),
cacheKey: `${pythonScript.id}:${pythonScript.version}`,
timeoutMs: 10000,
});
rendered.push(result.html);
} catch {
rendered.push('');
}
} else {
rendered.push('');
}
}
let result = content;
for (let i = matches.length - 1; i >= 0; i--) {
const m = matches[i];
result = result.slice(0, m.start) + rendered[i] + result.slice(m.end);
}
return result;
}
export function buildCanonicalPostPath(post: PostData): string {
const year = post.createdAt.getFullYear();
const month = String(post.createdAt.getMonth() + 1).padStart(2, '0');
@@ -898,12 +1018,19 @@ export class PageRenderer {
private readonly mediaEngine: MediaEngineContract;
private readonly postMediaEngine: PostMediaEngineContract;
private readonly postEngineForMacros?: PostEngineContract;
private readonly pythonMacroRenderer?: PythonMacroRendererContract;
private readonly liquid: Liquid;
constructor(mediaEngine: MediaEngineContract, postMediaEngine: PostMediaEngineContract, postEngineForMacros?: PostEngineContract) {
constructor(
mediaEngine: MediaEngineContract,
postMediaEngine: PostMediaEngineContract,
postEngineForMacros?: PostEngineContract,
pythonMacroRenderer?: PythonMacroRendererContract,
) {
this.mediaEngine = mediaEngine;
this.postMediaEngine = postMediaEngine;
this.postEngineForMacros = postEngineForMacros;
this.pythonMacroRenderer = pythonMacroRenderer;
const templateRoots = resolvePageRendererTemplateRoots();
@@ -951,10 +1078,9 @@ export class PageRenderer {
.catch(() => null)
: null;
const withMacros = content.replace(/\[\[(\w+)(?:\s+([^\]]+))?\]\]/g, (_match, macroName: string, rawParams: string | undefined) => {
const params = parseMacroParams(rawParams);
return renderMacro(macroName.toLowerCase(), params, postId, mediaItems, linkedMediaIds, tagUsage, renderLanguage);
});
const withMacros = await replaceAllMacrosAsync(
content, postId, mediaItems, linkedMediaIds, tagUsage, renderLanguage, this.pythonMacroRenderer,
);
const markdownHtml = await marked.parse(withMacros, { async: true, gfm: true, breaks: false });
const annotatedMarkdownHtml = annotateCodeBlocksWithLanguage(markdownHtml);

View File

@@ -19,7 +19,10 @@ import {
type HtmlRewriteContext,
type MediaEngineContract,
type PostMediaEngineContract,
type PythonMacroRendererContract,
} from './PageRenderer';
import { getScriptEngine } from './ScriptEngine';
import { getPythonMacroWorkerRuntime } from './PythonMacroWorkerRuntime';
import { getPicoStylesheetHref, sanitizePicoTheme, sanitizePicoThemeMode } from '../shared/picoThemes';
import { renderRouteWithSharedContext } from './SharedRouteRenderer';
import {
@@ -103,7 +106,7 @@ export class PreviewServer {
projectDescription: activeProject?.description ?? undefined,
};
});
this.pageRenderer = new PageRenderer(this.mediaEngine, this.postMediaEngine, this.postEngine);
this.pageRenderer = new PageRenderer(this.mediaEngine, this.postMediaEngine, this.postEngine, buildPythonMacroRenderer());
}
async start(preferredPort = 0): Promise<number> {
@@ -614,3 +617,21 @@ export class PreviewServer {
res.end(body);
}
}
function buildPythonMacroRenderer(): PythonMacroRendererContract {
return {
async getEnabledMacroScripts() {
const scripts = await getScriptEngine().getEnabledMacroScripts();
return scripts.map((s) => ({
id: s.id,
slug: s.slug,
entrypoint: s.entrypoint,
content: s.content,
version: s.version,
}));
},
async renderMacro(params) {
return getPythonMacroWorkerRuntime().renderMacro(params);
},
};
}

View File

@@ -0,0 +1,301 @@
import * as path from 'path';
import { Worker } from 'worker_threads';
interface WorkerRenderMacroRequest {
type: 'renderMacro';
requestId: string;
scriptContent: string;
entrypoint: string;
contextJson: string;
cacheKey?: string;
}
interface WorkerReadyMessage {
type: 'ready';
}
interface WorkerMacroResultMessage {
type: 'macroResult';
requestId: string;
html: string;
data?: Record<string, unknown>;
warnings?: string[];
}
interface WorkerMacroErrorMessage {
type: 'macroError';
requestId: string;
error: string;
}
interface WorkerFatalErrorMessage {
type: 'error';
error: string;
}
type WorkerResponseMessage = WorkerReadyMessage | WorkerMacroResultMessage | WorkerMacroErrorMessage | WorkerFatalErrorMessage;
export interface MacroRenderParams {
scriptContent: string;
entrypoint: string;
contextJson: string;
timeoutMs?: number;
cacheKey?: string;
}
export interface MacroRenderResult {
html: string;
data?: Record<string, unknown>;
warnings?: string[];
}
interface QueuedRequest {
request: WorkerRenderMacroRequest;
timeoutMs: number;
resolve: (value: MacroRenderResult) => void;
reject: (error: Error) => void;
}
interface ActiveRequest extends QueuedRequest {
timeoutId: ReturnType<typeof setTimeout> | null;
}
export class PythonMacroWorkerRuntime {
private worker: Worker | null = null;
private workerReady = false;
private workerStartPromise: Promise<void> | null = null;
private workerStartResolve: (() => void) | null = null;
private workerStartReject: ((error: Error) => void) | null = null;
private activeRequest: ActiveRequest | null = null;
private queue: QueuedRequest[] = [];
private requestCounter = 0;
private _macroCount = 0;
private _errorCount = 0;
private _timeoutCount = 0;
async renderMacro(params: MacroRenderParams): Promise<MacroRenderResult> {
const requestId = this.nextRequestId();
const timeoutMs = params.timeoutMs ?? 5000;
return new Promise<MacroRenderResult>((resolve, reject) => {
this.queue.push({
request: {
type: 'renderMacro',
requestId,
scriptContent: params.scriptContent,
entrypoint: params.entrypoint,
contextJson: params.contextJson,
cacheKey: params.cacheKey,
},
timeoutMs,
resolve,
reject,
});
this.dispatchNext().catch((error) => {
reject(error instanceof Error ? error : new Error(String(error)));
});
});
}
get macroCount(): number {
return this._macroCount;
}
get errorCount(): number {
return this._errorCount;
}
get timeoutCount(): number {
return this._timeoutCount;
}
resetCounters(): void {
this._macroCount = 0;
this._errorCount = 0;
this._timeoutCount = 0;
}
dispose(): void {
this.rejectStartPromise(new Error('Python macro worker runtime disposed'));
this.rejectActiveAndQueue(new Error('Python macro worker runtime disposed'));
this.resetWorker();
}
private async dispatchNext(): Promise<void> {
if (this.activeRequest || this.queue.length === 0) {
return;
}
await this.ensureWorkerStarted();
const nextRequest = this.queue.shift();
if (!nextRequest) {
return;
}
const timeoutId = setTimeout(() => {
if (!this.activeRequest || this.activeRequest.request.requestId !== nextRequest.request.requestId) {
return;
}
this._timeoutCount += 1;
const timeoutError = new Error(`Python macro timed out after ${nextRequest.timeoutMs}ms`);
this.activeRequest.reject(timeoutError);
this.activeRequest = null;
this.resetWorker();
void this.dispatchNext();
}, nextRequest.timeoutMs);
this.activeRequest = {
...nextRequest,
timeoutId,
};
this.worker?.postMessage(nextRequest.request);
}
private async ensureWorkerStarted(): Promise<void> {
if (this.worker && this.workerReady) {
return;
}
if (this.workerStartPromise) {
return this.workerStartPromise;
}
const workerPath = path.join(__dirname, 'pythonMacro.worker.js');
this.worker = new Worker(workerPath);
this.workerReady = false;
this.worker.on('message', (message: WorkerResponseMessage) => {
this.handleWorkerMessage(message);
});
this.worker.on('error', (error) => {
this.handleWorkerCrash(error instanceof Error ? error : new Error(String(error)));
});
this.worker.on('exit', (code) => {
if (code !== 0) {
this.handleWorkerCrash(new Error(`Python macro worker exited with code ${code}`));
}
});
this.workerStartPromise = new Promise<void>((resolve, reject) => {
this.workerStartResolve = resolve;
this.workerStartReject = reject;
});
return this.workerStartPromise;
}
private handleWorkerMessage(message: WorkerResponseMessage): void {
if (message.type === 'ready') {
this.workerReady = true;
this.resolveStartPromise();
return;
}
if (message.type === 'error') {
this.handleWorkerCrash(new Error(message.error));
return;
}
const active = this.activeRequest;
if (!active) {
return;
}
if (active.request.requestId !== message.requestId) {
return;
}
if (active.timeoutId) {
clearTimeout(active.timeoutId);
}
this.activeRequest = null;
this._macroCount += 1;
if (message.type === 'macroResult') {
active.resolve({
html: message.html,
data: message.data,
warnings: message.warnings,
});
} else {
this._errorCount += 1;
active.reject(new Error(message.error));
}
void this.dispatchNext();
}
private handleWorkerCrash(error: Error): void {
this.rejectStartPromise(error);
this.rejectActiveAndQueue(error);
this.resetWorker();
}
private rejectActiveAndQueue(error: Error): void {
if (this.activeRequest) {
if (this.activeRequest.timeoutId) {
clearTimeout(this.activeRequest.timeoutId);
}
this.activeRequest.reject(error);
this.activeRequest = null;
}
while (this.queue.length > 0) {
const queued = this.queue.shift();
queued?.reject(error);
}
}
private resolveStartPromise(): void {
if (this.workerStartResolve) {
this.workerStartResolve();
}
this.workerStartResolve = null;
this.workerStartReject = null;
this.workerStartPromise = null;
}
private rejectStartPromise(error: Error): void {
if (this.workerStartReject) {
this.workerStartReject(error);
}
this.workerStartResolve = null;
this.workerStartReject = null;
this.workerStartPromise = null;
}
private resetWorker(): void {
if (this.worker) {
this.worker.removeAllListeners();
this.worker.terminate();
}
this.worker = null;
this.workerReady = false;
this.workerStartPromise = null;
this.workerStartResolve = null;
this.workerStartReject = null;
}
private nextRequestId(): string {
this.requestCounter += 1;
return `py-macro-${this.requestCounter}`;
}
}
let pythonMacroWorkerRuntimeInstance: PythonMacroWorkerRuntime | null = null;
export function getPythonMacroWorkerRuntime(): PythonMacroWorkerRuntime {
if (!pythonMacroWorkerRuntimeInstance) {
pythonMacroWorkerRuntimeInstance = new PythonMacroWorkerRuntime();
}
return pythonMacroWorkerRuntimeInstance;
}

View File

@@ -222,6 +222,24 @@ export class ScriptEngine extends EventEmitter {
return Promise.all(rows.map((item) => this.toScriptData(item)));
}
async getEnabledMacroScripts(): Promise<ScriptData[]> {
const rows = await this.getAllScriptRows();
const macroRows = rows.filter((row) => row.kind === 'macro' && row.enabled);
return Promise.all(macroRows.map((item) => this.toScriptData(item)));
}
async getMacroScriptBySlug(slug: string): Promise<ScriptData | null> {
const normalizedSlug = slug.toLowerCase();
const rows = await this.getAllScriptRows();
const match = rows.find(
(row) => row.kind === 'macro' && row.enabled && row.slug.toLowerCase() === normalizedSlug,
);
if (!match) {
return null;
}
return this.toScriptData(match);
}
async rebuildDatabaseFromFiles(): Promise<void> {
const db = getDatabase().getLocal();
const scriptsDir = this.getScriptsDir();

View File

@@ -0,0 +1,124 @@
import { parentPort } from 'worker_threads';
interface WorkerRenderMacroRequest {
type: 'renderMacro';
requestId: string;
scriptContent: string;
entrypoint: string;
contextJson: string;
cacheKey?: string;
}
interface WorkerReadyMessage {
type: 'ready';
}
interface WorkerMacroResultMessage {
type: 'macroResult';
requestId: string;
html: string;
data?: Record<string, unknown>;
warnings?: string[];
}
interface WorkerMacroErrorMessage {
type: 'macroError';
requestId: string;
error: string;
}
interface WorkerFatalErrorMessage {
type: 'error';
error: string;
}
type WorkerResponseMessage = WorkerReadyMessage | WorkerMacroResultMessage | WorkerMacroErrorMessage | WorkerFatalErrorMessage;
type PyodideRuntime = {
globals: {
set: (name: string, value: unknown) => void;
} | any;
runPythonAsync: (code: string) => Promise<unknown>;
};
let runtimePromise: Promise<PyodideRuntime> | null = null;
let lastCacheKey: string | null = null;
function postMessage(message: WorkerResponseMessage): void {
parentPort?.postMessage(message);
}
async function getRuntime(): Promise<PyodideRuntime> {
if (!runtimePromise) {
runtimePromise = (async () => {
const pyodideModule = await import('pyodide');
return (await pyodideModule.loadPyodide()) as unknown as PyodideRuntime;
})();
}
return runtimePromise;
}
async function renderMacro(request: WorkerRenderMacroRequest): Promise<void> {
try {
const runtime = await getRuntime();
const shouldReloadScript = !request.cacheKey || request.cacheKey !== lastCacheKey;
if (shouldReloadScript) {
await runtime.runPythonAsync(request.scriptContent);
lastCacheKey = request.cacheKey ?? null;
}
runtime.globals.set('__bds_macro_context_json', request.contextJson);
runtime.globals.set('__bds_macro_entrypoint', request.entrypoint);
const rawResult = await runtime.runPythonAsync(`
import json as _json
_macro_ctx = _json.loads(__bds_macro_context_json)
_macro_ep = __bds_macro_entrypoint
_macro_fn = globals().get(_macro_ep)
if _macro_fn is None or not callable(_macro_fn):
raise RuntimeError(f"Macro entrypoint '{_macro_ep}' is not callable")
_macro_result = _macro_fn(_macro_ctx)
if _macro_result is None:
raise RuntimeError("Macro function returned None")
if not isinstance(_macro_result, dict):
raise RuntimeError("Macro function must return a dict with at least an 'html' key")
if "html" not in _macro_result:
raise RuntimeError("Macro result must contain an 'html' key")
_json.dumps(_macro_result)
`);
const parsed = JSON.parse(String(rawResult));
postMessage({
type: 'macroResult',
requestId: request.requestId,
html: typeof parsed.html === 'string' ? parsed.html : '',
data: parsed.data,
warnings: Array.isArray(parsed.warnings) ? parsed.warnings : undefined,
});
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
postMessage({ type: 'macroError', requestId: request.requestId, error: message });
}
}
parentPort?.on('message', (message: WorkerRenderMacroRequest) => {
if (message.type !== 'renderMacro') {
return;
}
void renderMacro(message);
});
void getRuntime()
.then(() => {
postMessage({ type: 'ready' });
})
.catch((error) => {
const message = error instanceof Error ? error.message : String(error);
postMessage({ type: 'error', error: message });
});

View File

@@ -24,6 +24,9 @@ export type {
MacroParams,
MacroRenderContext,
ParsedMacro,
PythonMacroInfo,
PythonMacroResolver,
PythonMacroRendererFn,
} from './types';
// Re-export registry functions
@@ -39,4 +42,5 @@ export {
renderMacro,
renderAllMacros,
getEditorPreview,
setPythonMacroResolver,
} from './registry';

View File

@@ -5,11 +5,22 @@
* Macros self-register using registerMacro() function.
*/
import type { MacroDefinition, MacroParams, MacroRenderContext, ParsedMacro } from './types';
import type {
MacroDefinition,
MacroParams,
MacroRenderContext,
ParsedMacro,
PythonMacroResolver,
PythonMacroRendererFn,
} from './types';
// Internal registry storage
const macroRegistry = new Map<string, MacroDefinition>();
// Python macro resolution
let pythonMacroResolverFn: PythonMacroResolver | null = null;
let pythonMacroRendererFn: PythonMacroRendererFn | null = null;
/**
* Register a macro definition.
* Call this from each macro definition file.
@@ -25,6 +36,18 @@ export function registerMacro(macro: MacroDefinition): void {
macroRegistry.set(name, macro);
}
/**
* Set the Python macro resolver and renderer for preview rendering.
* When a macro is not found in the JS registry, the resolver will be called.
*/
export function setPythonMacroResolver(
resolver: PythonMacroResolver | null,
renderer: PythonMacroRendererFn | null,
): void {
pythonMacroResolverFn = resolver;
pythonMacroRendererFn = renderer;
}
/**
* Get a macro definition by name.
*
@@ -124,6 +147,7 @@ export function parseMacros(markdown: string): ParsedMacro[] {
/**
* Render a single macro to HTML.
* First checks JS registry, then falls back to Python macro resolver.
*
* @param macro - The parsed macro
* @param context - Render context
@@ -135,10 +159,7 @@ export async function renderMacro(
): Promise<string> {
const definition = getMacro(macro.name);
if (!definition) {
return `<span class="macro-error" title="Unknown macro: ${macro.name}">${macro.rawText}</span>`;
}
if (definition) {
// Validate if validator exists
if (definition.validate) {
const error = definition.validate(macro.params);
@@ -156,6 +177,20 @@ export async function renderMacro(
}
}
if (pythonMacroResolverFn && pythonMacroRendererFn) {
try {
const pythonInfo = await pythonMacroResolverFn(macro.name);
if (pythonInfo) {
return await pythonMacroRendererFn(pythonInfo, macro.params, context);
}
} catch {
return `<span class="macro-error" title="Python macro error">${macro.rawText}</span>`;
}
}
return `<span class="macro-error" title="Unknown macro: ${macro.name}">${macro.rawText}</span>`;
}
/**
* Render all macros in a markdown string to HTML.
* Returns the markdown with macros replaced by their rendered HTML.

View File

@@ -75,3 +75,30 @@ export interface ParsedMacro {
/** End position in the source text */
end: number;
}
/**
* Resolved Python macro script information for rendering.
*/
export interface PythonMacroInfo {
scriptId: string;
slug: string;
code: string;
entrypoint: string;
version: number;
}
/**
* Resolver function that checks if a macro name maps to a Python script.
* Returns script info if found, null otherwise.
*/
export type PythonMacroResolver = (macroName: string) => Promise<PythonMacroInfo | null>;
/**
* Renderer function that executes a Python macro with the given context.
* Returns the rendered HTML string.
*/
export type PythonMacroRendererFn = (
info: PythonMacroInfo,
params: MacroParams,
context: MacroRenderContext,
) => Promise<string>;