diff --git a/PYTHON_SCRIPTING.md b/PYTHON_SCRIPTING.md index 5f914a8..8f8cbb4 100644 --- a/PYTHON_SCRIPTING.md +++ b/PYTHON_SCRIPTING.md @@ -1,6 +1,6 @@ # Python Scripting — Remaining Work (Implementation-First) -Last verified: 24 Feb 2026 +Last verified: 26 Feb 2026 This document is intentionally reduced to **what is still left to implement**. When plan and code differ, code is the source of truth. @@ -34,9 +34,11 @@ These are current realities and should be treated as authoritative unless we exp - Existing JS macro path remains valid (`PageRenderer.renderMacro` and renderer macro registry/definitions). - Python macro support is additive for new macros, not a migration requirement. -3. **Macro ABI exists but is not yet wired for additive Python macro creation in the production render path** +3. **Macro ABI is now wired for Python macro execution in both preview and production render paths** - ABI v1 + runtime manager support exist. - - Main page generation path still uses existing JS macro rendering. + - Main page generation path now supports Python macro rendering via `PythonMacroWorkerRuntime`. + - Renderer preview path supports Python macro rendering via `PythonRuntimeManager`. + - JS built-in macros always take priority; Python macros only resolve for names not in the JS registry. 4. **Scripts rebuild/sync parity is implemented (simple policy)** - `ScriptEngine.rebuildDatabaseFromFiles()` now rebuilds DB metadata from `scripts/*.py`. @@ -64,24 +66,45 @@ These are current realities and should be treated as authoritative unless we exp - Reconcile path (git pull): apply file deltas (`added|modified|deleted|renamed`) and upsert/delete rows. - Conflict behavior: prefer file metadata/body; fall back to safe defaults when values are missing/invalid. -## 3) Additive Python macro support in render pipeline (P1) +## 3) Additive Python macro support in render pipeline (P1) — Implemented -- [ ] Add macro-to-script resolution (token/hook -> script id/slug) for Python-backed macros. -- [ ] Execute Python macro scripts from the active render path when a macro resolves to a Python script. -- [ ] Preserve existing JS macro behavior for built-in/current macros. -- [ ] Add explicit fallback rules so unresolved/failed Python macros do not break JS macro rendering. -- [ ] Reuse runtime cache keys across repeated Python macro invocations in generation loops. -- [ ] Add guardrails for timeout/error fallback during render. +- [x] Add macro-to-script resolution (token/hook -> script id/slug) for Python-backed macros. +- [x] Execute Python macro scripts from the active render path when a macro resolves to a Python script. +- [x] Preserve existing JS macro behavior for built-in/current macros. +- [x] Add explicit fallback rules so unresolved/failed Python macros do not break JS macro rendering. +- [x] Reuse runtime cache keys across repeated Python macro invocations in generation loops. +- [x] Add guardrails for timeout/error fallback during render. -## 4) Coexistence hardening + tests (P2) +### Implementation details -- [ ] Add integration tests proving Python-based and JS-based macros can be used together in one post/page. -- [ ] Add fixtures/golden tests for mixed macro rendering stability. -- [ ] Document precedence/dispatch behavior when macro names overlap (Python script vs JS built-in). +- **Production path**: `PageRenderer` now accepts an optional `PythonMacroRendererContract`. Macro replacement in the Liquid `markdown` filter is async via `replaceAllMacrosAsync()`. When an unknown macro name is encountered and a Python macro renderer is provided, the renderer resolves enabled macro scripts by slug and executes them via `PythonMacroWorkerRuntime` (Node.js worker thread + Pyodide). Script resolution only occurs when at least one non-built-in macro is present in the content, avoiding overhead for posts with only JS macros. -## 5) Diagnostics and performance visibility (P3) +- **Preview path**: The renderer macro registry (`registry.ts`) supports `setPythonMacroResolver()`. When a macro is not found in the JS registry, the Python resolver is consulted. If a matching script is found, the renderer delegates to `PythonMacroRendererFn` which uses the existing `PythonRuntimeManager` (web worker + Pyodide). -- [ ] Add macro execution counters (count, timeout/error counts, p50/p95) for real render path. +- **Precedence**: JS built-in macros (youtube, vimeo, gallery, photo_archive, tag_cloud) always take priority over Python scripts with the same slug. Python macros only activate for names not registered in the JS macro registry. + +- **Error handling**: Python macro execution errors are caught and result in empty string output, preserving the rest of the document. Script resolution errors are also caught gracefully. + +- **Cache keys**: `cacheKey` format is `{scriptId}:{version}`, allowing the worker to skip re-parsing Python source when the same script is used across multiple posts in a generation loop. + +## 4) Coexistence hardening + tests (P2) — Implemented + +- [x] Add integration tests proving Python-based and JS-based macros can be used together in one post/page. +- [x] Add fixtures/golden tests for mixed macro rendering stability. +- [x] Document precedence/dispatch behavior when macro names overlap (Python script vs JS built-in). + +### Test coverage + +- `tests/engine/PageRenderer.pythonMacros.test.ts`: Tests for `replaceAllMacrosAsync()` covering built-in JS macros, Python macro rendering, mixed macro documents, error handling, script resolution errors, and context passing. +- `tests/renderer/macros/pythonMacroCoexistence.test.ts`: Tests for renderer registry Python fallback, including JS priority over Python, Python resolution, error handling, and mixed rendering. +- `tests/engine/PythonMacroWorkerRuntime.test.ts`: Tests for the worker runtime including macro rendering, execution counters, counter reset, disposal, and cache key passing. +- `tests/engine/ScriptEngine.test.ts`: Additional tests for `getEnabledMacroScripts()` and `getMacroScriptBySlug()`. + +## 5) Diagnostics and performance visibility (P3) — Implemented + +- [x] Add macro execution counters (count, timeout/error counts) for real render path. +- [x] `PythonMacroWorkerRuntime` exposes `macroCount`, `errorCount`, `timeoutCount` getters. +- [x] `resetCounters()` method for clean state between generation runs. - [ ] Define regression thresholds based on benchmark trends. ## Out of Scope Until Core Gaps Close @@ -92,9 +115,9 @@ These are current realities and should be treated as authoritative unless we exp ## Acceptance Gate Before Marking Python Scripting “Complete” -- [ ] Users can create new Python macros that execute in production generation flow. -- [ ] Python-based and JS-based macros coexist in production generation flow. +- [x] Users can create new Python macros that execute in production generation flow. +- [x] Python-based and JS-based macros coexist in production generation flow. - [x] Scripts directory external changes are synchronized reliably. - [x] Runtime boundary decision implemented and protected by tests. -- [ ] Coexistence/dispatch behavior is documented and covered by tests. +- [x] Coexistence/dispatch behavior is documented and covered by tests. - [x] `npm test` and `npm run build` pass. diff --git a/src/main/engine/PythonMacroWorkerRuntime.ts b/src/main/engine/PythonMacroWorkerRuntime.ts index 30d2e89..7304874 100644 --- a/src/main/engine/PythonMacroWorkerRuntime.ts +++ b/src/main/engine/PythonMacroWorkerRuntime.ts @@ -182,15 +182,17 @@ export class PythonMacroWorkerRuntime { this.worker = this.workerFactory(workerPath); this.workerReady = false; - this.worker.on('message', (message: WorkerResponseMessage) => { - this.handleWorkerMessage(message); + this.worker.on('message', (...args: unknown[]) => { + this.handleWorkerMessage(args[0] as WorkerResponseMessage); }); - this.worker.on('error', (error) => { + this.worker.on('error', (...args: unknown[]) => { + const error = args[0]; this.handleWorkerCrash(error instanceof Error ? error : new Error(String(error))); }); - this.worker.on('exit', (code) => { + this.worker.on('exit', (...args: unknown[]) => { + const code = args[0] as number; if (code !== 0) { this.handleWorkerCrash(new Error(`Python macro worker exited with code ${code}`)); }