From ce050f98c3d7c017d12b1c10cf189fbb4b593f40 Mon Sep 17 00:00:00 2001 From: hugo Date: Sun, 22 Feb 2026 21:29:16 +0100 Subject: [PATCH] feat: python scripting phase 0 --- PYTHON_SCRIPTING.md | 35 ++- package-lock.json | 20 ++ package.json | 2 + scripts/python-runtime-benchmark.ts | 28 ++ src/renderer/python/PythonRuntimeManager.ts | 258 ++++++++++++++++++ src/renderer/python/abiV1.ts | 52 ++++ .../python/createPythonRuntimeWorker.ts | 5 + src/renderer/python/pythonRuntime.worker.ts | 128 +++++++++ src/renderer/python/pythonRuntimeBenchmark.ts | 117 ++++++++ src/renderer/python/runtimeProtocol.ts | 22 ++ .../python/PythonRuntimeManager.test.ts | 194 +++++++++++++ .../python/pythonRuntimeBenchmark.test.ts | 45 +++ 12 files changed, 895 insertions(+), 11 deletions(-) create mode 100644 scripts/python-runtime-benchmark.ts create mode 100644 src/renderer/python/PythonRuntimeManager.ts create mode 100644 src/renderer/python/abiV1.ts create mode 100644 src/renderer/python/createPythonRuntimeWorker.ts create mode 100644 src/renderer/python/pythonRuntime.worker.ts create mode 100644 src/renderer/python/pythonRuntimeBenchmark.ts create mode 100644 src/renderer/python/runtimeProtocol.ts create mode 100644 tests/renderer/python/PythonRuntimeManager.test.ts create mode 100644 tests/renderer/python/pythonRuntimeBenchmark.test.ts diff --git a/PYTHON_SCRIPTING.md b/PYTHON_SCRIPTING.md index ce741c1..8d79890 100644 --- a/PYTHON_SCRIPTING.md +++ b/PYTHON_SCRIPTING.md @@ -86,15 +86,22 @@ Contract rules: Objective: prove runtime viability before product surface growth. Deliverables: -- [ ] Add `pyodide` dependency and worker boot sequence. -- [ ] Run a sample script end-to-end (`run_script`, timeout, captured stdout). -- [ ] Benchmark baseline cold start + warm run + repeated macro calls. -- [ ] Define initial macro ABI (`render(context) -> result`) and schema docs. +- [x] Add `pyodide` dependency and worker boot sequence. +- [x] Run a sample script end-to-end (`run_script`, timeout, captured stdout). +- [x] Benchmark baseline cold start + warm run + repeated macro calls. +- [x] Define initial macro ABI (`render(context) -> result`) and schema docs. Exit criteria: -- Warm script execution is stable. -- Timeout recovery works. -- Measured baseline captured in repo docs. +- Warm script execution is stable. ✅ +- Timeout recovery works. ✅ +- Measured baseline captured in repo docs. ✅ + +Baseline benchmark (22 Feb 2026, local macOS run): +- Command: `npm run bench:python-runtime -- 200` +- Cold start: `701.11 ms` +- Warm run: `5.74 ms` +- Repeated macro (200 calls): `p50 0.17 ms`, `p95 0.29 ms`, `mean 0.19 ms` +- Notes: one-machine baseline only; use trend comparisons for regressions. ## Phase 1 — MVP (Minimal but Usable) @@ -447,9 +454,15 @@ PR-14+: Optional advanced capabilities ## 10. Current Status -Status: Revised staged plan (MVP-first, full-scope preserved). +Status: Phase 0 in progress (MVP-first, full-scope preserved). + +Progress update (22 Feb 2026): +- [x] PR-00 complete: Pyodide dependency + renderer worker bootstrap + ready signal. +- [x] PR-01 complete: worker run/stdout/error protocol + timeout watchdog + runtime recovery. +- [x] PR-02 complete: ABI v1 shared types/schemas + caller/worker validation. +- [x] Phase 0 benchmark harness + baseline capture complete. Recommended next action: -1. Approve Phase 0 scope and benchmarks. -2. Implement spike and record numbers. -3. Lock ABI before building full UI and migration layers. +1. Start Phase 1 PR-03: script persistence model (`scripts/*.py` + index metadata). +2. Add round-trip tests for create/update/delete between filesystem and DB. +3. Keep benchmark command in CI/manual perf checks for regressions. diff --git a/package-lock.json b/package-lock.json index 7d55db5..39c9c9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "liquidjs": "^10.24.0", "marked-react": "^3.0.2", "monaco-editor": "^0.55.1", + "pyodide": "^0.29.3", "react": "^19.2.4", "react-arborist": "^3.4.3", "react-dom": "^19.2.4", @@ -5265,6 +5266,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -12753,6 +12760,19 @@ "node": ">=6" } }, + "node_modules/pyodide": { + "version": "0.29.3", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.3.tgz", + "integrity": "sha512-22UBuhOJawj7vKUnS7/F3xK+515LJdjiMAHoCfuS6/PbHiOrSQVnYwDe+2sbVwiOZ3sMMexdXICew6NqOMQGgA==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", diff --git a/package.json b/package.json index 0e628d7..21a7eac 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:ui": "vitest --ui", + "bench:python-runtime": "node ./node_modules/tsx/dist/cli.mjs scripts/python-runtime-benchmark.ts", "lint": "eslint \"src/renderer/**/*.{ts,tsx}\" --max-warnings 0", "lint:i18n": "eslint \"src/renderer/**/*.{ts,tsx}\" --max-warnings 0", "db:generate": "node ./node_modules/drizzle-kit/bin.cjs generate", @@ -95,6 +96,7 @@ "liquidjs": "^10.24.0", "marked-react": "^3.0.2", "monaco-editor": "^0.55.1", + "pyodide": "^0.29.3", "react": "^19.2.4", "react-arborist": "^3.4.3", "react-dom": "^19.2.4", diff --git a/scripts/python-runtime-benchmark.ts b/scripts/python-runtime-benchmark.ts new file mode 100644 index 0000000..3c16882 --- /dev/null +++ b/scripts/python-runtime-benchmark.ts @@ -0,0 +1,28 @@ +import { runPythonRuntimeBenchmark } from '../src/renderer/python/pythonRuntimeBenchmark'; + +async function main(): Promise { + const iterationsArg = process.argv[2]; + const iterations = iterationsArg ? Number(iterationsArg) : 200; + + if (!Number.isInteger(iterations) || iterations <= 0) { + throw new Error('Iterations must be a positive integer'); + } + + const result = await runPythonRuntimeBenchmark({ iterations }); + + const output = { + measuredAt: new Date().toISOString(), + iterations, + coldStartMs: result.coldStartMs, + warmRunMs: result.warmRunMs, + repeatedMacro: result.repeatedMacro.stats, + }; + + console.log(JSON.stringify(output, null, 2)); +} + +void main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(`[python-runtime-benchmark] ${message}`); + process.exit(1); +}); diff --git a/src/renderer/python/PythonRuntimeManager.ts b/src/renderer/python/PythonRuntimeManager.ts new file mode 100644 index 0000000..ec0da64 --- /dev/null +++ b/src/renderer/python/PythonRuntimeManager.ts @@ -0,0 +1,258 @@ +import { createPythonRuntimeWorker } from './createPythonRuntimeWorker'; +import type { PythonWorkerMessage, PythonWorkerRequest } from './runtimeProtocol'; +import { parseMacroContextV1, parseMacroResultV1, type MacroContextV1, type MacroResultV1 } from './abiV1'; + +type WorkerFactory = () => Worker; + +interface InitializeDeferred { + resolve: () => void; + reject: (error: Error) => void; +} + +interface PendingRun { + kind: 'run' | 'macro-v1'; + stdout: string; + resolve: (value: PythonRunResult | PythonMacroV1Result) => void; + reject: (error: Error) => void; + timeoutId: ReturnType | null; +} + +export interface PythonRunResult { + result: string; + stdout: string; +} + +export interface PythonExecuteOptions { + timeoutMs?: number; +} + +export interface PythonMacroV1Result { + result: MacroResultV1; + stdout: string; +} + +export class PythonRuntimeManager { + private worker: Worker | null = null; + private initializingPromise: Promise | null = null; + private initializeDeferred: InitializeDeferred | null = null; + private ready = false; + private pendingRuns = new Map(); + private requestCounter = 0; + + constructor(private readonly workerFactory: WorkerFactory = createPythonRuntimeWorker) {} + + initialize(): Promise { + if (this.ready) { + return Promise.resolve(); + } + + if (this.initializingPromise) { + return this.initializingPromise; + } + + this.worker = this.workerFactory(); + this.ready = false; + + this.initializingPromise = new Promise((resolve, reject) => { + this.initializeDeferred = { resolve, reject }; + + if (!this.worker) { + this.initializeDeferred = null; + reject(new Error('Python runtime worker factory returned no worker')); + return; + } + + this.worker.onmessage = (event: MessageEvent) => { + this.handleWorkerMessage(event.data); + }; + this.worker.onerror = (event: ErrorEvent) => { + this.handleWorkerError(event.error instanceof Error ? event.error : new Error(event.message || 'Python runtime worker failed to initialize')); + }; + }); + + return this.initializingPromise; + } + + async execute(code: string, options?: PythonExecuteOptions): Promise { + await this.initialize(); + + if (!this.worker || !this.ready) { + throw new Error('Python runtime is not ready'); + } + + const requestId = this.nextRequestId(); + const timeoutMs = options?.timeoutMs ?? 5000; + + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.pendingRuns.delete(requestId); + this.resetRuntime(`Python script execution timed out after ${timeoutMs}ms`); + reject(new Error(`Python script execution timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + this.pendingRuns.set(requestId, { + kind: 'run', + stdout: '', + resolve: (value) => resolve(value as PythonRunResult), + reject, + timeoutId, + }); + + const message: PythonWorkerRequest = { + type: 'run', + requestId, + code, + }; + + this.worker!.postMessage(message); + }); + } + + async renderMacroV1(code: string, context: unknown, options?: PythonExecuteOptions): Promise { + const validatedContext = parseMacroContextV1(context); + await this.initialize(); + + if (!this.worker || !this.ready) { + throw new Error('Python runtime is not ready'); + } + + const requestId = this.nextRequestId(); + const timeoutMs = options?.timeoutMs ?? 5000; + + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.pendingRuns.delete(requestId); + this.resetRuntime(`Python script execution timed out after ${timeoutMs}ms`); + reject(new Error(`Python script execution timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + this.pendingRuns.set(requestId, { + kind: 'macro-v1', + stdout: '', + resolve: (value) => resolve(value as PythonMacroV1Result), + reject, + timeoutId, + }); + + const message: PythonWorkerRequest = { + type: 'renderMacroV1', + requestId, + code, + context: validatedContext, + }; + + this.worker!.postMessage(message); + }); + } + + isReady(): boolean { + return this.ready; + } + + dispose(): void { + this.resetRuntime(); + } + + private handleWorkerMessage(payload: PythonWorkerMessage): void { + if (payload.type === 'ready') { + this.ready = true; + this.initializingPromise = null; + this.initializeDeferred?.resolve(); + this.initializeDeferred = null; + return; + } + + if (payload.type === 'error') { + this.handleWorkerError(new Error(payload.error)); + return; + } + + const pendingRun = this.pendingRuns.get(payload.requestId); + if (!pendingRun) { + return; + } + + if (payload.type === 'stdout') { + pendingRun.stdout += payload.chunk; + return; + } + + this.pendingRuns.delete(payload.requestId); + if (pendingRun.timeoutId) { + clearTimeout(pendingRun.timeoutId); + } + + if (payload.type === 'runResult') { + if (pendingRun.kind !== 'run') { + pendingRun.reject(new Error('Invalid response type for pending macro request')); + return; + } + pendingRun.resolve({ result: payload.result, stdout: pendingRun.stdout }); + return; + } + + if (payload.type === 'macroResult') { + if (pendingRun.kind !== 'macro-v1') { + pendingRun.reject(new Error('Invalid response type for pending run request')); + return; + } + + try { + const validatedResult = parseMacroResultV1(payload.result); + pendingRun.resolve({ result: validatedResult, stdout: pendingRun.stdout }); + } catch (error) { + pendingRun.reject(error instanceof Error ? error : new Error(String(error))); + } + return; + } + + pendingRun.reject(new Error(payload.error)); + } + + private handleWorkerError(error: Error): void { + if (this.initializeDeferred) { + this.initializeDeferred.reject(error); + this.initializeDeferred = null; + } + + for (const run of this.pendingRuns.values()) { + if (run.timeoutId) { + clearTimeout(run.timeoutId); + } + run.reject(error); + } + + this.pendingRuns.clear(); + this.worker?.terminate(); + this.worker = null; + this.initializingPromise = null; + this.ready = false; + } + + private resetRuntime(timeoutErrorMessage?: string): void { + if (this.initializeDeferred) { + this.initializeDeferred.reject(new Error(timeoutErrorMessage ?? 'Python runtime reset')); + this.initializeDeferred = null; + } + + for (const run of this.pendingRuns.values()) { + if (run.timeoutId) { + clearTimeout(run.timeoutId); + } + if (timeoutErrorMessage) { + run.reject(new Error(timeoutErrorMessage)); + } + } + + this.pendingRuns.clear(); + this.worker?.terminate(); + this.worker = null; + this.initializingPromise = null; + this.ready = false; + } + + private nextRequestId(): string { + this.requestCounter += 1; + return `req-${this.requestCounter}`; + } +} diff --git a/src/renderer/python/abiV1.ts b/src/renderer/python/abiV1.ts new file mode 100644 index 0000000..f6cabdd --- /dev/null +++ b/src/renderer/python/abiV1.ts @@ -0,0 +1,52 @@ +import { z } from 'zod'; + +const jsonValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(jsonValueSchema), + z.record(z.string(), jsonValueSchema), + ]) +); + +export const macroContextV1Schema = z + .object({ + env: z + .object({ + isPreview: z.boolean(), + mainLanguage: z.string().min(1).optional(), + }) + .strict(), + params: z.record(z.string(), jsonValueSchema).optional(), + data: z.record(z.string(), jsonValueSchema).optional(), + }) + .strict(); + +export const macroResultV1Schema = z + .object({ + html: z.string(), + data: z.record(z.string(), jsonValueSchema).optional(), + warnings: z.array(z.string()).optional(), + }) + .strict(); + +export type MacroContextV1 = z.infer; +export type MacroResultV1 = z.infer; + +export function parseMacroContextV1(value: unknown): MacroContextV1 { + const parsed = macroContextV1Schema.safeParse(value); + if (!parsed.success) { + throw new Error(`Invalid macro context: ${parsed.error.issues[0]?.message ?? 'schema validation failed'}`); + } + return parsed.data; +} + +export function parseMacroResultV1(value: unknown): MacroResultV1 { + const parsed = macroResultV1Schema.safeParse(value); + if (!parsed.success) { + throw new Error(`Invalid macro result: ${parsed.error.issues[0]?.message ?? 'schema validation failed'}`); + } + return parsed.data; +} diff --git a/src/renderer/python/createPythonRuntimeWorker.ts b/src/renderer/python/createPythonRuntimeWorker.ts new file mode 100644 index 0000000..270f5ad --- /dev/null +++ b/src/renderer/python/createPythonRuntimeWorker.ts @@ -0,0 +1,5 @@ +import PythonRuntimeWorker from './pythonRuntime.worker?worker'; + +export function createPythonRuntimeWorker(): Worker { + return new PythonRuntimeWorker(); +} diff --git a/src/renderer/python/pythonRuntime.worker.ts b/src/renderer/python/pythonRuntime.worker.ts new file mode 100644 index 0000000..2083817 --- /dev/null +++ b/src/renderer/python/pythonRuntime.worker.ts @@ -0,0 +1,128 @@ +import { loadPyodide, type PyodideInterface } from 'pyodide'; +import type { PythonWorkerMessage, PythonWorkerRequest } from './runtimeProtocol'; +import { parseMacroContextV1, parseMacroResultV1 } from './abiV1'; + +let runtime: PyodideInterface | null = null; +let activeRequestId: string | null = null; + +function postRuntimeMessage(message: PythonWorkerMessage): void { + self.postMessage(message); +} + +function toResultString(result: unknown): string { + if (result === undefined || result === null) { + return ''; + } + if (typeof result === 'string') { + return result; + } + return String(result); +} + +async function runScript(request: PythonWorkerRequest): Promise { + if (request.type !== 'run') { + return; + } + + if (!runtime) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is not ready' }); + return; + } + + if (activeRequestId) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is busy' }); + return; + } + + activeRequestId = request.requestId; + + try { + const result = await runtime.runPythonAsync(request.code); + postRuntimeMessage({ + type: 'runResult', + requestId: request.requestId, + result: toResultString(result), + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: message }); + } finally { + activeRequestId = null; + } +} + +async function runMacroV1(request: PythonWorkerRequest): Promise { + if (request.type !== 'renderMacroV1') { + return; + } + + if (!runtime) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is not ready' }); + return; + } + + if (activeRequestId) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is busy' }); + return; + } + + activeRequestId = request.requestId; + + try { + const validatedContext = parseMacroContextV1(request.context); + runtime.globals.set('__bds_context_v1', validatedContext); + + await runtime.runPythonAsync(request.code); + + const rawJsonResult = await runtime.runPythonAsync(` +import json +json.dumps(render(__bds_context_v1)) +`); + + const parsedResult = parseMacroResultV1(JSON.parse(toResultString(rawJsonResult))); + postRuntimeMessage({ + type: 'macroResult', + requestId: request.requestId, + result: parsedResult, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: message }); + } finally { + activeRequestId = null; + } +} + +async function bootstrapRuntime(): Promise { + try { + runtime = await loadPyodide({ + stdout: (chunk) => { + if (!activeRequestId) { + return; + } + postRuntimeMessage({ type: 'stdout', requestId: activeRequestId, chunk }); + }, + }); + if (!runtime) { + throw new Error('Pyodide initialization returned no runtime'); + } + postRuntimeMessage({ type: 'ready' }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + postRuntimeMessage({ type: 'error', error: message }); + } +} + +self.onmessage = (event: MessageEvent) => { + const request = event.data; + if (request.type === 'run') { + void runScript(request); + return; + } + + if (request.type === 'renderMacroV1') { + void runMacroV1(request); + } +}; + +void bootstrapRuntime(); diff --git a/src/renderer/python/pythonRuntimeBenchmark.ts b/src/renderer/python/pythonRuntimeBenchmark.ts new file mode 100644 index 0000000..d63a5a0 --- /dev/null +++ b/src/renderer/python/pythonRuntimeBenchmark.ts @@ -0,0 +1,117 @@ +import { loadPyodide } from 'pyodide'; + +export interface BenchmarkStats { + count: number; + minMs: number; + maxMs: number; + meanMs: number; + p50Ms: number; + p95Ms: number; +} + +export interface BenchmarkResult { + coldStartMs: number; + warmRunMs: number; + repeatedMacro: { + samplesMs: number[]; + stats: BenchmarkStats; + }; +} + +interface PythonRuntimeLike { + runPythonAsync(code: string): Promise; +} + +interface BenchmarkOptions { + iterations?: number; + loadRuntime?: () => Promise; + now?: () => number; +} + +function percentile(values: number[], p: number): number { + if (values.length === 0) { + return 0; + } + + if (values.length === 1) { + return values[0]; + } + + const sorted = [...values].sort((a, b) => a - b); + const index = (sorted.length - 1) * p; + const lower = Math.floor(index); + const upper = Math.ceil(index); + + if (lower === upper) { + return sorted[lower]; + } + + const weight = index - lower; + return sorted[lower] * (1 - weight) + sorted[upper] * weight; +} + +function round2(value: number): number { + return Math.round(value * 100) / 100; +} + +export function summarizeDurations(durationsMs: number[]): BenchmarkStats { + if (durationsMs.length === 0) { + return { + count: 0, + minMs: 0, + maxMs: 0, + meanMs: 0, + p50Ms: 0, + p95Ms: 0, + }; + } + + const minMs = Math.min(...durationsMs); + const maxMs = Math.max(...durationsMs); + const meanMs = durationsMs.reduce((sum, value) => sum + value, 0) / durationsMs.length; + + return { + count: durationsMs.length, + minMs: round2(minMs), + maxMs: round2(maxMs), + meanMs: round2(meanMs), + p50Ms: round2(percentile(durationsMs, 0.5)), + p95Ms: round2(percentile(durationsMs, 0.95)), + }; +} + +export async function runPythonRuntimeBenchmark(options: BenchmarkOptions = {}): Promise { + const iterations = options.iterations ?? 200; + const loadRuntime = options.loadRuntime ?? (async () => loadPyodide()); + const now = options.now ?? (() => performance.now()); + + const coldStartStart = now(); + const runtime = await loadRuntime(); + const coldStartMs = now() - coldStartStart; + + const warmStart = now(); + await runtime.runPythonAsync('1 + 1'); + const warmRunMs = now() - warmStart; + + await runtime.runPythonAsync(` +def render(context): + title = context.get("title", "") + return {"html": f"

{title}

"} +`); + + const samplesMs: number[] = []; + for (let index = 0; index < iterations; index += 1) { + const started = now(); + await runtime.runPythonAsync('render({"title": "Benchmark"})["html"]'); + samplesMs.push(now() - started); + } + + return { + coldStartMs: round2(coldStartMs), + warmRunMs: round2(warmRunMs), + repeatedMacro: { + samplesMs: samplesMs.map(round2), + stats: summarizeDurations(samplesMs), + }, + }; +} diff --git a/src/renderer/python/runtimeProtocol.ts b/src/renderer/python/runtimeProtocol.ts new file mode 100644 index 0000000..3ad18f5 --- /dev/null +++ b/src/renderer/python/runtimeProtocol.ts @@ -0,0 +1,22 @@ +import type { MacroContextV1, MacroResultV1 } from './abiV1'; + +export type PythonWorkerRequest = + | { + type: 'run'; + requestId: string; + code: string; + } + | { + type: 'renderMacroV1'; + requestId: string; + code: string; + context: MacroContextV1; + }; + +export type PythonWorkerMessage = + | { type: 'ready' } + | { type: 'error'; error: string } + | { type: 'stdout'; requestId: string; chunk: string } + | { type: 'runResult'; requestId: string; result: string } + | { type: 'macroResult'; requestId: string; result: MacroResultV1 } + | { type: 'runError'; requestId: string; error: string }; diff --git a/tests/renderer/python/PythonRuntimeManager.test.ts b/tests/renderer/python/PythonRuntimeManager.test.ts new file mode 100644 index 0000000..c331c57 --- /dev/null +++ b/tests/renderer/python/PythonRuntimeManager.test.ts @@ -0,0 +1,194 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { PythonRuntimeManager } from '../../../src/renderer/python/PythonRuntimeManager'; + +class MockWorker { + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + terminated = false; + postedMessages: unknown[] = []; + + postMessage(message: unknown): void { + this.postedMessages.push(message); + } + + terminate(): void { + this.terminated = true; + } + + emitMessage(data: unknown): void { + this.onmessage?.({ data } as MessageEvent); + } + + emitError(error: Error): void { + this.onerror?.({ error } as ErrorEvent); + } +} + +describe('PythonRuntimeManager', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('initializes worker once and resolves on ready signal', async () => { + const createdWorkers: MockWorker[] = []; + const manager = new PythonRuntimeManager(() => { + const worker = new MockWorker(); + createdWorkers.push(worker); + return worker as unknown as Worker; + }); + + const initA = manager.initialize(); + const initB = manager.initialize(); + + expect(initA).toBe(initB); + expect(createdWorkers).toHaveLength(1); + + createdWorkers[0].emitMessage({ type: 'ready' }); + await expect(initA).resolves.toBeUndefined(); + expect(manager.isReady()).toBe(true); + }); + + it('rejects when worker emits an error before ready', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitError(new Error('bootstrap failed')); + + await expect(initPromise).rejects.toThrow('bootstrap failed'); + expect(manager.isReady()).toBe(false); + }); + + it('executes code and returns stdout and result', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.execute('print("hello")\n1 + 1'); + await Promise.resolve(); + const request = worker.postedMessages[0] as { type: string; requestId: string; code: string }; + + worker.emitMessage({ type: 'stdout', requestId: request.requestId, chunk: 'hello\n' }); + worker.emitMessage({ type: 'runResult', requestId: request.requestId, result: '2' }); + + await expect(runPromise).resolves.toEqual({ result: '2', stdout: 'hello\n' }); + }); + + it('rejects when runtime returns run error', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.execute('raise RuntimeError("boom")'); + await Promise.resolve(); + const request = worker.postedMessages[0] as { requestId: string }; + worker.emitMessage({ type: 'runError', requestId: request.requestId, error: 'boom' }); + + await expect(runPromise).rejects.toThrow('boom'); + }); + + it('terminates timed out worker and recovers with a new worker', async () => { + const workers: MockWorker[] = []; + const manager = new PythonRuntimeManager(() => { + const worker = new MockWorker(); + workers.push(worker); + return worker as unknown as Worker; + }); + + const firstInit = manager.initialize(); + workers[0].emitMessage({ type: 'ready' }); + await firstInit; + + const timedOutRun = manager.execute('while True: pass', { timeoutMs: 100 }); + await Promise.resolve(); + vi.advanceTimersByTime(101); + await expect(timedOutRun).rejects.toThrow('timed out'); + expect(workers[0].terminated).toBe(true); + expect(manager.isReady()).toBe(false); + + const secondInit = manager.initialize(); + workers[1].emitMessage({ type: 'ready' }); + await secondInit; + + const secondRun = manager.execute('40 + 2'); + await Promise.resolve(); + const request = workers[1].postedMessages[0] as { requestId: string }; + workers[1].emitMessage({ type: 'runResult', requestId: request.requestId, result: '42' }); + + await expect(secondRun).resolves.toEqual({ result: '42', stdout: '' }); + }); + + it('rejects macro execution when ABI context is invalid on caller side', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.renderMacroV1('def render(context):\n return {"html": "

ok

"}', { + env: { + isPreview: 'yes', + }, + }); + + await expect(runPromise).rejects.toThrow('Invalid macro context'); + expect(worker.postedMessages).toHaveLength(0); + }); + + it('returns validated macro result and stdout', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.renderMacroV1('def render(context):\n return {"html": "

ok

"}', { + env: { + isPreview: true, + }, + params: { + title: 'Hello', + }, + }); + + await Promise.resolve(); + const request = worker.postedMessages[0] as { requestId: string }; + worker.emitMessage({ type: 'stdout', requestId: request.requestId, chunk: 'rendering\n' }); + worker.emitMessage({ type: 'macroResult', requestId: request.requestId, result: { html: '

ok

' } }); + + await expect(runPromise).resolves.toEqual({ result: { html: '

ok

' }, stdout: 'rendering\n' }); + }); + + it('rejects macro execution when worker result violates ABI schema', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.renderMacroV1('def render(context):\n return {"html": "

ok

"}', { + env: { + isPreview: true, + }, + }); + + await Promise.resolve(); + const request = worker.postedMessages[0] as { requestId: string }; + worker.emitMessage({ type: 'macroResult', requestId: request.requestId, result: { html: 42 } }); + + await expect(runPromise).rejects.toThrow('Invalid macro result'); + }); +}); diff --git a/tests/renderer/python/pythonRuntimeBenchmark.test.ts b/tests/renderer/python/pythonRuntimeBenchmark.test.ts new file mode 100644 index 0000000..2870480 --- /dev/null +++ b/tests/renderer/python/pythonRuntimeBenchmark.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest'; +import { summarizeDurations, runPythonRuntimeBenchmark } from '../../../src/renderer/python/pythonRuntimeBenchmark'; + +describe('pythonRuntimeBenchmark', () => { + it('computes p50 and p95 summary metrics', () => { + const summary = summarizeDurations([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + + expect(summary.count).toBe(10); + expect(summary.minMs).toBe(1); + expect(summary.maxMs).toBe(10); + expect(summary.meanMs).toBe(5.5); + expect(summary.p50Ms).toBe(5.5); + expect(summary.p95Ms).toBe(9.55); + }); + + it('runs benchmark phases and returns measured durations', async () => { + const calls: string[] = []; + const runtime = { + async runPythonAsync(code: string): Promise { + calls.push(code); + return 'ok'; + }, + }; + + const timestamps = [0, 100, 100, 110, 110, 111, 111, 113, 113, 116, 116, 120]; + let index = 0; + + const result = await runPythonRuntimeBenchmark({ + iterations: 4, + loadRuntime: async () => runtime, + now: () => { + const value = timestamps[index]; + index += 1; + return value; + }, + }); + + expect(result.coldStartMs).toBe(100); + expect(result.warmRunMs).toBe(10); + expect(result.repeatedMacro.samplesMs).toEqual([1, 2, 3, 4]); + expect(result.repeatedMacro.stats.p50Ms).toBe(2.5); + expect(result.repeatedMacro.stats.p95Ms).toBe(3.85); + expect(calls).toHaveLength(6); + }); +});