From ce050f98c3d7c017d12b1c10cf189fbb4b593f40 Mon Sep 17 00:00:00 2001 From: hugo Date: Sun, 22 Feb 2026 21:29:16 +0100 Subject: [PATCH 01/16] feat: python scripting phase 0 --- PYTHON_SCRIPTING.md | 35 ++- package-lock.json | 20 ++ package.json | 2 + scripts/python-runtime-benchmark.ts | 28 ++ src/renderer/python/PythonRuntimeManager.ts | 258 ++++++++++++++++++ src/renderer/python/abiV1.ts | 52 ++++ .../python/createPythonRuntimeWorker.ts | 5 + src/renderer/python/pythonRuntime.worker.ts | 128 +++++++++ src/renderer/python/pythonRuntimeBenchmark.ts | 117 ++++++++ src/renderer/python/runtimeProtocol.ts | 22 ++ .../python/PythonRuntimeManager.test.ts | 194 +++++++++++++ .../python/pythonRuntimeBenchmark.test.ts | 45 +++ 12 files changed, 895 insertions(+), 11 deletions(-) create mode 100644 scripts/python-runtime-benchmark.ts create mode 100644 src/renderer/python/PythonRuntimeManager.ts create mode 100644 src/renderer/python/abiV1.ts create mode 100644 src/renderer/python/createPythonRuntimeWorker.ts create mode 100644 src/renderer/python/pythonRuntime.worker.ts create mode 100644 src/renderer/python/pythonRuntimeBenchmark.ts create mode 100644 src/renderer/python/runtimeProtocol.ts create mode 100644 tests/renderer/python/PythonRuntimeManager.test.ts create mode 100644 tests/renderer/python/pythonRuntimeBenchmark.test.ts diff --git a/PYTHON_SCRIPTING.md b/PYTHON_SCRIPTING.md index ce741c1..8d79890 100644 --- a/PYTHON_SCRIPTING.md +++ b/PYTHON_SCRIPTING.md @@ -86,15 +86,22 @@ Contract rules: Objective: prove runtime viability before product surface growth. Deliverables: -- [ ] Add `pyodide` dependency and worker boot sequence. -- [ ] Run a sample script end-to-end (`run_script`, timeout, captured stdout). -- [ ] Benchmark baseline cold start + warm run + repeated macro calls. -- [ ] Define initial macro ABI (`render(context) -> result`) and schema docs. +- [x] Add `pyodide` dependency and worker boot sequence. +- [x] Run a sample script end-to-end (`run_script`, timeout, captured stdout). +- [x] Benchmark baseline cold start + warm run + repeated macro calls. +- [x] Define initial macro ABI (`render(context) -> result`) and schema docs. Exit criteria: -- Warm script execution is stable. -- Timeout recovery works. -- Measured baseline captured in repo docs. +- Warm script execution is stable. ✅ +- Timeout recovery works. ✅ +- Measured baseline captured in repo docs. ✅ + +Baseline benchmark (22 Feb 2026, local macOS run): +- Command: `npm run bench:python-runtime -- 200` +- Cold start: `701.11 ms` +- Warm run: `5.74 ms` +- Repeated macro (200 calls): `p50 0.17 ms`, `p95 0.29 ms`, `mean 0.19 ms` +- Notes: one-machine baseline only; use trend comparisons for regressions. ## Phase 1 — MVP (Minimal but Usable) @@ -447,9 +454,15 @@ PR-14+: Optional advanced capabilities ## 10. Current Status -Status: Revised staged plan (MVP-first, full-scope preserved). +Status: Phase 0 in progress (MVP-first, full-scope preserved). + +Progress update (22 Feb 2026): +- [x] PR-00 complete: Pyodide dependency + renderer worker bootstrap + ready signal. +- [x] PR-01 complete: worker run/stdout/error protocol + timeout watchdog + runtime recovery. +- [x] PR-02 complete: ABI v1 shared types/schemas + caller/worker validation. +- [x] Phase 0 benchmark harness + baseline capture complete. Recommended next action: -1. Approve Phase 0 scope and benchmarks. -2. Implement spike and record numbers. -3. Lock ABI before building full UI and migration layers. +1. Start Phase 1 PR-03: script persistence model (`scripts/*.py` + index metadata). +2. Add round-trip tests for create/update/delete between filesystem and DB. +3. Keep benchmark command in CI/manual perf checks for regressions. diff --git a/package-lock.json b/package-lock.json index 7d55db5..39c9c9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "liquidjs": "^10.24.0", "marked-react": "^3.0.2", "monaco-editor": "^0.55.1", + "pyodide": "^0.29.3", "react": "^19.2.4", "react-arborist": "^3.4.3", "react-dom": "^19.2.4", @@ -5265,6 +5266,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -12753,6 +12760,19 @@ "node": ">=6" } }, + "node_modules/pyodide": { + "version": "0.29.3", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.3.tgz", + "integrity": "sha512-22UBuhOJawj7vKUnS7/F3xK+515LJdjiMAHoCfuS6/PbHiOrSQVnYwDe+2sbVwiOZ3sMMexdXICew6NqOMQGgA==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", diff --git a/package.json b/package.json index 0e628d7..21a7eac 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:ui": "vitest --ui", + "bench:python-runtime": "node ./node_modules/tsx/dist/cli.mjs scripts/python-runtime-benchmark.ts", "lint": "eslint \"src/renderer/**/*.{ts,tsx}\" --max-warnings 0", "lint:i18n": "eslint \"src/renderer/**/*.{ts,tsx}\" --max-warnings 0", "db:generate": "node ./node_modules/drizzle-kit/bin.cjs generate", @@ -95,6 +96,7 @@ "liquidjs": "^10.24.0", "marked-react": "^3.0.2", "monaco-editor": "^0.55.1", + "pyodide": "^0.29.3", "react": "^19.2.4", "react-arborist": "^3.4.3", "react-dom": "^19.2.4", diff --git a/scripts/python-runtime-benchmark.ts b/scripts/python-runtime-benchmark.ts new file mode 100644 index 0000000..3c16882 --- /dev/null +++ b/scripts/python-runtime-benchmark.ts @@ -0,0 +1,28 @@ +import { runPythonRuntimeBenchmark } from '../src/renderer/python/pythonRuntimeBenchmark'; + +async function main(): Promise { + const iterationsArg = process.argv[2]; + const iterations = iterationsArg ? Number(iterationsArg) : 200; + + if (!Number.isInteger(iterations) || iterations <= 0) { + throw new Error('Iterations must be a positive integer'); + } + + const result = await runPythonRuntimeBenchmark({ iterations }); + + const output = { + measuredAt: new Date().toISOString(), + iterations, + coldStartMs: result.coldStartMs, + warmRunMs: result.warmRunMs, + repeatedMacro: result.repeatedMacro.stats, + }; + + console.log(JSON.stringify(output, null, 2)); +} + +void main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(`[python-runtime-benchmark] ${message}`); + process.exit(1); +}); diff --git a/src/renderer/python/PythonRuntimeManager.ts b/src/renderer/python/PythonRuntimeManager.ts new file mode 100644 index 0000000..ec0da64 --- /dev/null +++ b/src/renderer/python/PythonRuntimeManager.ts @@ -0,0 +1,258 @@ +import { createPythonRuntimeWorker } from './createPythonRuntimeWorker'; +import type { PythonWorkerMessage, PythonWorkerRequest } from './runtimeProtocol'; +import { parseMacroContextV1, parseMacroResultV1, type MacroContextV1, type MacroResultV1 } from './abiV1'; + +type WorkerFactory = () => Worker; + +interface InitializeDeferred { + resolve: () => void; + reject: (error: Error) => void; +} + +interface PendingRun { + kind: 'run' | 'macro-v1'; + stdout: string; + resolve: (value: PythonRunResult | PythonMacroV1Result) => void; + reject: (error: Error) => void; + timeoutId: ReturnType | null; +} + +export interface PythonRunResult { + result: string; + stdout: string; +} + +export interface PythonExecuteOptions { + timeoutMs?: number; +} + +export interface PythonMacroV1Result { + result: MacroResultV1; + stdout: string; +} + +export class PythonRuntimeManager { + private worker: Worker | null = null; + private initializingPromise: Promise | null = null; + private initializeDeferred: InitializeDeferred | null = null; + private ready = false; + private pendingRuns = new Map(); + private requestCounter = 0; + + constructor(private readonly workerFactory: WorkerFactory = createPythonRuntimeWorker) {} + + initialize(): Promise { + if (this.ready) { + return Promise.resolve(); + } + + if (this.initializingPromise) { + return this.initializingPromise; + } + + this.worker = this.workerFactory(); + this.ready = false; + + this.initializingPromise = new Promise((resolve, reject) => { + this.initializeDeferred = { resolve, reject }; + + if (!this.worker) { + this.initializeDeferred = null; + reject(new Error('Python runtime worker factory returned no worker')); + return; + } + + this.worker.onmessage = (event: MessageEvent) => { + this.handleWorkerMessage(event.data); + }; + this.worker.onerror = (event: ErrorEvent) => { + this.handleWorkerError(event.error instanceof Error ? event.error : new Error(event.message || 'Python runtime worker failed to initialize')); + }; + }); + + return this.initializingPromise; + } + + async execute(code: string, options?: PythonExecuteOptions): Promise { + await this.initialize(); + + if (!this.worker || !this.ready) { + throw new Error('Python runtime is not ready'); + } + + const requestId = this.nextRequestId(); + const timeoutMs = options?.timeoutMs ?? 5000; + + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.pendingRuns.delete(requestId); + this.resetRuntime(`Python script execution timed out after ${timeoutMs}ms`); + reject(new Error(`Python script execution timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + this.pendingRuns.set(requestId, { + kind: 'run', + stdout: '', + resolve: (value) => resolve(value as PythonRunResult), + reject, + timeoutId, + }); + + const message: PythonWorkerRequest = { + type: 'run', + requestId, + code, + }; + + this.worker!.postMessage(message); + }); + } + + async renderMacroV1(code: string, context: unknown, options?: PythonExecuteOptions): Promise { + const validatedContext = parseMacroContextV1(context); + await this.initialize(); + + if (!this.worker || !this.ready) { + throw new Error('Python runtime is not ready'); + } + + const requestId = this.nextRequestId(); + const timeoutMs = options?.timeoutMs ?? 5000; + + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.pendingRuns.delete(requestId); + this.resetRuntime(`Python script execution timed out after ${timeoutMs}ms`); + reject(new Error(`Python script execution timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + this.pendingRuns.set(requestId, { + kind: 'macro-v1', + stdout: '', + resolve: (value) => resolve(value as PythonMacroV1Result), + reject, + timeoutId, + }); + + const message: PythonWorkerRequest = { + type: 'renderMacroV1', + requestId, + code, + context: validatedContext, + }; + + this.worker!.postMessage(message); + }); + } + + isReady(): boolean { + return this.ready; + } + + dispose(): void { + this.resetRuntime(); + } + + private handleWorkerMessage(payload: PythonWorkerMessage): void { + if (payload.type === 'ready') { + this.ready = true; + this.initializingPromise = null; + this.initializeDeferred?.resolve(); + this.initializeDeferred = null; + return; + } + + if (payload.type === 'error') { + this.handleWorkerError(new Error(payload.error)); + return; + } + + const pendingRun = this.pendingRuns.get(payload.requestId); + if (!pendingRun) { + return; + } + + if (payload.type === 'stdout') { + pendingRun.stdout += payload.chunk; + return; + } + + this.pendingRuns.delete(payload.requestId); + if (pendingRun.timeoutId) { + clearTimeout(pendingRun.timeoutId); + } + + if (payload.type === 'runResult') { + if (pendingRun.kind !== 'run') { + pendingRun.reject(new Error('Invalid response type for pending macro request')); + return; + } + pendingRun.resolve({ result: payload.result, stdout: pendingRun.stdout }); + return; + } + + if (payload.type === 'macroResult') { + if (pendingRun.kind !== 'macro-v1') { + pendingRun.reject(new Error('Invalid response type for pending run request')); + return; + } + + try { + const validatedResult = parseMacroResultV1(payload.result); + pendingRun.resolve({ result: validatedResult, stdout: pendingRun.stdout }); + } catch (error) { + pendingRun.reject(error instanceof Error ? error : new Error(String(error))); + } + return; + } + + pendingRun.reject(new Error(payload.error)); + } + + private handleWorkerError(error: Error): void { + if (this.initializeDeferred) { + this.initializeDeferred.reject(error); + this.initializeDeferred = null; + } + + for (const run of this.pendingRuns.values()) { + if (run.timeoutId) { + clearTimeout(run.timeoutId); + } + run.reject(error); + } + + this.pendingRuns.clear(); + this.worker?.terminate(); + this.worker = null; + this.initializingPromise = null; + this.ready = false; + } + + private resetRuntime(timeoutErrorMessage?: string): void { + if (this.initializeDeferred) { + this.initializeDeferred.reject(new Error(timeoutErrorMessage ?? 'Python runtime reset')); + this.initializeDeferred = null; + } + + for (const run of this.pendingRuns.values()) { + if (run.timeoutId) { + clearTimeout(run.timeoutId); + } + if (timeoutErrorMessage) { + run.reject(new Error(timeoutErrorMessage)); + } + } + + this.pendingRuns.clear(); + this.worker?.terminate(); + this.worker = null; + this.initializingPromise = null; + this.ready = false; + } + + private nextRequestId(): string { + this.requestCounter += 1; + return `req-${this.requestCounter}`; + } +} diff --git a/src/renderer/python/abiV1.ts b/src/renderer/python/abiV1.ts new file mode 100644 index 0000000..f6cabdd --- /dev/null +++ b/src/renderer/python/abiV1.ts @@ -0,0 +1,52 @@ +import { z } from 'zod'; + +const jsonValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(jsonValueSchema), + z.record(z.string(), jsonValueSchema), + ]) +); + +export const macroContextV1Schema = z + .object({ + env: z + .object({ + isPreview: z.boolean(), + mainLanguage: z.string().min(1).optional(), + }) + .strict(), + params: z.record(z.string(), jsonValueSchema).optional(), + data: z.record(z.string(), jsonValueSchema).optional(), + }) + .strict(); + +export const macroResultV1Schema = z + .object({ + html: z.string(), + data: z.record(z.string(), jsonValueSchema).optional(), + warnings: z.array(z.string()).optional(), + }) + .strict(); + +export type MacroContextV1 = z.infer; +export type MacroResultV1 = z.infer; + +export function parseMacroContextV1(value: unknown): MacroContextV1 { + const parsed = macroContextV1Schema.safeParse(value); + if (!parsed.success) { + throw new Error(`Invalid macro context: ${parsed.error.issues[0]?.message ?? 'schema validation failed'}`); + } + return parsed.data; +} + +export function parseMacroResultV1(value: unknown): MacroResultV1 { + const parsed = macroResultV1Schema.safeParse(value); + if (!parsed.success) { + throw new Error(`Invalid macro result: ${parsed.error.issues[0]?.message ?? 'schema validation failed'}`); + } + return parsed.data; +} diff --git a/src/renderer/python/createPythonRuntimeWorker.ts b/src/renderer/python/createPythonRuntimeWorker.ts new file mode 100644 index 0000000..270f5ad --- /dev/null +++ b/src/renderer/python/createPythonRuntimeWorker.ts @@ -0,0 +1,5 @@ +import PythonRuntimeWorker from './pythonRuntime.worker?worker'; + +export function createPythonRuntimeWorker(): Worker { + return new PythonRuntimeWorker(); +} diff --git a/src/renderer/python/pythonRuntime.worker.ts b/src/renderer/python/pythonRuntime.worker.ts new file mode 100644 index 0000000..2083817 --- /dev/null +++ b/src/renderer/python/pythonRuntime.worker.ts @@ -0,0 +1,128 @@ +import { loadPyodide, type PyodideInterface } from 'pyodide'; +import type { PythonWorkerMessage, PythonWorkerRequest } from './runtimeProtocol'; +import { parseMacroContextV1, parseMacroResultV1 } from './abiV1'; + +let runtime: PyodideInterface | null = null; +let activeRequestId: string | null = null; + +function postRuntimeMessage(message: PythonWorkerMessage): void { + self.postMessage(message); +} + +function toResultString(result: unknown): string { + if (result === undefined || result === null) { + return ''; + } + if (typeof result === 'string') { + return result; + } + return String(result); +} + +async function runScript(request: PythonWorkerRequest): Promise { + if (request.type !== 'run') { + return; + } + + if (!runtime) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is not ready' }); + return; + } + + if (activeRequestId) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is busy' }); + return; + } + + activeRequestId = request.requestId; + + try { + const result = await runtime.runPythonAsync(request.code); + postRuntimeMessage({ + type: 'runResult', + requestId: request.requestId, + result: toResultString(result), + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: message }); + } finally { + activeRequestId = null; + } +} + +async function runMacroV1(request: PythonWorkerRequest): Promise { + if (request.type !== 'renderMacroV1') { + return; + } + + if (!runtime) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is not ready' }); + return; + } + + if (activeRequestId) { + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: 'Python runtime is busy' }); + return; + } + + activeRequestId = request.requestId; + + try { + const validatedContext = parseMacroContextV1(request.context); + runtime.globals.set('__bds_context_v1', validatedContext); + + await runtime.runPythonAsync(request.code); + + const rawJsonResult = await runtime.runPythonAsync(` +import json +json.dumps(render(__bds_context_v1)) +`); + + const parsedResult = parseMacroResultV1(JSON.parse(toResultString(rawJsonResult))); + postRuntimeMessage({ + type: 'macroResult', + requestId: request.requestId, + result: parsedResult, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + postRuntimeMessage({ type: 'runError', requestId: request.requestId, error: message }); + } finally { + activeRequestId = null; + } +} + +async function bootstrapRuntime(): Promise { + try { + runtime = await loadPyodide({ + stdout: (chunk) => { + if (!activeRequestId) { + return; + } + postRuntimeMessage({ type: 'stdout', requestId: activeRequestId, chunk }); + }, + }); + if (!runtime) { + throw new Error('Pyodide initialization returned no runtime'); + } + postRuntimeMessage({ type: 'ready' }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + postRuntimeMessage({ type: 'error', error: message }); + } +} + +self.onmessage = (event: MessageEvent) => { + const request = event.data; + if (request.type === 'run') { + void runScript(request); + return; + } + + if (request.type === 'renderMacroV1') { + void runMacroV1(request); + } +}; + +void bootstrapRuntime(); diff --git a/src/renderer/python/pythonRuntimeBenchmark.ts b/src/renderer/python/pythonRuntimeBenchmark.ts new file mode 100644 index 0000000..d63a5a0 --- /dev/null +++ b/src/renderer/python/pythonRuntimeBenchmark.ts @@ -0,0 +1,117 @@ +import { loadPyodide } from 'pyodide'; + +export interface BenchmarkStats { + count: number; + minMs: number; + maxMs: number; + meanMs: number; + p50Ms: number; + p95Ms: number; +} + +export interface BenchmarkResult { + coldStartMs: number; + warmRunMs: number; + repeatedMacro: { + samplesMs: number[]; + stats: BenchmarkStats; + }; +} + +interface PythonRuntimeLike { + runPythonAsync(code: string): Promise; +} + +interface BenchmarkOptions { + iterations?: number; + loadRuntime?: () => Promise; + now?: () => number; +} + +function percentile(values: number[], p: number): number { + if (values.length === 0) { + return 0; + } + + if (values.length === 1) { + return values[0]; + } + + const sorted = [...values].sort((a, b) => a - b); + const index = (sorted.length - 1) * p; + const lower = Math.floor(index); + const upper = Math.ceil(index); + + if (lower === upper) { + return sorted[lower]; + } + + const weight = index - lower; + return sorted[lower] * (1 - weight) + sorted[upper] * weight; +} + +function round2(value: number): number { + return Math.round(value * 100) / 100; +} + +export function summarizeDurations(durationsMs: number[]): BenchmarkStats { + if (durationsMs.length === 0) { + return { + count: 0, + minMs: 0, + maxMs: 0, + meanMs: 0, + p50Ms: 0, + p95Ms: 0, + }; + } + + const minMs = Math.min(...durationsMs); + const maxMs = Math.max(...durationsMs); + const meanMs = durationsMs.reduce((sum, value) => sum + value, 0) / durationsMs.length; + + return { + count: durationsMs.length, + minMs: round2(minMs), + maxMs: round2(maxMs), + meanMs: round2(meanMs), + p50Ms: round2(percentile(durationsMs, 0.5)), + p95Ms: round2(percentile(durationsMs, 0.95)), + }; +} + +export async function runPythonRuntimeBenchmark(options: BenchmarkOptions = {}): Promise { + const iterations = options.iterations ?? 200; + const loadRuntime = options.loadRuntime ?? (async () => loadPyodide()); + const now = options.now ?? (() => performance.now()); + + const coldStartStart = now(); + const runtime = await loadRuntime(); + const coldStartMs = now() - coldStartStart; + + const warmStart = now(); + await runtime.runPythonAsync('1 + 1'); + const warmRunMs = now() - warmStart; + + await runtime.runPythonAsync(` +def render(context): + title = context.get("title", "") + return {"html": f"

{title}

"} +`); + + const samplesMs: number[] = []; + for (let index = 0; index < iterations; index += 1) { + const started = now(); + await runtime.runPythonAsync('render({"title": "Benchmark"})["html"]'); + samplesMs.push(now() - started); + } + + return { + coldStartMs: round2(coldStartMs), + warmRunMs: round2(warmRunMs), + repeatedMacro: { + samplesMs: samplesMs.map(round2), + stats: summarizeDurations(samplesMs), + }, + }; +} diff --git a/src/renderer/python/runtimeProtocol.ts b/src/renderer/python/runtimeProtocol.ts new file mode 100644 index 0000000..3ad18f5 --- /dev/null +++ b/src/renderer/python/runtimeProtocol.ts @@ -0,0 +1,22 @@ +import type { MacroContextV1, MacroResultV1 } from './abiV1'; + +export type PythonWorkerRequest = + | { + type: 'run'; + requestId: string; + code: string; + } + | { + type: 'renderMacroV1'; + requestId: string; + code: string; + context: MacroContextV1; + }; + +export type PythonWorkerMessage = + | { type: 'ready' } + | { type: 'error'; error: string } + | { type: 'stdout'; requestId: string; chunk: string } + | { type: 'runResult'; requestId: string; result: string } + | { type: 'macroResult'; requestId: string; result: MacroResultV1 } + | { type: 'runError'; requestId: string; error: string }; diff --git a/tests/renderer/python/PythonRuntimeManager.test.ts b/tests/renderer/python/PythonRuntimeManager.test.ts new file mode 100644 index 0000000..c331c57 --- /dev/null +++ b/tests/renderer/python/PythonRuntimeManager.test.ts @@ -0,0 +1,194 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { PythonRuntimeManager } from '../../../src/renderer/python/PythonRuntimeManager'; + +class MockWorker { + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + terminated = false; + postedMessages: unknown[] = []; + + postMessage(message: unknown): void { + this.postedMessages.push(message); + } + + terminate(): void { + this.terminated = true; + } + + emitMessage(data: unknown): void { + this.onmessage?.({ data } as MessageEvent); + } + + emitError(error: Error): void { + this.onerror?.({ error } as ErrorEvent); + } +} + +describe('PythonRuntimeManager', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('initializes worker once and resolves on ready signal', async () => { + const createdWorkers: MockWorker[] = []; + const manager = new PythonRuntimeManager(() => { + const worker = new MockWorker(); + createdWorkers.push(worker); + return worker as unknown as Worker; + }); + + const initA = manager.initialize(); + const initB = manager.initialize(); + + expect(initA).toBe(initB); + expect(createdWorkers).toHaveLength(1); + + createdWorkers[0].emitMessage({ type: 'ready' }); + await expect(initA).resolves.toBeUndefined(); + expect(manager.isReady()).toBe(true); + }); + + it('rejects when worker emits an error before ready', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitError(new Error('bootstrap failed')); + + await expect(initPromise).rejects.toThrow('bootstrap failed'); + expect(manager.isReady()).toBe(false); + }); + + it('executes code and returns stdout and result', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.execute('print("hello")\n1 + 1'); + await Promise.resolve(); + const request = worker.postedMessages[0] as { type: string; requestId: string; code: string }; + + worker.emitMessage({ type: 'stdout', requestId: request.requestId, chunk: 'hello\n' }); + worker.emitMessage({ type: 'runResult', requestId: request.requestId, result: '2' }); + + await expect(runPromise).resolves.toEqual({ result: '2', stdout: 'hello\n' }); + }); + + it('rejects when runtime returns run error', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.execute('raise RuntimeError("boom")'); + await Promise.resolve(); + const request = worker.postedMessages[0] as { requestId: string }; + worker.emitMessage({ type: 'runError', requestId: request.requestId, error: 'boom' }); + + await expect(runPromise).rejects.toThrow('boom'); + }); + + it('terminates timed out worker and recovers with a new worker', async () => { + const workers: MockWorker[] = []; + const manager = new PythonRuntimeManager(() => { + const worker = new MockWorker(); + workers.push(worker); + return worker as unknown as Worker; + }); + + const firstInit = manager.initialize(); + workers[0].emitMessage({ type: 'ready' }); + await firstInit; + + const timedOutRun = manager.execute('while True: pass', { timeoutMs: 100 }); + await Promise.resolve(); + vi.advanceTimersByTime(101); + await expect(timedOutRun).rejects.toThrow('timed out'); + expect(workers[0].terminated).toBe(true); + expect(manager.isReady()).toBe(false); + + const secondInit = manager.initialize(); + workers[1].emitMessage({ type: 'ready' }); + await secondInit; + + const secondRun = manager.execute('40 + 2'); + await Promise.resolve(); + const request = workers[1].postedMessages[0] as { requestId: string }; + workers[1].emitMessage({ type: 'runResult', requestId: request.requestId, result: '42' }); + + await expect(secondRun).resolves.toEqual({ result: '42', stdout: '' }); + }); + + it('rejects macro execution when ABI context is invalid on caller side', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.renderMacroV1('def render(context):\n return {"html": "

ok

"}', { + env: { + isPreview: 'yes', + }, + }); + + await expect(runPromise).rejects.toThrow('Invalid macro context'); + expect(worker.postedMessages).toHaveLength(0); + }); + + it('returns validated macro result and stdout', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.renderMacroV1('def render(context):\n return {"html": "

ok

"}', { + env: { + isPreview: true, + }, + params: { + title: 'Hello', + }, + }); + + await Promise.resolve(); + const request = worker.postedMessages[0] as { requestId: string }; + worker.emitMessage({ type: 'stdout', requestId: request.requestId, chunk: 'rendering\n' }); + worker.emitMessage({ type: 'macroResult', requestId: request.requestId, result: { html: '

ok

' } }); + + await expect(runPromise).resolves.toEqual({ result: { html: '

ok

' }, stdout: 'rendering\n' }); + }); + + it('rejects macro execution when worker result violates ABI schema', async () => { + const worker = new MockWorker(); + const manager = new PythonRuntimeManager(() => worker as unknown as Worker); + + const initPromise = manager.initialize(); + worker.emitMessage({ type: 'ready' }); + await initPromise; + + const runPromise = manager.renderMacroV1('def render(context):\n return {"html": "

ok

"}', { + env: { + isPreview: true, + }, + }); + + await Promise.resolve(); + const request = worker.postedMessages[0] as { requestId: string }; + worker.emitMessage({ type: 'macroResult', requestId: request.requestId, result: { html: 42 } }); + + await expect(runPromise).rejects.toThrow('Invalid macro result'); + }); +}); diff --git a/tests/renderer/python/pythonRuntimeBenchmark.test.ts b/tests/renderer/python/pythonRuntimeBenchmark.test.ts new file mode 100644 index 0000000..2870480 --- /dev/null +++ b/tests/renderer/python/pythonRuntimeBenchmark.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest'; +import { summarizeDurations, runPythonRuntimeBenchmark } from '../../../src/renderer/python/pythonRuntimeBenchmark'; + +describe('pythonRuntimeBenchmark', () => { + it('computes p50 and p95 summary metrics', () => { + const summary = summarizeDurations([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + + expect(summary.count).toBe(10); + expect(summary.minMs).toBe(1); + expect(summary.maxMs).toBe(10); + expect(summary.meanMs).toBe(5.5); + expect(summary.p50Ms).toBe(5.5); + expect(summary.p95Ms).toBe(9.55); + }); + + it('runs benchmark phases and returns measured durations', async () => { + const calls: string[] = []; + const runtime = { + async runPythonAsync(code: string): Promise { + calls.push(code); + return 'ok'; + }, + }; + + const timestamps = [0, 100, 100, 110, 110, 111, 111, 113, 113, 116, 116, 120]; + let index = 0; + + const result = await runPythonRuntimeBenchmark({ + iterations: 4, + loadRuntime: async () => runtime, + now: () => { + const value = timestamps[index]; + index += 1; + return value; + }, + }); + + expect(result.coldStartMs).toBe(100); + expect(result.warmRunMs).toBe(10); + expect(result.repeatedMacro.samplesMs).toEqual([1, 2, 3, 4]); + expect(result.repeatedMacro.stats.p50Ms).toBe(2.5); + expect(result.repeatedMacro.stats.p95Ms).toBe(3.85); + expect(calls).toHaveLength(6); + }); +}); From 3ec8819d6dad008b6ae3255b9ab2adf0a46649e5 Mon Sep 17 00:00:00 2001 From: hugo Date: Sun, 22 Feb 2026 22:12:30 +0100 Subject: [PATCH 02/16] feat: phase 1 of python scripting --- PYTHON_SCRIPTING.md | 17 +- drizzle/0005_short_sally_floyd.sql | 15 + drizzle/meta/0005_snapshot.json | 913 ++++++++++++++++++ drizzle/meta/_journal.json | 7 + src/main/database/schema.ts | 20 + src/main/engine/ScriptEngine.ts | 264 +++++ src/main/engine/index.ts | 8 + src/main/ipc/handlers.ts | 32 + src/main/preload.ts | 9 + src/main/shared/electronApi.ts | 38 + .../components/ActivityBar/ActivityBar.tsx | 13 + src/renderer/components/Editor/Editor.tsx | 2 + src/renderer/components/Panel/Panel.css | 16 + src/renderer/components/Panel/Panel.tsx | 13 +- .../components/ScriptsView/ScriptsView.css | 32 + .../components/ScriptsView/ScriptsView.tsx | 114 +++ src/renderer/components/Sidebar/Sidebar.tsx | 122 ++- src/renderer/components/TabBar/TabBar.tsx | 10 + src/renderer/components/index.ts | 1 + src/renderer/i18n/locales/de.json | 10 + src/renderer/i18n/locales/en.json | 10 + src/renderer/i18n/locales/es.json | 10 + src/renderer/i18n/locales/fr.json | 10 + src/renderer/i18n/locales/it.json | 10 + src/renderer/navigation/activityBehavior.ts | 9 +- src/renderer/navigation/editorRouting.ts | 4 +- .../navigation/sidebarViewRegistry.ts | 1 + src/renderer/navigation/tabPolicy.ts | 19 + src/renderer/python/runtimeManagerInstance.ts | 7 + src/renderer/store/appStore.ts | 17 +- src/renderer/store/index.ts | 1 + tests/engine/ScriptEngine.test.ts | 137 +++ tests/ipc/handlers.test.ts | 121 +++ tests/renderer/components/Panel.test.tsx | 18 + .../components/ScriptsView.styles.test.ts | 23 + .../renderer/components/ScriptsView.test.tsx | 80 ++ .../components/SidebarScripts.test.tsx | 147 +++ .../navigation/activityBehavior.test.ts | 17 +- .../renderer/navigation/editorRouting.test.ts | 1 + .../navigation/sidebarViewRegistry.test.ts | 1 + tests/renderer/navigation/tabPolicy.test.ts | 32 + tests/renderer/viteConfig.test.ts | 6 + vite.config.ts | 6 + 43 files changed, 2329 insertions(+), 14 deletions(-) create mode 100644 drizzle/0005_short_sally_floyd.sql create mode 100644 drizzle/meta/0005_snapshot.json create mode 100644 src/main/engine/ScriptEngine.ts create mode 100644 src/renderer/components/ScriptsView/ScriptsView.css create mode 100644 src/renderer/components/ScriptsView/ScriptsView.tsx create mode 100644 src/renderer/python/runtimeManagerInstance.ts create mode 100644 tests/engine/ScriptEngine.test.ts create mode 100644 tests/renderer/components/ScriptsView.styles.test.ts create mode 100644 tests/renderer/components/ScriptsView.test.tsx create mode 100644 tests/renderer/components/SidebarScripts.test.tsx diff --git a/PYTHON_SCRIPTING.md b/PYTHON_SCRIPTING.md index 8d79890..0784ef4 100644 --- a/PYTHON_SCRIPTING.md +++ b/PYTHON_SCRIPTING.md @@ -108,10 +108,10 @@ Baseline benchmark (22 Feb 2026, local macOS run): Objective: user can create/run scripts and see output. Deliverables: -- [ ] Script storage model (DB index + filesystem source in `scripts/*.py`). -- [ ] CRUD APIs in `main/engine` + `ipc` handlers. -- [ ] Renderer scripts list + editor + run button. -- [ ] Console/output capture in existing bottom output area. +- [x] Script storage model (DB index + filesystem source in `scripts/*.py`). +- [x] CRUD APIs in `main/engine` + `ipc` handlers. +- [x] Renderer scripts list + editor + run button. +- [x] Console/output capture in existing bottom output area. - [ ] Project rebuild picks up `scripts/` changes. Out of scope for MVP: @@ -454,15 +454,18 @@ PR-14+: Optional advanced capabilities ## 10. Current Status -Status: Phase 0 in progress (MVP-first, full-scope preserved). +Status: Phase 1 in progress (MVP-first, full-scope preserved). Progress update (22 Feb 2026): - [x] PR-00 complete: Pyodide dependency + renderer worker bootstrap + ready signal. - [x] PR-01 complete: worker run/stdout/error protocol + timeout watchdog + runtime recovery. - [x] PR-02 complete: ABI v1 shared types/schemas + caller/worker validation. - [x] Phase 0 benchmark harness + baseline capture complete. +- [x] PR-03 complete: scripts table + `ScriptEngine` filesystem/DB round-trip persistence. +- [x] PR-04 complete: script CRUD IPC handlers + preload/shared API typing + IPC tests. +- [x] PR-05 complete: renderer scripts list/editor/run flow + output panel integration. Recommended next action: -1. Start Phase 1 PR-03: script persistence model (`scripts/*.py` + index metadata). -2. Add round-trip tests for create/update/delete between filesystem and DB. +1. Start Phase 1 PR-06: include `scripts/` in rebuild/meta-diff synchronization. +2. Keep scripts API access in renderer views/store paths only (no deep component IPC sprawl). 3. Keep benchmark command in CI/manual perf checks for regressions. diff --git a/drizzle/0005_short_sally_floyd.sql b/drizzle/0005_short_sally_floyd.sql new file mode 100644 index 0000000..c1c1a45 --- /dev/null +++ b/drizzle/0005_short_sally_floyd.sql @@ -0,0 +1,15 @@ +CREATE TABLE `scripts` ( + `id` text PRIMARY KEY NOT NULL, + `project_id` text NOT NULL, + `slug` text NOT NULL, + `title` text NOT NULL, + `kind` text DEFAULT 'utility' NOT NULL, + `entrypoint` text DEFAULT 'render' NOT NULL, + `enabled` integer DEFAULT true NOT NULL, + `version` integer DEFAULT 1 NOT NULL, + `file_path` text NOT NULL, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `scripts_project_slug_idx` ON `scripts` (`project_id`,`slug`); \ No newline at end of file diff --git a/drizzle/meta/0005_snapshot.json b/drizzle/meta/0005_snapshot.json new file mode 100644 index 0000000..5370a9e --- /dev/null +++ b/drizzle/meta/0005_snapshot.json @@ -0,0 +1,913 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b157a762-0743-4499-a635-16ac3fb5ee18", + "prevId": "46702982-9f8a-4c7e-8fb6-3270c3fbe120", + "tables": { + "chat_conversations": { + "name": "chat_conversations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_session_id": { + "name": "copilot_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "generated_file_hashes": { + "name": "generated_file_hashes", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "generated_file_hashes_project_path_idx": { + "name": "generated_file_hashes_project_path_idx", + "columns": [ + "project_id", + "relative_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "import_definitions": { + "name": "import_definitions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "wxr_file_path": { + "name": "wxr_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploads_folder_path": { + "name": "uploads_folder_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_analysis_result": { + "name": "last_analysis_result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "media": { + "name": "media", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alt": { + "name": "alt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "caption": { + "name": "caption", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sidecar_path": { + "name": "sidecar_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_links": { + "name": "post_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_post_id": { + "name": "source_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_post_id": { + "name": "target_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "link_text": { + "name": "link_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_media": { + "name": "post_media", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media_id": { + "name": "media_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "post_media_post_media_idx": { + "name": "post_media_post_media_idx", + "columns": [ + "post_id", + "media_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "posts": { + "name": "posts", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "excerpt": { + "name": "excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "categories": { + "name": "categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_title": { + "name": "published_title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_content": { + "name": "published_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_tags": { + "name": "published_tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_categories": { + "name": "published_categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_excerpt": { + "name": "published_excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "posts_project_slug_idx": { + "name": "posts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data_path": { + "name": "data_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scripts": { + "name": "scripts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'utility'" + }, + "entrypoint": { + "name": "entrypoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'render'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "scripts_project_slug_idx": { + "name": "scripts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tags": { + "name": "tags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tags_project_name_idx": { + "name": "tags_project_name_idx", + "columns": [ + "project_id", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index bbc9190..5c1762b 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1771605253203, "tag": "0004_overjoyed_paper_doll", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1771792324840, + "tag": "0005_short_sally_floyd", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index 5453e2e..be695c1 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -151,6 +151,24 @@ export const importDefinitions = sqliteTable('import_definitions', { updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(), }); +// Scripts table - stores metadata for Python scripts persisted in scripts/*.py +export const scripts = sqliteTable('scripts', { + id: text('id').primaryKey(), + projectId: text('project_id').notNull(), + slug: text('slug').notNull(), + title: text('title').notNull(), + kind: text('kind', { enum: ['macro', 'utility', 'transform'] }).notNull().default('utility'), + entrypoint: text('entrypoint').notNull().default('render'), + enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true), + version: integer('version').notNull().default(1), + filePath: text('file_path').notNull(), + createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), + updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(), +}, (table) => ({ + // Composite unique index: slug must be unique within each project + projectSlugIdx: uniqueIndex('scripts_project_slug_idx').on(table.projectId, table.slug), +})); + // Types for TypeScript export type Project = typeof projects.$inferSelect; export type NewProject = typeof projects.$inferInsert; @@ -174,3 +192,5 @@ export type ChatMessage = typeof chatMessages.$inferSelect; export type NewChatMessage = typeof chatMessages.$inferInsert; export type ImportDefinition = typeof importDefinitions.$inferSelect; export type NewImportDefinition = typeof importDefinitions.$inferInsert; +export type Script = typeof scripts.$inferSelect; +export type NewScript = typeof scripts.$inferInsert; diff --git a/src/main/engine/ScriptEngine.ts b/src/main/engine/ScriptEngine.ts new file mode 100644 index 0000000..63d3b65 --- /dev/null +++ b/src/main/engine/ScriptEngine.ts @@ -0,0 +1,264 @@ +import { EventEmitter } from 'events'; +import { v4 as uuidv4 } from 'uuid'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { app } from 'electron'; +import { and, desc, eq } from 'drizzle-orm'; +import { getDatabase } from '../database'; +import { scripts, type NewScript, type Script } from '../database/schema'; + +export type ScriptKind = 'macro' | 'utility' | 'transform'; + +export interface ScriptData { + id: string; + projectId: string; + slug: string; + title: string; + kind: ScriptKind; + entrypoint: string; + enabled: boolean; + version: number; + filePath: string; + content: string; + createdAt: Date; + updatedAt: Date; +} + +export interface CreateScriptInput { + title: string; + kind: ScriptKind; + content: string; + slug?: string; + entrypoint?: string; + enabled?: boolean; +} + +export interface UpdateScriptInput { + title?: string; + kind?: ScriptKind; + content?: string; + slug?: string; + entrypoint?: string; + enabled?: boolean; +} + +export class ScriptEngine extends EventEmitter { + private currentProjectId = 'default'; + private dataDir: string | null = null; + + setProjectContext(projectId: string, dataDir?: string): void { + this.currentProjectId = projectId; + this.dataDir = dataDir || null; + } + + getProjectContext(): string { + return this.currentProjectId; + } + + async createScript(input: CreateScriptInput): Promise { + const now = new Date(); + const allScripts = await this.getAllScriptRows(); + const desiredSlug = this.normalizeSlug(input.slug || input.title || 'script'); + const uniqueSlug = this.ensureUniqueSlug(desiredSlug, allScripts); + const scriptId = uuidv4(); + const filePath = this.getScriptFilePath(uniqueSlug); + + await fs.mkdir(this.getScriptsDir(), { recursive: true }); + await fs.writeFile(filePath, input.content, 'utf-8'); + + const row: NewScript = { + id: scriptId, + projectId: this.currentProjectId, + slug: uniqueSlug, + title: input.title, + kind: input.kind, + entrypoint: input.entrypoint || 'render', + enabled: input.enabled ?? true, + version: 1, + filePath, + createdAt: now, + updatedAt: now, + }; + + await getDatabase().getLocal().insert(scripts).values(row); + + const created = await this.toScriptData(row as Script); + this.emit('scriptCreated', created); + return created; + } + + async updateScript(id: string, updates: UpdateScriptInput): Promise { + const existing = await this.getScriptRow(id); + if (!existing) { + return null; + } + + const allScripts = await this.getAllScriptRows(); + const desiredSlug = this.normalizeSlug(updates.slug || updates.title || existing.slug); + const nextSlug = this.ensureUniqueSlug(desiredSlug, allScripts, existing.id); + const nextFilePath = this.getScriptFilePath(nextSlug); + const now = new Date(); + + if (existing.filePath !== nextFilePath) { + await fs.mkdir(this.getScriptsDir(), { recursive: true }); + await fs.rename(existing.filePath, nextFilePath); + } + + if (typeof updates.content === 'string') { + await fs.writeFile(nextFilePath, updates.content, 'utf-8'); + } + + await getDatabase().getLocal() + .update(scripts) + .set({ + title: updates.title ?? existing.title, + slug: nextSlug, + kind: updates.kind ?? existing.kind, + entrypoint: updates.entrypoint ?? existing.entrypoint, + enabled: updates.enabled ?? existing.enabled, + filePath: nextFilePath, + version: existing.version + 1, + updatedAt: now, + }) + .where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + + const updatedRow = await this.getScriptRow(existing.id); + if (!updatedRow) { + return null; + } + + const updated = await this.toScriptData(updatedRow); + this.emit('scriptUpdated', updated); + return updated; + } + + async deleteScript(id: string): Promise { + const existing = await this.getScriptRow(id); + if (!existing) { + return false; + } + + await getDatabase().getLocal() + .delete(scripts) + .where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + + try { + await fs.unlink(existing.filePath); + } catch (error) { + const fsError = error as NodeJS.ErrnoException; + if (fsError.code !== 'ENOENT') { + throw error; + } + } + + this.emit('scriptDeleted', id); + return true; + } + + async getScript(id: string): Promise { + const row = await this.getScriptRow(id); + if (!row) { + return null; + } + return this.toScriptData(row); + } + + async getAllScripts(): Promise { + const rows = await this.getAllScriptRows(); + return Promise.all(rows.map((item) => this.toScriptData(item))); + } + + private async getScriptRow(id: string): Promise