diff --git a/PYTHON_SCRIPTING.md b/PYTHON_SCRIPTING.md index b89d445..c9028fd 100644 --- a/PYTHON_SCRIPTING.md +++ b/PYTHON_SCRIPTING.md @@ -10,6 +10,7 @@ When plan and code differ, code is the source of truth. - [x] Pyodide dependency integrated. - [x] Renderer worker runtime exists (`pythonRuntime.worker.ts`) with ready/error/stdout/run protocol. - [x] Runtime timeout watchdog + reset/recovery implemented in `PythonRuntimeManager`. +- [x] Renderer runtime request queueing implemented (concurrent calls are serialized in manager). - [x] ABI v1 schemas and validation for macro context/result implemented (`abiV1.ts`). - [x] Benchmark harness implemented (`npm run bench:python-runtime -- `). - [x] Script persistence model implemented (`scripts` DB table + `scripts/*.py` files). @@ -18,15 +19,16 @@ When plan and code differ, code is the source of truth. - [x] Preload + shared API typings for scripts implemented. - [x] Renderer scripts UX implemented (sidebar list, editor, save, run, delete). - [x] Script syntax check + entrypoint discovery integrated in editor UX. -- [x] Blogmark transform pipeline executes Python transform scripts (`kind='transform'`). +- [x] Blogmark transform pipeline executes Python transform scripts (`kind='transform'`) via a queued worker runtime by default. +- [x] Project preference `pythonRuntimeMode` added with Settings → Technology section. ## Confirmed Deviations from Original Plan These are current realities and should be treated as authoritative unless we explicitly decide to change them. -1. **Transform script runtime location differs** - - Original plan: untrusted Python runs in renderer worker only. - - Actual implementation: Blogmark transform scripts run in **main process** Pyodide (`BlogmarkTransformService`). +1. **Transform runtime is now configurable (project-level)** + - Default: `webworker` (worker-thread based Python runtime with queued requests). + - Optional fallback: `main-thread` legacy execution mode. 2. **Render-time macro migration has not happened yet** - Original plan: all render-time macros become Python-backed. @@ -36,24 +38,31 @@ These are current realities and should be treated as authoritative unless we exp - ABI v1 + runtime manager support exist. - Main page generation path still uses existing JS macro rendering. -4. **Scripts rebuild/meta-diff sync is still missing** - - Script CRUD works via app APIs. - - No implemented project-wide “rebuild from files” parity for `scripts/` equivalent to posts/media rebuild flows. +4. **Scripts rebuild/sync parity is implemented (simple policy)** + - `ScriptEngine.rebuildDatabaseFromFiles()` now rebuilds DB metadata from `scripts/*.py`. + - `ScriptEngine.reconcileScriptsFromGitChanges()` now handles added/modified/deleted/renamed script files after git pull. + - Settings → Data now includes **Rebuild Scripts** button (`scripts:rebuildFromFiles`) for manual parity with posts/media rebuild. ## Remaining Work Only -## 1) Decide and enforce Python runtime boundary (P0) +## 1) Python runtime boundary (P0) — Implemented -- [ ] Decide if `transform` scripts should stay in main process or move to renderer worker. -- [ ] If staying in main process: add explicit timeout/kill/recovery safeguards equivalent to worker watchdog behavior. -- [ ] If moving to worker: route transform execution through typed IPC/worker bridge and remove main-process execution path. -- [ ] Document final security model in this file after decision. +- [x] Worker model introduced for Blogmark transform execution with queued communication. +- [x] Runtime mode made project-configurable via Settings → Technology (`pythonRuntimeMode`). +- [x] Legacy main-thread mode retained as explicit fallback option. -## 2) Add scripts file-system rebuild/sync (P1) +## 2) Add scripts file-system rebuild/sync (P1) — Implemented -- [ ] Implement rebuild/meta-diff style synchronization for `scripts/` so external file edits are detected. -- [ ] Define conflict handling policy between DB metadata and script file frontmatter/body. -- [ ] Add tests for create/edit/delete performed outside app while app is closed/open. +- [x] Implement rebuild/meta-diff style synchronization for `scripts/` so external file edits are detected. +- [x] Define conflict handling policy between DB metadata and script file frontmatter/body. +- [x] Add tests for create/edit/delete performed outside app while app is closed/open. + +### Implemented policy (simple) + +- Source of truth: script file + docstring frontmatter when present/valid. +- Rebuild path: delete current `scripts` rows for active project and re-import from `scripts/*.py`. +- Reconcile path (git pull): apply file deltas (`added|modified|deleted|renamed`) and upsert/delete rows. +- Conflict behavior: prefer file metadata/body; fall back to safe defaults when values are missing/invalid. ## 3) Wire Python macros into render pipeline (P1) @@ -88,7 +97,7 @@ These are current realities and should be treated as authoritative unless we exp ## Acceptance Gate Before Marking Python Scripting “Complete” - [ ] Render-time macros run through Python script path in production generation flow. -- [ ] Scripts directory external changes are synchronized reliably. -- [ ] Runtime boundary decision implemented and protected by tests. +- [x] Scripts directory external changes are synchronized reliably. +- [x] Runtime boundary decision implemented and protected by tests. - [ ] Legacy JS macro path removed (or explicitly retained with documented rationale). -- [ ] `npm test` and `npm run build` pass. +- [x] `npm test` and `npm run build` pass. diff --git a/src/main/engine/BlogmarkPythonWorkerRuntime.ts b/src/main/engine/BlogmarkPythonWorkerRuntime.ts new file mode 100644 index 0000000..f3256d5 --- /dev/null +++ b/src/main/engine/BlogmarkPythonWorkerRuntime.ts @@ -0,0 +1,261 @@ +import * as path from 'path'; +import { Worker } from 'worker_threads'; + +interface WorkerRunTransformRequest { + type: 'runTransform'; + requestId: string; + scriptContent: string; + entrypoint: string; + payloadJson: string; +} + +interface WorkerReadyMessage { + type: 'ready'; +} + +interface WorkerResultMessage { + type: 'transformResult'; + requestId: string; + output: unknown; + toasts: string[]; +} + +interface WorkerErrorMessage { + type: 'transformError'; + requestId: string; + error: string; +} + +interface WorkerFatalErrorMessage { + type: 'error'; + error: string; +} + +type WorkerResponseMessage = WorkerReadyMessage | WorkerResultMessage | WorkerErrorMessage | WorkerFatalErrorMessage; + +interface QueuedRequest { + request: WorkerRunTransformRequest; + timeoutMs: number; + resolve: (value: { output: unknown; toasts: string[] }) => void; + reject: (error: Error) => void; +} + +interface ActiveRequest extends QueuedRequest { + timeoutId: ReturnType | null; +} + +export class BlogmarkPythonWorkerRuntime { + private worker: Worker | null = null; + private workerReady = false; + private workerStartPromise: Promise | null = null; + private workerStartResolve: (() => void) | null = null; + private workerStartReject: ((error: Error) => void) | null = null; + private activeRequest: ActiveRequest | null = null; + private queue: QueuedRequest[] = []; + private requestCounter = 0; + + async executeTransform(params: { + scriptContent: string; + entrypoint: string; + payloadJson: string; + timeoutMs?: number; + }): Promise<{ output: unknown; toasts: string[] }> { + const requestId = this.nextRequestId(); + const timeoutMs = params.timeoutMs ?? 5000; + + return new Promise<{ output: unknown; toasts: string[] }>((resolve, reject) => { + this.queue.push({ + request: { + type: 'runTransform', + requestId, + scriptContent: params.scriptContent, + entrypoint: params.entrypoint, + payloadJson: params.payloadJson, + }, + timeoutMs, + resolve, + reject, + }); + + this.dispatchNext().catch((error) => { + reject(error instanceof Error ? error : new Error(String(error))); + }); + }); + } + + dispose(): void { + this.rejectStartPromise(new Error('Python worker runtime disposed')); + this.rejectActiveAndQueue(new Error('Python worker runtime disposed')); + this.resetWorker(); + } + + private async dispatchNext(): Promise { + if (this.activeRequest || this.queue.length === 0) { + return; + } + + await this.ensureWorkerStarted(); + + const nextRequest = this.queue.shift(); + if (!nextRequest) { + return; + } + + const timeoutId = setTimeout(() => { + if (!this.activeRequest || this.activeRequest.request.requestId !== nextRequest.request.requestId) { + return; + } + + const timeoutError = new Error(`Python transform timed out after ${nextRequest.timeoutMs}ms`); + this.activeRequest.reject(timeoutError); + this.activeRequest = null; + this.resetWorker(); + void this.dispatchNext(); + }, nextRequest.timeoutMs); + + this.activeRequest = { + ...nextRequest, + timeoutId, + }; + + this.worker?.postMessage(nextRequest.request); + } + + private async ensureWorkerStarted(): Promise { + if (this.worker && this.workerReady) { + return; + } + + if (this.workerStartPromise) { + return this.workerStartPromise; + } + + const workerPath = path.join(__dirname, 'blogmarkPython.worker.js'); + this.worker = new Worker(workerPath); + this.workerReady = false; + + this.worker.on('message', (message: WorkerResponseMessage) => { + this.handleWorkerMessage(message); + }); + + this.worker.on('error', (error) => { + this.handleWorkerCrash(error instanceof Error ? error : new Error(String(error))); + }); + + this.worker.on('exit', (code) => { + if (code !== 0) { + this.handleWorkerCrash(new Error(`Python worker exited with code ${code}`)); + } + }); + + this.workerStartPromise = new Promise((resolve, reject) => { + this.workerStartResolve = resolve; + this.workerStartReject = reject; + }); + + return this.workerStartPromise; + } + + private handleWorkerMessage(message: WorkerResponseMessage): void { + if (message.type === 'ready') { + this.workerReady = true; + this.resolveStartPromise(); + return; + } + + if (message.type === 'error') { + this.handleWorkerCrash(new Error(message.error)); + return; + } + + const active = this.activeRequest; + if (!active) { + return; + } + + if (active.request.requestId !== message.requestId) { + return; + } + + if (active.timeoutId) { + clearTimeout(active.timeoutId); + } + + this.activeRequest = null; + + if (message.type === 'transformResult') { + active.resolve({ output: message.output, toasts: message.toasts }); + } else { + active.reject(new Error(message.error)); + } + + void this.dispatchNext(); + } + + private handleWorkerCrash(error: Error): void { + this.rejectStartPromise(error); + this.rejectActiveAndQueue(error); + this.resetWorker(); + } + + private rejectActiveAndQueue(error: Error): void { + if (this.activeRequest) { + if (this.activeRequest.timeoutId) { + clearTimeout(this.activeRequest.timeoutId); + } + this.activeRequest.reject(error); + this.activeRequest = null; + } + + while (this.queue.length > 0) { + const queued = this.queue.shift(); + queued?.reject(error); + } + } + + private resolveStartPromise(): void { + if (this.workerStartResolve) { + this.workerStartResolve(); + } + this.workerStartResolve = null; + this.workerStartReject = null; + this.workerStartPromise = null; + } + + private rejectStartPromise(error: Error): void { + if (this.workerStartReject) { + this.workerStartReject(error); + } + this.workerStartResolve = null; + this.workerStartReject = null; + this.workerStartPromise = null; + } + + private resetWorker(): void { + if (this.worker) { + this.worker.removeAllListeners(); + this.worker.terminate(); + } + + this.worker = null; + this.workerReady = false; + this.workerStartPromise = null; + this.workerStartResolve = null; + this.workerStartReject = null; + } + + private nextRequestId(): string { + this.requestCounter += 1; + return `blogmark-py-${this.requestCounter}`; + } +} + +let blogmarkPythonWorkerRuntimeInstance: BlogmarkPythonWorkerRuntime | null = null; + +export function getBlogmarkPythonWorkerRuntime(): BlogmarkPythonWorkerRuntime { + if (!blogmarkPythonWorkerRuntimeInstance) { + blogmarkPythonWorkerRuntimeInstance = new BlogmarkPythonWorkerRuntime(); + } + + return blogmarkPythonWorkerRuntimeInstance; +} diff --git a/src/main/engine/BlogmarkTransformService.ts b/src/main/engine/BlogmarkTransformService.ts index 7f65c66..a888682 100644 --- a/src/main/engine/BlogmarkTransformService.ts +++ b/src/main/engine/BlogmarkTransformService.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; import { getScriptEngine } from './ScriptEngine'; +import { getMetaEngine } from './MetaEngine'; +import { getBlogmarkPythonWorkerRuntime } from './BlogmarkPythonWorkerRuntime'; const transformPostSchema = z.object({ title: z.string().trim().min(1), @@ -55,6 +57,8 @@ export interface BlogmarkTransformResult { toasts: string[]; } +export type PythonRuntimeMode = 'webworker' | 'main-thread'; + const MAX_TOASTS_PER_SCRIPT = 5; const MAX_TOASTS_TOTAL = 20; const MAX_TOAST_LENGTH = 300; @@ -142,6 +146,28 @@ function toErrorMessage(error: unknown): string { return String(error); } +function resolveTransformEntrypoint(value: string): string { + const nextEntrypoint = typeof value === 'string' ? value.trim() : ''; + if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(nextEntrypoint) && nextEntrypoint !== 'main') { + return nextEntrypoint; + } + + return 'transform'; +} + +function resolvePythonRuntimeMode(value: unknown): PythonRuntimeMode { + if (value === 'main-thread') { + return 'main-thread'; + } + + return 'webworker'; +} + +async function getConfiguredPythonRuntimeMode(): Promise { + const metadata = await getMetaEngine().getProjectMetadata(); + return resolvePythonRuntimeMode((metadata as { pythonRuntimeMode?: unknown } | null)?.pythonRuntimeMode); +} + class PythonBlogmarkTransformExecutor implements BlogmarkTransformExecutor { private runtimePromise: Promise | null = null; @@ -169,7 +195,7 @@ def toast(message): await runtime.runPythonAsync(script.content); - const requestedEntrypoint = this.resolveEntrypoint(script.entrypoint); + const requestedEntrypoint = resolveTransformEntrypoint(script.entrypoint); const payload = JSON.stringify(input); runtime.globals.set('__bds_transform_payload_json', payload); runtime.globals.set('__bds_transform_entrypoint', requestedEntrypoint); @@ -200,15 +226,6 @@ json.dumps(_result) }; } - private resolveEntrypoint(value: string): string { - const nextEntrypoint = typeof value === 'string' ? value.trim() : ''; - if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(nextEntrypoint) && nextEntrypoint !== 'main') { - return nextEntrypoint; - } - - return 'transform'; - } - private async getRuntime(): Promise { if (!this.runtimePromise) { this.runtimePromise = (async () => { @@ -221,11 +238,26 @@ json.dumps(_result) } } +class PythonWorkerBlogmarkTransformExecutor implements BlogmarkTransformExecutor { + async runTransform(script: BlogmarkTransformScriptRecord, input: BlogmarkTransformInput): Promise { + return getBlogmarkPythonWorkerRuntime().executeTransform({ + scriptContent: script.content, + entrypoint: resolveTransformEntrypoint(script.entrypoint), + payloadJson: JSON.stringify(input), + }); + } +} + +const mainThreadExecutor = new PythonBlogmarkTransformExecutor(); +const workerExecutor = new PythonWorkerBlogmarkTransformExecutor(); + export class BlogmarkTransformService { constructor( private readonly dependencies: { provider?: BlogmarkTransformScriptProvider; executor?: BlogmarkTransformExecutor; + resolvePythonRuntimeMode?: () => Promise; + executors?: Partial>; } = {}, ) {} @@ -237,7 +269,7 @@ export class BlogmarkTransformService { }; const provider = this.dependencies.provider ?? scriptEngineBackedProvider; - const executor = this.dependencies.executor ?? new PythonBlogmarkTransformExecutor(); + const executor = this.dependencies.executor ?? await this.resolveExecutorForConfiguredRuntime(); const scripts = await provider.getScripts(); const activeTransforms = scripts @@ -303,6 +335,18 @@ export class BlogmarkTransformService { toasts, }; } + + private async resolveExecutorForConfiguredRuntime(): Promise { + const resolveMode = this.dependencies.resolvePythonRuntimeMode ?? getConfiguredPythonRuntimeMode; + const mode = await resolveMode(); + const executors = this.dependencies.executors ?? {}; + + if (mode === 'main-thread') { + return executors['main-thread'] ?? mainThreadExecutor; + } + + return executors.webworker ?? workerExecutor; + } } let blogmarkTransformServiceInstance: BlogmarkTransformService | null = null; diff --git a/src/main/engine/GitEngine.ts b/src/main/engine/GitEngine.ts index 587ba38..0eec483 100644 --- a/src/main/engine/GitEngine.ts +++ b/src/main/engine/GitEngine.ts @@ -140,6 +140,14 @@ export interface GitPostFileChange { previousPath?: string; } +export type GitScriptFileChangeStatus = 'added' | 'modified' | 'deleted' | 'renamed'; + +export interface GitScriptFileChange { + status: GitScriptFileChangeStatus; + path: string; + previousPath?: string; +} + type GitProvider = 'unknown' | 'github' | 'gitlab' | 'gitea-forgejo'; let gitEngineInstance: GitEngine | null = null; @@ -526,7 +534,12 @@ export class GitEngine { return this.markdownExtensions.has(extension); } - private parseNameStatusOutput(raw: string): GitPostFileChange[] { + private isScriptsPythonPath(value: string): boolean { + const normalized = this.normalizeRepoRelativePath(value); + return normalized.startsWith('scripts/') && path.extname(normalized).toLowerCase() === '.py'; + } + + private parseNameStatusOutput(raw: string, pathMatcher: (value: string) => boolean): GitPostFileChange[] { const tokens = raw.split('\0').filter((token) => token.length > 0); const changes: GitPostFileChange[] = []; @@ -543,7 +556,7 @@ export class GitEngine { const previousPath = this.normalizeRepoRelativePath(previousPathRaw); const pathValue = this.normalizeRepoRelativePath(nextPathRaw); - if (this.isPostsMarkdownPath(previousPath) || this.isPostsMarkdownPath(pathValue)) { + if (pathMatcher(previousPath) || pathMatcher(pathValue)) { changes.push({ status: 'renamed', path: pathValue, @@ -555,7 +568,7 @@ export class GitEngine { const filePathRaw = tokens[index++] ?? ''; const filePath = this.normalizeRepoRelativePath(filePathRaw); - if (!this.isPostsMarkdownPath(filePath)) { + if (!pathMatcher(filePath)) { continue; } @@ -1338,13 +1351,40 @@ export class GitEngine { try { const output = await git.raw(args); - return this.parseNameStatusOutput(output); + return this.parseNameStatusOutput(output, (value) => this.isPostsMarkdownPath(value)); } catch (error) { const message = error instanceof Error ? error.message : String(error ?? ''); if (this.isSpawnBadFileDescriptorError(message)) { try { const output = await this.runGitCli(projectPath, args); - return this.parseNameStatusOutput(output); + return this.parseNameStatusOutput(output, (value) => this.isPostsMarkdownPath(value)); + } catch { + return []; + } + } + return []; + } + } + + async getChangedScriptFilesBetween(projectPath: string, fromCommit: string, toCommit: string): Promise { + const fromRef = fromCommit.trim(); + const toRef = toCommit.trim(); + if (!fromRef || !toRef || fromRef === toRef) { + return []; + } + + const git = this.createNonInteractiveGit(projectPath); + const args = ['diff', '--name-status', '--find-renames', '-z', `${fromRef}..${toRef}`, '--', 'scripts']; + + try { + const output = await git.raw(args); + return this.parseNameStatusOutput(output, (value) => this.isScriptsPythonPath(value)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error ?? ''); + if (this.isSpawnBadFileDescriptorError(message)) { + try { + const output = await this.runGitCli(projectPath, args); + return this.parseNameStatusOutput(output, (value) => this.isScriptsPythonPath(value)); } catch { return []; } diff --git a/src/main/engine/MetaEngine.ts b/src/main/engine/MetaEngine.ts index 0ed48bd..85ad141 100644 --- a/src/main/engine/MetaEngine.ts +++ b/src/main/engine/MetaEngine.ts @@ -24,6 +24,7 @@ export interface ProjectMetadata { defaultAuthor?: string; // Default author for new posts and media maxPostsPerPage?: number; // Preview/server maximum posts per page (default 50) blogmarkCategory?: string; // Category used for externally captured bookmark posts + pythonRuntimeMode?: 'webworker' | 'main-thread'; // Runtime mode for Python script execution picoTheme?: PicoThemeName; // Selected Pico CSS theme for preview/rendering categoryMetadata?: Record; // Per-category metadata for UI/rendering categorySettings?: Record; // Per-category list rendering preferences @@ -85,6 +86,7 @@ function normalizeProjectMetadata(metadata: ProjectMetadata): ProjectMetadata { const blogmarkCategory = typeof metadata.blogmarkCategory === 'string' ? normalizeNonEmptyTaxonomyTerm(metadata.blogmarkCategory) ?? undefined : undefined; + const pythonRuntimeMode = metadata.pythonRuntimeMode === 'main-thread' ? 'main-thread' : 'webworker'; const picoTheme = sanitizePicoTheme(metadata.picoTheme); const categoryMetadata = normalizeCategoryMetadata(metadata.categoryMetadata ?? metadata.categorySettings); return { @@ -92,6 +94,7 @@ function normalizeProjectMetadata(metadata: ProjectMetadata): ProjectMetadata { publicUrl, maxPostsPerPage, blogmarkCategory, + pythonRuntimeMode, picoTheme, categoryMetadata, categorySettings: undefined, @@ -306,6 +309,7 @@ export class MetaEngine extends EventEmitter { defaultAuthor: normalizedUpdates.defaultAuthor, maxPostsPerPage: normalizedUpdates.maxPostsPerPage, blogmarkCategory: normalizedUpdates.blogmarkCategory, + pythonRuntimeMode: normalizedUpdates.pythonRuntimeMode, picoTheme: normalizedUpdates.picoTheme, categoryMetadata: normalizedUpdates.categoryMetadata, }); diff --git a/src/main/engine/ScriptEngine.ts b/src/main/engine/ScriptEngine.ts index ed9b379..c31f2b4 100644 --- a/src/main/engine/ScriptEngine.ts +++ b/src/main/engine/ScriptEngine.ts @@ -42,6 +42,37 @@ export interface UpdateScriptInput { enabled?: boolean; } +export type GitScriptFileChangeStatus = 'added' | 'modified' | 'deleted' | 'renamed'; + +export interface GitScriptFileChange { + status: GitScriptFileChangeStatus; + path: string; + previousPath?: string; +} + +export interface ScriptReconcileResult { + created: number; + updated: number; + deleted: number; + processedFiles: number; +} + +interface ParsedScriptFile { + metadata: { + id?: string; + projectId?: string; + slug?: string; + title?: string; + kind?: string; + entrypoint?: string; + enabled?: boolean; + version?: number; + createdAt?: string; + updatedAt?: string; + }; + body: string; +} + export class ScriptEngine extends EventEmitter { private currentProjectId = 'default'; private dataDir: string | null = null; @@ -191,6 +222,205 @@ export class ScriptEngine extends EventEmitter { return Promise.all(rows.map((item) => this.toScriptData(item))); } + async rebuildDatabaseFromFiles(): Promise { + const db = getDatabase().getLocal(); + const scriptsDir = this.getScriptsDir(); + + await db.delete(scripts).where(eq(scripts.projectId, this.currentProjectId)); + + const pythonFiles = await this.scanScriptFiles(scriptsDir); + if (pythonFiles.length === 0) { + this.emit('scriptsRebuilt'); + return; + } + + const usedIds = new Set(); + const insertedRows: Script[] = []; + + for (const filePath of pythonFiles) { + const parsed = await this.readScriptFileWithMetadata(filePath); + if (!parsed) { + continue; + } + + const desiredSlug = this.normalizeSlug(parsed.metadata.slug || path.basename(filePath, '.py')); + const slug = this.ensureUniqueSlug(desiredSlug, insertedRows); + + const desiredId = typeof parsed.metadata.id === 'string' && parsed.metadata.id.trim().length > 0 + ? parsed.metadata.id.trim() + : uuidv4(); + const id = usedIds.has(desiredId) ? uuidv4() : desiredId; + + const now = new Date(); + const row: NewScript = { + id, + projectId: this.currentProjectId, + slug, + title: this.normalizeTitle(parsed.metadata.title, slug), + kind: this.normalizeKind(parsed.metadata.kind), + entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint), + enabled: this.normalizeEnabled(parsed.metadata.enabled), + version: this.normalizeVersion(parsed.metadata.version), + filePath, + createdAt: this.normalizeDate(parsed.metadata.createdAt, now), + updatedAt: this.normalizeDate(parsed.metadata.updatedAt, now), + }; + + await db.insert(scripts).values(row); + insertedRows.push(row as Script); + usedIds.add(id); + } + + this.emit('scriptsRebuilt'); + } + + async reconcileScriptsFromGitChanges(projectPath: string, changes: GitScriptFileChange[]): Promise { + const db = getDatabase().getLocal(); + const normalizedProjectPath = path.resolve(projectPath); + + const relevantChanges = changes.filter((change) => { + if (!this.isPythonScriptPath(change.path)) { + return false; + } + if (change.status === 'renamed' && change.previousPath && !this.isPythonScriptPath(change.previousPath) && !this.isPythonScriptPath(change.path)) { + return false; + } + return true; + }); + + if (relevantChanges.length === 0) { + return { created: 0, updated: 0, deleted: 0, processedFiles: 0 }; + } + + const scriptRows = await this.getAllScriptRows(); + const scriptsByPath = new Map(); + for (const row of scriptRows) { + scriptsByPath.set(this.normalizePathForCompare(row.filePath), row); + } + + let created = 0; + let updated = 0; + let deleted = 0; + let processedFiles = 0; + + for (const change of relevantChanges) { + const absolutePath = this.normalizePathForCompare(path.resolve(normalizedProjectPath, change.path)); + const previousAbsolutePath = change.previousPath + ? this.normalizePathForCompare(path.resolve(normalizedProjectPath, change.previousPath)) + : null; + + if (change.status === 'deleted') { + const existing = scriptsByPath.get(absolutePath); + if (!existing) { + continue; + } + + await db.delete(scripts).where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + scriptsByPath.delete(absolutePath); + this.emit('scriptDeleted', existing.id); + deleted += 1; + processedFiles += 1; + continue; + } + + let existing = previousAbsolutePath + ? (scriptsByPath.get(previousAbsolutePath) || scriptsByPath.get(absolutePath)) + : scriptsByPath.get(absolutePath); + + const parsed = await this.readScriptFileWithMetadata(absolutePath); + if (!parsed) { + continue; + } + + const allRows = await this.getAllScriptRows(); + const parsedId = typeof parsed.metadata.id === 'string' ? parsed.metadata.id.trim() : ''; + if (!existing && parsedId.length > 0) { + const byId = allRows.find((row) => row.id === parsedId); + if (byId) { + existing = byId; + } + } + const desiredSlug = this.normalizeSlug(parsed.metadata.slug || path.basename(absolutePath, '.py')); + const slug = this.ensureUniqueSlug(desiredSlug, allRows, existing?.id); + + if (existing) { + const updateNow = new Date(); + const nextRow = { + title: this.normalizeTitle(parsed.metadata.title, slug, existing.title), + slug, + kind: this.normalizeKind(parsed.metadata.kind, existing.kind), + entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint, existing.entrypoint), + enabled: this.normalizeEnabled(parsed.metadata.enabled, existing.enabled), + version: this.normalizeVersion(parsed.metadata.version, existing.version), + filePath: absolutePath, + createdAt: this.normalizeDate(parsed.metadata.createdAt, existing.createdAt), + updatedAt: this.normalizeDate(parsed.metadata.updatedAt, updateNow), + }; + + await db.update(scripts) + .set(nextRow) + .where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + + const updatedRow = await this.getScriptRow(existing.id); + if (updatedRow) { + const updatedScript = await this.toScriptData(updatedRow); + this.emit('scriptUpdated', updatedScript); + } + + if (previousAbsolutePath) { + scriptsByPath.delete(previousAbsolutePath); + } + scriptsByPath.set(absolutePath, { + ...existing, + ...nextRow, + }); + updated += 1; + processedFiles += 1; + continue; + } + + const desiredId = typeof parsed.metadata.id === 'string' && parsed.metadata.id.trim().length > 0 + ? parsed.metadata.id.trim() + : uuidv4(); + const idExists = allRows.some((row) => row.id === desiredId); + const rowId = idExists ? uuidv4() : desiredId; + const now = new Date(); + + const newRow: NewScript = { + id: rowId, + projectId: this.currentProjectId, + slug, + title: this.normalizeTitle(parsed.metadata.title, slug), + kind: this.normalizeKind(parsed.metadata.kind), + entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint), + enabled: this.normalizeEnabled(parsed.metadata.enabled), + version: this.normalizeVersion(parsed.metadata.version), + filePath: absolutePath, + createdAt: this.normalizeDate(parsed.metadata.createdAt, now), + updatedAt: this.normalizeDate(parsed.metadata.updatedAt, now), + }; + + await db.insert(scripts).values(newRow); + + const createdRow = await this.getScriptRow(newRow.id); + if (createdRow) { + const createdScript = await this.toScriptData(createdRow); + this.emit('scriptCreated', createdScript); + } + + scriptsByPath.set(absolutePath, newRow as Script); + created += 1; + processedFiles += 1; + } + + return { + created, + updated, + deleted, + processedFiles, + }; + } + private async getScriptRow(id: string): Promise