diff --git a/PYTHON_SCRIPTING.md b/PYTHON_SCRIPTING.md index e12c15d..c9028fd 100644 --- a/PYTHON_SCRIPTING.md +++ b/PYTHON_SCRIPTING.md @@ -38,9 +38,10 @@ These are current realities and should be treated as authoritative unless we exp - ABI v1 + runtime manager support exist. - Main page generation path still uses existing JS macro rendering. -4. **Scripts rebuild/meta-diff sync is still missing** - - Script CRUD works via app APIs. - - No implemented project-wide “rebuild from files” parity for `scripts/` equivalent to posts/media rebuild flows. +4. **Scripts rebuild/sync parity is implemented (simple policy)** + - `ScriptEngine.rebuildDatabaseFromFiles()` now rebuilds DB metadata from `scripts/*.py`. + - `ScriptEngine.reconcileScriptsFromGitChanges()` now handles added/modified/deleted/renamed script files after git pull. + - Settings → Data now includes **Rebuild Scripts** button (`scripts:rebuildFromFiles`) for manual parity with posts/media rebuild. ## Remaining Work Only @@ -50,11 +51,18 @@ These are current realities and should be treated as authoritative unless we exp - [x] Runtime mode made project-configurable via Settings → Technology (`pythonRuntimeMode`). - [x] Legacy main-thread mode retained as explicit fallback option. -## 2) Add scripts file-system rebuild/sync (P1) +## 2) Add scripts file-system rebuild/sync (P1) — Implemented -- [ ] Implement rebuild/meta-diff style synchronization for `scripts/` so external file edits are detected. -- [ ] Define conflict handling policy between DB metadata and script file frontmatter/body. -- [ ] Add tests for create/edit/delete performed outside app while app is closed/open. +- [x] Implement rebuild/meta-diff style synchronization for `scripts/` so external file edits are detected. +- [x] Define conflict handling policy between DB metadata and script file frontmatter/body. +- [x] Add tests for create/edit/delete performed outside app while app is closed/open. + +### Implemented policy (simple) + +- Source of truth: script file + docstring frontmatter when present/valid. +- Rebuild path: delete current `scripts` rows for active project and re-import from `scripts/*.py`. +- Reconcile path (git pull): apply file deltas (`added|modified|deleted|renamed`) and upsert/delete rows. +- Conflict behavior: prefer file metadata/body; fall back to safe defaults when values are missing/invalid. ## 3) Wire Python macros into render pipeline (P1) @@ -89,7 +97,7 @@ These are current realities and should be treated as authoritative unless we exp ## Acceptance Gate Before Marking Python Scripting “Complete” - [ ] Render-time macros run through Python script path in production generation flow. -- [ ] Scripts directory external changes are synchronized reliably. +- [x] Scripts directory external changes are synchronized reliably. - [x] Runtime boundary decision implemented and protected by tests. - [ ] Legacy JS macro path removed (or explicitly retained with documented rationale). -- [ ] `npm test` and `npm run build` pass. +- [x] `npm test` and `npm run build` pass. diff --git a/src/main/engine/GitEngine.ts b/src/main/engine/GitEngine.ts index 587ba38..0eec483 100644 --- a/src/main/engine/GitEngine.ts +++ b/src/main/engine/GitEngine.ts @@ -140,6 +140,14 @@ export interface GitPostFileChange { previousPath?: string; } +export type GitScriptFileChangeStatus = 'added' | 'modified' | 'deleted' | 'renamed'; + +export interface GitScriptFileChange { + status: GitScriptFileChangeStatus; + path: string; + previousPath?: string; +} + type GitProvider = 'unknown' | 'github' | 'gitlab' | 'gitea-forgejo'; let gitEngineInstance: GitEngine | null = null; @@ -526,7 +534,12 @@ export class GitEngine { return this.markdownExtensions.has(extension); } - private parseNameStatusOutput(raw: string): GitPostFileChange[] { + private isScriptsPythonPath(value: string): boolean { + const normalized = this.normalizeRepoRelativePath(value); + return normalized.startsWith('scripts/') && path.extname(normalized).toLowerCase() === '.py'; + } + + private parseNameStatusOutput(raw: string, pathMatcher: (value: string) => boolean): GitPostFileChange[] { const tokens = raw.split('\0').filter((token) => token.length > 0); const changes: GitPostFileChange[] = []; @@ -543,7 +556,7 @@ export class GitEngine { const previousPath = this.normalizeRepoRelativePath(previousPathRaw); const pathValue = this.normalizeRepoRelativePath(nextPathRaw); - if (this.isPostsMarkdownPath(previousPath) || this.isPostsMarkdownPath(pathValue)) { + if (pathMatcher(previousPath) || pathMatcher(pathValue)) { changes.push({ status: 'renamed', path: pathValue, @@ -555,7 +568,7 @@ export class GitEngine { const filePathRaw = tokens[index++] ?? ''; const filePath = this.normalizeRepoRelativePath(filePathRaw); - if (!this.isPostsMarkdownPath(filePath)) { + if (!pathMatcher(filePath)) { continue; } @@ -1338,13 +1351,40 @@ export class GitEngine { try { const output = await git.raw(args); - return this.parseNameStatusOutput(output); + return this.parseNameStatusOutput(output, (value) => this.isPostsMarkdownPath(value)); } catch (error) { const message = error instanceof Error ? error.message : String(error ?? ''); if (this.isSpawnBadFileDescriptorError(message)) { try { const output = await this.runGitCli(projectPath, args); - return this.parseNameStatusOutput(output); + return this.parseNameStatusOutput(output, (value) => this.isPostsMarkdownPath(value)); + } catch { + return []; + } + } + return []; + } + } + + async getChangedScriptFilesBetween(projectPath: string, fromCommit: string, toCommit: string): Promise { + const fromRef = fromCommit.trim(); + const toRef = toCommit.trim(); + if (!fromRef || !toRef || fromRef === toRef) { + return []; + } + + const git = this.createNonInteractiveGit(projectPath); + const args = ['diff', '--name-status', '--find-renames', '-z', `${fromRef}..${toRef}`, '--', 'scripts']; + + try { + const output = await git.raw(args); + return this.parseNameStatusOutput(output, (value) => this.isScriptsPythonPath(value)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error ?? ''); + if (this.isSpawnBadFileDescriptorError(message)) { + try { + const output = await this.runGitCli(projectPath, args); + return this.parseNameStatusOutput(output, (value) => this.isScriptsPythonPath(value)); } catch { return []; } diff --git a/src/main/engine/ScriptEngine.ts b/src/main/engine/ScriptEngine.ts index ed9b379..c31f2b4 100644 --- a/src/main/engine/ScriptEngine.ts +++ b/src/main/engine/ScriptEngine.ts @@ -42,6 +42,37 @@ export interface UpdateScriptInput { enabled?: boolean; } +export type GitScriptFileChangeStatus = 'added' | 'modified' | 'deleted' | 'renamed'; + +export interface GitScriptFileChange { + status: GitScriptFileChangeStatus; + path: string; + previousPath?: string; +} + +export interface ScriptReconcileResult { + created: number; + updated: number; + deleted: number; + processedFiles: number; +} + +interface ParsedScriptFile { + metadata: { + id?: string; + projectId?: string; + slug?: string; + title?: string; + kind?: string; + entrypoint?: string; + enabled?: boolean; + version?: number; + createdAt?: string; + updatedAt?: string; + }; + body: string; +} + export class ScriptEngine extends EventEmitter { private currentProjectId = 'default'; private dataDir: string | null = null; @@ -191,6 +222,205 @@ export class ScriptEngine extends EventEmitter { return Promise.all(rows.map((item) => this.toScriptData(item))); } + async rebuildDatabaseFromFiles(): Promise { + const db = getDatabase().getLocal(); + const scriptsDir = this.getScriptsDir(); + + await db.delete(scripts).where(eq(scripts.projectId, this.currentProjectId)); + + const pythonFiles = await this.scanScriptFiles(scriptsDir); + if (pythonFiles.length === 0) { + this.emit('scriptsRebuilt'); + return; + } + + const usedIds = new Set(); + const insertedRows: Script[] = []; + + for (const filePath of pythonFiles) { + const parsed = await this.readScriptFileWithMetadata(filePath); + if (!parsed) { + continue; + } + + const desiredSlug = this.normalizeSlug(parsed.metadata.slug || path.basename(filePath, '.py')); + const slug = this.ensureUniqueSlug(desiredSlug, insertedRows); + + const desiredId = typeof parsed.metadata.id === 'string' && parsed.metadata.id.trim().length > 0 + ? parsed.metadata.id.trim() + : uuidv4(); + const id = usedIds.has(desiredId) ? uuidv4() : desiredId; + + const now = new Date(); + const row: NewScript = { + id, + projectId: this.currentProjectId, + slug, + title: this.normalizeTitle(parsed.metadata.title, slug), + kind: this.normalizeKind(parsed.metadata.kind), + entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint), + enabled: this.normalizeEnabled(parsed.metadata.enabled), + version: this.normalizeVersion(parsed.metadata.version), + filePath, + createdAt: this.normalizeDate(parsed.metadata.createdAt, now), + updatedAt: this.normalizeDate(parsed.metadata.updatedAt, now), + }; + + await db.insert(scripts).values(row); + insertedRows.push(row as Script); + usedIds.add(id); + } + + this.emit('scriptsRebuilt'); + } + + async reconcileScriptsFromGitChanges(projectPath: string, changes: GitScriptFileChange[]): Promise { + const db = getDatabase().getLocal(); + const normalizedProjectPath = path.resolve(projectPath); + + const relevantChanges = changes.filter((change) => { + if (!this.isPythonScriptPath(change.path)) { + return false; + } + if (change.status === 'renamed' && change.previousPath && !this.isPythonScriptPath(change.previousPath) && !this.isPythonScriptPath(change.path)) { + return false; + } + return true; + }); + + if (relevantChanges.length === 0) { + return { created: 0, updated: 0, deleted: 0, processedFiles: 0 }; + } + + const scriptRows = await this.getAllScriptRows(); + const scriptsByPath = new Map(); + for (const row of scriptRows) { + scriptsByPath.set(this.normalizePathForCompare(row.filePath), row); + } + + let created = 0; + let updated = 0; + let deleted = 0; + let processedFiles = 0; + + for (const change of relevantChanges) { + const absolutePath = this.normalizePathForCompare(path.resolve(normalizedProjectPath, change.path)); + const previousAbsolutePath = change.previousPath + ? this.normalizePathForCompare(path.resolve(normalizedProjectPath, change.previousPath)) + : null; + + if (change.status === 'deleted') { + const existing = scriptsByPath.get(absolutePath); + if (!existing) { + continue; + } + + await db.delete(scripts).where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + scriptsByPath.delete(absolutePath); + this.emit('scriptDeleted', existing.id); + deleted += 1; + processedFiles += 1; + continue; + } + + let existing = previousAbsolutePath + ? (scriptsByPath.get(previousAbsolutePath) || scriptsByPath.get(absolutePath)) + : scriptsByPath.get(absolutePath); + + const parsed = await this.readScriptFileWithMetadata(absolutePath); + if (!parsed) { + continue; + } + + const allRows = await this.getAllScriptRows(); + const parsedId = typeof parsed.metadata.id === 'string' ? parsed.metadata.id.trim() : ''; + if (!existing && parsedId.length > 0) { + const byId = allRows.find((row) => row.id === parsedId); + if (byId) { + existing = byId; + } + } + const desiredSlug = this.normalizeSlug(parsed.metadata.slug || path.basename(absolutePath, '.py')); + const slug = this.ensureUniqueSlug(desiredSlug, allRows, existing?.id); + + if (existing) { + const updateNow = new Date(); + const nextRow = { + title: this.normalizeTitle(parsed.metadata.title, slug, existing.title), + slug, + kind: this.normalizeKind(parsed.metadata.kind, existing.kind), + entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint, existing.entrypoint), + enabled: this.normalizeEnabled(parsed.metadata.enabled, existing.enabled), + version: this.normalizeVersion(parsed.metadata.version, existing.version), + filePath: absolutePath, + createdAt: this.normalizeDate(parsed.metadata.createdAt, existing.createdAt), + updatedAt: this.normalizeDate(parsed.metadata.updatedAt, updateNow), + }; + + await db.update(scripts) + .set(nextRow) + .where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + + const updatedRow = await this.getScriptRow(existing.id); + if (updatedRow) { + const updatedScript = await this.toScriptData(updatedRow); + this.emit('scriptUpdated', updatedScript); + } + + if (previousAbsolutePath) { + scriptsByPath.delete(previousAbsolutePath); + } + scriptsByPath.set(absolutePath, { + ...existing, + ...nextRow, + }); + updated += 1; + processedFiles += 1; + continue; + } + + const desiredId = typeof parsed.metadata.id === 'string' && parsed.metadata.id.trim().length > 0 + ? parsed.metadata.id.trim() + : uuidv4(); + const idExists = allRows.some((row) => row.id === desiredId); + const rowId = idExists ? uuidv4() : desiredId; + const now = new Date(); + + const newRow: NewScript = { + id: rowId, + projectId: this.currentProjectId, + slug, + title: this.normalizeTitle(parsed.metadata.title, slug), + kind: this.normalizeKind(parsed.metadata.kind), + entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint), + enabled: this.normalizeEnabled(parsed.metadata.enabled), + version: this.normalizeVersion(parsed.metadata.version), + filePath: absolutePath, + createdAt: this.normalizeDate(parsed.metadata.createdAt, now), + updatedAt: this.normalizeDate(parsed.metadata.updatedAt, now), + }; + + await db.insert(scripts).values(newRow); + + const createdRow = await this.getScriptRow(newRow.id); + if (createdRow) { + const createdScript = await this.toScriptData(createdRow); + this.emit('scriptCreated', createdScript); + } + + scriptsByPath.set(absolutePath, newRow as Script); + created += 1; + processedFiles += 1; + } + + return { + created, + updated, + deleted, + processedFiles, + }; + } + private async getScriptRow(id: string): Promise