diff --git a/.vscode/settings.json b/.vscode/settings.json index 47a447a..6cadd54 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "chat.tools.terminal.autoApprove": { "npx vitest": true, "npx tsc": true, - "git remote": true + "git remote": true, + "npx asar": true } } \ No newline at end of file diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index c245c1d..4ffde4e 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -10,6 +10,7 @@ - [Working with pages](#working-with-pages) - [Working with media](#working-with-media) - [Using macros](#using-macros) +- [Using scripting (early access)](#using-scripting-early-access) - [Organizing with tags](#organizing-with-tags) - [Importing from WordPress (WXR)](#importing-from-wordpress-wxr) - [Using Git (Source Control)](#using-git-source-control) @@ -150,41 +151,37 @@ Macros let you insert dynamic content blocks directly inside post/page Markdown Use macros when you need reusable rich blocks (for example embedded videos, media galleries, archive grids, or computed tag clouds) without writing raw HTML. -### Supported macros +### YouTube macro -- `[[youtube id="VIDEO_ID" title="Optional title"]]` - - Embeds a YouTube video. - - `id` is required. - - `title` is optional (used for accessibility label). +Use `[[youtube id="VIDEO_ID" title="Optional title"]]` when you want to embed a YouTube clip directly in a post or page. This macro is best for video references, walkthroughs, and embedded talks that should stay in the editorial flow instead of linking out to another tab. -- `[[vimeo id="VIDEO_ID" title="Optional title"]]` - - Embeds a Vimeo video. - - `id` is required. - - `title` is optional. +The `id` parameter is required and should contain the YouTube video ID. The `title` parameter is optional, but recommended for accessibility because it becomes the reader-facing label for the embedded frame. -- `[[gallery columns="3" caption="Optional caption"]]` - - Renders a lightbox-enabled gallery using media linked to the current post. - - `columns` is optional (`1` to `6`, default `3`). - - `caption` is optional. +### Vimeo macro -- `[[photo_archive year="2025" month="2"]]` - - Renders a photo archive grid from media dates. - - `year` is optional (when omitted, recent months are shown). - - `month` is optional (used with `year` for a single month view). - - Legacy alias `[[photo_album ...]]` is also supported. +Use `[[vimeo id="VIDEO_ID" title="Optional title"]]` for Vimeo-hosted video content. It behaves similarly to the YouTube macro, but targets Vimeo as the video source. -- `[[tag_cloud orientation="mixed_diagonal" width="900" height="420"]]` - - Builds a word cloud from published tag usage counts. - - Word size scales by usage quantity. - - Word color is theme-aware and uses Pico CSS semantic colors. - - Colors are distributed by quantity quantiles with easing, so dense datasets still show visible variation. - - Visual order remains least-to-most: blue → green → yellow → orange → red. - - Clicking a word opens that tag archive route. - - `orientation` is optional and supports: - - `horizontal` (all words horizontal) - - `mixed_hv` (mix of horizontal and vertical) - - `mixed_diagonal` (mix of horizontal/vertical/diagonal angles) - - `width` and `height` are optional (defaults `900` and `420`). +As with YouTube, `id` is required and `title` is optional. Use `title` whenever possible so screen-reader and assistive-technology users receive useful context. + +### Gallery macro + +Use `[[gallery columns="3" caption="Optional caption"]]` to render a lightbox-enabled media gallery from assets linked to the current post. This macro is appropriate when several related images belong together and should be browsed as one visual group. + +The `columns` parameter controls layout density and accepts values from `1` to `6` (default is `3`). The optional `caption` parameter adds context above or below the gallery depending on theme presentation. + +### Photo archive macro + +Use `[[photo_archive year="2025" month="2"]]` when you want an archive-style grid based on media dates. This is useful for timeline-oriented projects where readers should navigate image collections by month or year. + +Both `year` and `month` are optional. If `year` is omitted, bDS shows recent months. If `year` is provided without `month`, bDS presents the year scope. The legacy alias `[[photo_album ...]]` is still supported for compatibility. + +### Tag cloud macro + +Use `[[tag_cloud orientation="mixed_diagonal" width="900" height="420"]]` to visualize published tag usage as a weighted cloud. This macro is best for discovery pages, thematic overviews, and archive entry points where content density matters. + +Word size scales with usage counts. Colors are theme-aware and distributed by quantity quantiles using eased interpolation so high-volume datasets stay readable. The color progression remains least-to-most (blue → green → yellow → orange → red), and clicking a word opens that tag archive route. + +The optional `orientation` parameter supports `horizontal`, `mixed_hv`, and `mixed_diagonal`. The optional `width` and `height` parameters control canvas size and default to `900` and `420`. ### Key takeaways @@ -196,6 +193,68 @@ Use macros when you need reusable rich blocks (for example embedded videos, medi --- +## Using scripting (early access) + +The scripting feature is an incremental capability and should currently be treated as early access. Scripts are stored as Python files in the project filesystem, while script metadata is tracked in the project database and embedded in the file metadata docstring block. This keeps scripts portable and inspectable while still allowing reliable indexing in the app. + +Each script exposes an **Entrypoint** selector. bDS always provides a synthetic `main` entrypoint. Selecting `main` runs the full script body as before. In addition, bDS inspects your script to list top-level Python function names, which can be selected as entrypoints for upcoming execution modes and integrations. + +At this stage, scripting is intended for controlled project workflows where scripts interact with application-provided tools. Keep scripts versioned through your normal Git workflow, review changes carefully, and prefer small, explicit scripts over monolithic utility files. + +For transform scripts, bDS provides a built-in Python helper named `toast(message)`. It accepts a single string and emits a UI intent that the app handles on the renderer side. This keeps script ergonomics simple while preserving a controlled bridge between script runtime and user interface. + +When transform scripts fail during a pipeline run, bDS automatically surfaces an error toast so users are notified immediately. Detailed transform diagnostics (applied scripts and per-script errors) are also written to the Output panel. + +### Example transform script + +Use a transform function to modify incoming bookmark/blogmark content before bDS creates the post. The function receives a mutable `post` dictionary and should return that dictionary. + +```python +def normalize_blogmark(post): + # 1) Manipulate title + title = (post.get("title") or "").strip() + if title and not title.startswith("[Clipped]"): + post["title"] = f"[Clipped] {title}" + + # 2) Manipulate text/content + content = (post.get("content") or "").strip() + prefix = "Imported from blogmark\n\n" + if content and not content.startswith(prefix): + post["content"] = prefix + content + + # 3) Set or replace categories + post["categories"] = ["Inbox", "Research"] + + # 4) Add and normalize tags + tags = post.get("tags") or [] + tags.append("blogmark") + tags.append("clipped") + post["tags"] = sorted({str(tag).strip().lower() for tag in tags if str(tag).strip()}) + + # 5) Optional user notification + toast(f"Transform applied: {post.get('title')}") + return post +``` + +Notes: +- `title` and `content` are strings. +- `categories` and `tags` are string lists (e.g., `['News', 'AI']`). +- Return the mutated `post` dict from your transform function. +- Keep transforms small and deterministic, especially when multiple active transforms run in sequence. + +### Key takeaways + +- Scripting is available and intentionally evolving in small steps. +- `main` is always available and preserves whole-script execution behavior. +- Script files and metadata remain filesystem-friendly and Git-reviewable. +- Transform scripts can call `toast("...")` to send user-facing UI notifications. +- Transform scripts can directly manipulate `title`, `content`, `categories`, and `tags`. +- Transform pipeline failures always trigger automatic error toasts. + +[↑ Back to In this article](#in-this-article) + +--- + ## Organizing with tags Tags are your precision taxonomy tool. Over time, even well-managed projects accumulate near-duplicate tags, naming inconsistencies, and labels that no longer serve users. The Tags section exists to keep taxonomy useful and prevent search and filtering quality from degrading. diff --git a/PYTHON_SCRIPTING.md b/PYTHON_SCRIPTING.md index ce741c1..b89d445 100644 --- a/PYTHON_SCRIPTING.md +++ b/PYTHON_SCRIPTING.md @@ -1,455 +1,94 @@ -# Python Scripting Integration Plan (Electron + Pyodide) +# Python Scripting — Remaining Work (Implementation-First) -## 1. Goal and Scope +Last verified: 23 Feb 2026 -Primary goal: all render-time macros run in Python, with predictable performance and safe sandboxing. +This document is intentionally reduced to **what is still left to implement**. +When plan and code differ, code is the source of truth. -Secondary goals: -- User-editable Python scripts with project persistence (`scripts/` folder + DB index). -- Python scripting reuse in bookmarklet/post-processing pipelines. -- Keep architecture consistent with existing `main/engine` + `ipc` + `renderer` boundaries. +## Implemented (checked) -This document defines a staged path from MVP to full scope. +- [x] Pyodide dependency integrated. +- [x] Renderer worker runtime exists (`pythonRuntime.worker.ts`) with ready/error/stdout/run protocol. +- [x] Runtime timeout watchdog + reset/recovery implemented in `PythonRuntimeManager`. +- [x] ABI v1 schemas and validation for macro context/result implemented (`abiV1.ts`). +- [x] Benchmark harness implemented (`npm run bench:python-runtime -- `). +- [x] Script persistence model implemented (`scripts` DB table + `scripts/*.py` files). +- [x] Script metadata model implemented (`kind`, `entrypoint`, `enabled`, `version`, etc.). +- [x] Main process script CRUD engine + IPC handlers implemented. +- [x] Preload + shared API typings for scripts implemented. +- [x] Renderer scripts UX implemented (sidebar list, editor, save, run, delete). +- [x] Script syntax check + entrypoint discovery integrated in editor UX. +- [x] Blogmark transform pipeline executes Python transform scripts (`kind='transform'`). ---- +## Confirmed Deviations from Original Plan -## 2. Viability Summary (Realistic Expectations) +These are current realities and should be treated as authoritative unless we explicitly decide to change them. -### Is this realistic? -Yes, if we optimize for **low bridge overhead** and **stable execution contracts**. +1. **Transform script runtime location differs** + - Original plan: untrusted Python runs in renderer worker only. + - Actual implementation: Blogmark transform scripts run in **main process** Pyodide (`BlogmarkTransformService`). -Key reality checks: -- Pyodide in a worker is viable for user scripting and sandboxing. -- Macro execution in render loops can be fast enough if we avoid frequent JS↔Python conversions. -- `< 1ms` per macro call is possible only for simple macros with precompiled code and minimal marshaling; it must be treated as a benchmark target, not a guarantee. -- For heavy loops, the work should stay inside Python once called (coarse-grained calls), not bounce per item between JS and Python. +2. **Render-time macro migration has not happened yet** + - Original plan: all render-time macros become Python-backed. + - Actual implementation: render-time macros are still JS-based (`PageRenderer.renderMacro` and renderer macro registry/definitions). -Decision: -- Keep Pyodide as the default engine. -- Design a strict, minimal ABI (Application Binary Interface-like contract) for macro inputs/outputs. -- Use preloading, precompilation, and caching before adding advanced optimizations. +3. **Macro ABI exists but is not the production render path** + - ABI v1 + runtime manager support exist. + - Main page generation path still uses existing JS macro rendering. ---- +4. **Scripts rebuild/meta-diff sync is still missing** + - Script CRUD works via app APIs. + - No implemented project-wide “rebuild from files” parity for `scripts/` equivalent to posts/media rebuild flows. -## 3. Architecture Fit for bDS +## Remaining Work Only -### 3.1 Layering +## 1) Decide and enforce Python runtime boundary (P0) -Keep existing project boundaries: -- `src/main/engine`: script metadata, script storage/indexing, render orchestration, benchmark/logging. -- `src/main/ipc`: typed handlers for script CRUD, run, and diagnostics. -- `src/renderer`: script editor UI, run controls, output panel integration. -- Python runtime (Pyodide) stays in renderer-side worker context; main process never executes untrusted script code. +- [ ] Decide if `transform` scripts should stay in main process or move to renderer worker. +- [ ] If staying in main process: add explicit timeout/kill/recovery safeguards equivalent to worker watchdog behavior. +- [ ] If moving to worker: route transform execution through typed IPC/worker bridge and remove main-process execution path. +- [ ] Document final security model in this file after decision. -### 3.2 Runtime Placement +## 2) Add scripts file-system rebuild/sync (P1) -- Host a long-lived `PythonRuntimeWorker` in renderer. -- Initialize Pyodide once per app session (or lazily on first script run). -- Maintain an in-memory registry of loaded scripts and compiled callables. +- [ ] Implement rebuild/meta-diff style synchronization for `scripts/` so external file edits are detected. +- [ ] Define conflict handling policy between DB metadata and script file frontmatter/body. +- [ ] Add tests for create/edit/delete performed outside app while app is closed/open. -### 3.3 Macro Execution Contract (Performance-Critical) +## 3) Wire Python macros into render pipeline (P1) -Use one narrow contract for all macros: +- [ ] Add macro-to-script resolution (token/hook -> script id/slug). +- [ ] Execute Python macro scripts from the active render path. +- [ ] Reuse runtime cache keys across repeated macro invocations in generation loops. +- [ ] Add guardrails for timeout/error fallback during render. -Python side: -```python -def render(context: dict) -> dict: - ... -``` +## 4) Macro parity migration + cleanup (P2) -Contract rules: -- Input `context` is plain JSON-compatible data only. -- Output is plain JSON-compatible data only. -- No Node/Electron direct access from Python. -- No per-token/per-node callbacks into JS while rendering. +- [ ] Port built-in macros to Python scripts with parity fixtures. +- [ ] Keep fixtures/golden tests for parity verification. +- [ ] Remove legacy JS macro path only after parity is proven. -### 3.4 Bridge Strategy (Keep Conversions Simple) +## 5) Default script seeding/versioning (P2) -- Preferred: pass compact JSON payloads (single call in, single result out). -- Avoid dynamic proxy-style JS objects in hot paths. -- Avoid `toPy()/toJs()` inside tight loops. -- Use `pyodide.globals` only for stable utility bindings set once during worker startup. +- [ ] Bundle default scripts in repo/app resources. +- [ ] Seed missing defaults on project init/startup deterministically. +- [ ] Add versioned update strategy for default scripts. -### 3.5 Security Model +## 6) Diagnostics and performance visibility (P3) -- Scripts execute only in worker. -- Hard timeout + termination + runtime restart on runaway scripts. -- Allowlist API surface exposed to Python (pure functions where possible). -- Validate and sanitize all script outputs in JS before applying to render pipeline. +- [ ] Add macro execution counters (count, timeout/error counts, p50/p95) for real render path. +- [ ] Define regression thresholds based on benchmark trends. ---- +## Out of Scope Until Core Gaps Close -## 4. Staged Delivery Plan +- [ ] AI assistant tooling exposure from Python scripts. +- [ ] General Python package/dependency policy expansion. +- [ ] Advanced bridge optimizations (only if metrics prove need). -## Phase 0 — Technical Spike (timeboxed) +## Acceptance Gate Before Marking Python Scripting “Complete” -Objective: prove runtime viability before product surface growth. - -Deliverables: -- [ ] Add `pyodide` dependency and worker boot sequence. -- [ ] Run a sample script end-to-end (`run_script`, timeout, captured stdout). -- [ ] Benchmark baseline cold start + warm run + repeated macro calls. -- [ ] Define initial macro ABI (`render(context) -> result`) and schema docs. - -Exit criteria: -- Warm script execution is stable. -- Timeout recovery works. -- Measured baseline captured in repo docs. - -## Phase 1 — MVP (Minimal but Usable) - -Objective: user can create/run scripts and see output. - -Deliverables: -- [ ] Script storage model (DB index + filesystem source in `scripts/*.py`). -- [ ] CRUD APIs in `main/engine` + `ipc` handlers. -- [ ] Renderer scripts list + editor + run button. -- [ ] Console/output capture in existing bottom output area. -- [ ] Project rebuild picks up `scripts/` changes. - -Out of scope for MVP: -- Macro replacement. -- Bookmarklet integration. -- AI assistant tool access from Python. - -Exit criteria: -- Scripts can be created, persisted, run, and debugged. -- Script files round-trip correctly with filesystem. - -## Phase 2 — Macro Runtime Foundation - -Objective: integrate Python macros into renderer loop with low overhead. - -Deliverables: -- [ ] Add script type/metadata (`kind: macro | utility | transform`). -- [ ] Resolve macro references from content to script IDs. -- [ ] Implement macro runtime cache: module load once, callable reuse. -- [ ] Convert existing macro parameter parsing into typed context object once per macro invocation. -- [ ] Add perf counters (call count, p50/p95 runtime, timeout count). - -Exit criteria: -- Python macro path is feature-equivalent for at least 1–2 existing macros. -- Measured overhead acceptable against baseline. - -## Phase 3 — Macro Migration (Full Goal) - -Objective: all current built-in macros are Python-backed. - -Deliverables: -- [ ] Port each existing macro implementation to Python scripts. -- [ ] Keep default macro scripts versioned in repo and bundled with app. -- [ ] On startup/project init, seed missing default macro scripts into filesystem + DB. -- [ ] Add script-as-macro assignment in metadata and editor UX. -- [ ] Keep parameter typing rules explicit (`"123"` quoted string stays string; unquoted numerics map to int/float). - -Exit criteria: -- All built-in macros execute via Python runtime. -- Legacy JS macro path is removed after parity confirmation. - -## Phase 4 — Performance Hardening - -Objective: reach production-grade speed and stability for render loops. - -Deliverables: -- [ ] Precompile/load scripts once per worker lifecycle. -- [ ] Batch render APIs where beneficial (`render_many(contexts)`). -- [ ] Reduce marshaling size (compact context shape, no redundant fields). -- [ ] Optional SharedArrayBuffer experiments only if measured need justifies added complexity. -- [ ] Failure isolation and automatic runtime reset strategy. - -Exit criteria: -- Stable long-run benchmarks in CI/manual perf suite. -- No UI thread stalls during heavy generation. - -## Phase 5 — Bookmarklet/Post Transform Integration - -Objective: reuse Python runtime for post-ingest transformations. - -Deliverables: -- [ ] Hook script transforms into bookmarklet pipeline after data sanitization. -- [ ] Input: validated post object; output: transformed validated post object. -- [ ] Add transform-specific script type and error handling/reporting. - -Exit criteria: -- Transform scripts can safely modify incoming post content. -- Fallback behavior exists when transform fails. - -## Phase 6 — Advanced Capabilities (Optional) - -Objective: add power-user features only after core stability. - -Candidates: -- [ ] Python-accessible app tools (strict allowlist). -- [ ] AI assistant tooling from Python scripts. -- [ ] Script package/dependency policy for curated modules. - ---- - -## 5. Data and Storage Design - -- Source of truth for scripts follows existing pattern: filesystem + DB index. -- Files: `scripts/.py`. -- Metadata can be stored in: - - DB columns (preferred for indexing/query), and/or - - leading Python block comment for file portability. -- Rebuild/meta-diff must include `scripts/` exactly like posts/media flow. - -Recommended script metadata: -- `id`, `slug`, `title`, `kind`, `entrypoint`, `enabled`, `version`, `updatedAt`. - ---- - -## 6. Performance Plan (Macro-Critical) - -Principles: -- Coarse-grained calls: one macro invocation should do meaningful work in Python. -- Stable ABI: small, predictable context payload. -- Warm runtime reuse: no repeated Pyodide boot. -- Compile/load once, execute many. - -Initial target envelope (to validate in Phase 0/2): -- Warm invocation overhead target: low single-digit milliseconds for typical macros. -- p95 render stability target under large generation batches. -- Timeout and memory guardrails for pathological scripts. - -Note: The previous strict `<1ms` universal target is replaced by benchmark tiers by macro class (simple/medium/heavy), which is more realistic. - ---- - -## 7. Security and Reliability - -- No direct filesystem/network/process APIs in Python runtime. -- Worker watchdog timeout and hard-kill policy. -- Structured errors returned to UI and logs. -- Script output validation before use in rendering. -- Versioned default scripts to ensure deterministic behavior across app updates. - ---- - -## 8. Testing and Rollout Strategy - -- Unit tests for engine-level script registry, metadata, and macro resolution. -- Integration tests for worker protocol and timeout recovery. -- Golden tests to compare macro output parity before/after migration. -- Performance regression checks for macro hot paths. -- Feature flag for staged rollout before removing legacy macro path. - ---- - -## 9. Coding Agent Execution Pack - -This section makes the plan directly executable by coding agents. - -### 9.1 Working Rules for Agents - -- Work one phase at a time; do not start the next phase before exit criteria pass. -- Keep changes layered by architecture boundary (`main/engine`, `main/ipc`, `renderer`). -- For each task: write/adjust tests first where feasible, then implement minimal code. -- Keep runtime contract stable once introduced; changes require updating ABI docs and tests. -- Do not add broad API exposure from JS/Electron into Python; only allowlisted calls. - -### 9.2 Definition of Done (Per Phase) - -Each phase is done only if all are true: -- [ ] Deliverables implemented. -- [ ] Exit criteria verified. -- [ ] Relevant tests pass. -- [ ] Full test suite passes (`npm test`). -- [ ] Full build passes (`npm run build`). -- [ ] Plan document updated with decisions/benchmarks where applicable. - -### 9.3 Task Card Template (Use for Every Agent Task) - -```md -Task: -Scope: -Files expected to change: -Out of scope: -Acceptance checks: -Commands to run: -Notes/Risks: -``` - -### 9.4 Phase-by-Phase Agent Backlog (Suggested) - -#### Phase 0 backlog - -1. Runtime bootstrap spike -- Scope: add Pyodide dependency and worker startup path only. -- Files likely: `package.json`, new worker file under `src/renderer/`. -- Acceptance: worker initializes once, reports ready state. - -2. Safe execute protocol -- Scope: request/response protocol (`run`, `stdout`, `error`, `timeout`). -- Files likely: renderer runtime manager + worker + related types. -- Acceptance: sample script run succeeds; timeout kills and recovers runtime. - -3. Baseline benchmark harness -- Scope: cold start, warm run, repeated macro invoke metrics. -- Files likely: engine/diagnostic service or dedicated benchmark utility + docs. -- Acceptance: numbers recorded in this document or linked benchmark doc. - -4. ABI v1 spec -- Scope: formal JSON schema for macro `context` and `result`. -- Files likely: shared type definitions + docs. -- Acceptance: schema used by both caller and worker-side validator. - -#### Phase 1 backlog - -1. Script persistence model -- Scope: DB + filesystem mapping for `scripts/*.py`. -- Acceptance: create/update/delete round-trips both stores. - -2. Main engine + IPC CRUD -- Scope: add script engine methods and typed IPC handlers. -- Acceptance: renderer can list/read/write scripts through IPC only. - -3. Renderer MVP UI -- Scope: scripts list, editor panel, run button, output panel integration. -- Acceptance: user edits script, runs it, sees stdout/errors. - -4. Rebuild/meta-diff integration -- Scope: include scripts in existing rebuild and metadata diff flow. -- Acceptance: external file changes in `scripts/` are detected and synchronized. - -#### Phase 2 backlog - -1. Macro script typing + mapping -- Scope: `kind` metadata and mapping from macro token to script id. -- Acceptance: at least one macro resolved to Python script. - -2. Runtime cache path -- Scope: load/compile once; callable reuse. -- Acceptance: repeated macro invocations avoid re-init/re-import. - -3. Context adapter -- Scope: convert existing macro params into ABI v1 `context` once per invocation. -- Acceptance: typed values obey conversion rules. - -4. Perf counters -- Scope: call count, p50/p95, timeout/error counts. -- Acceptance: counters visible in logs/diagnostics. - -#### Phase 3 backlog - -1. Built-in macro parity migration -- Scope: port each macro to Python scripts and add parity tests. -- Acceptance: output parity with legacy macros for baseline fixtures. - -2. Default script seeding/versioning -- Scope: bundle defaults, seed missing scripts on init. -- Acceptance: clean project bootstraps required macro scripts automatically. - -3. Legacy path removal -- Scope: remove JS macro implementations after parity gate. -- Acceptance: tests pass with Python-only macro path. - -#### Phase 4–6 backlog - -- Keep as optimization/integration tracks only after parity and stability gates pass. - -### 9.5 Anti-Patterns for Agents (Do Not Do) - -- Do not call JS functions per token/item from Python in hot paths. -- Do not pass large proxy objects through the bridge in render loops. -- Do not introduce direct filesystem/network access in Python runtime. -- Do not couple UI/editor work with macro migration in one PR-sized change. -- Do not remove legacy macro code before golden parity tests pass. - -### 9.6 Handoff Checklist (Agent to Agent) - -Every handoff should include: -- Completed task cards and remaining task cards. -- Files changed and rationale. -- Test/build command outputs summary. -- Known risks and benchmark deltas. -- Any ABI changes (must be explicit). - -### 9.7 Suggested PR Boundaries (One Task, One PR) - -Use small PRs with one primary purpose each. - -PR-00: Pyodide bootstrap spike -- Includes: dependency, worker init, ready signal. -- Excludes: script persistence, UI/editor. -- Merge gate: runtime initializes and tests/build pass. - -PR-01: Worker run protocol + timeout recovery -- Includes: run/stdout/error/timeout messaging, watchdog + restart behavior. -- Excludes: macro integration. -- Merge gate: timeout test and recovery test pass. - -PR-02: ABI v1 types + schema validation -- Includes: shared types and validation for `context/result`. -- Excludes: macro migration. -- Merge gate: caller and worker both use ABI validators. - -PR-03: Script persistence model -- Includes: DB + filesystem model for `scripts/*.py`. -- Excludes: renderer UI. -- Merge gate: round-trip persistence tests pass. - -PR-04: Script engine + IPC CRUD -- Includes: `main/engine` methods and typed `ipc` handlers. -- Excludes: macro runtime. -- Merge gate: IPC integration tests pass. - -PR-05: Renderer MVP scripts UI -- Includes: scripts list/editor/run/output integration. -- Excludes: macro substitution. -- Merge gate: end-to-end manual run path works + tests/build pass. - -PR-06: Rebuild/meta-diff integration -- Includes: include `scripts/` in rebuild and metadata diff paths. -- Excludes: macro migration. -- Merge gate: external script file changes are detected and synchronized. - -PR-07: Macro mapping + runtime cache foundation -- Includes: macro-to-script mapping, callable cache, first Python-backed macro. -- Excludes: full macro parity. -- Merge gate: at least one macro parity fixture passes. - -PR-08: Macro parity migration batch A -- Includes: port a small set of built-in macros (e.g., 2–3) + golden tests. -- Excludes: removal of legacy path. -- Merge gate: parity fixtures pass for migrated macros. - -PR-09: Macro parity migration batch B (repeat as needed) -- Includes: additional macro ports + fixtures. -- Excludes: removal of legacy path. -- Merge gate: all targeted macro parity tests pass. - -PR-10: Default script seeding/versioning -- Includes: bundled default scripts + startup seeding behavior. -- Excludes: advanced scripting APIs. -- Merge gate: clean project gets default scripts deterministically. - -PR-11: Legacy JS macro path removal -- Includes: delete legacy macro implementations after full parity. -- Excludes: bookmarklet transforms. -- Merge gate: full test suite and render parity suite pass. - -PR-12: Performance hardening -- Includes: benchmark harness refinements, caching improvements, optional batch APIs. -- Excludes: unrelated UI changes. -- Merge gate: regression thresholds (p50/p95) stay within agreed envelope. - -PR-13: Bookmarklet transform integration -- Includes: transform script type, pipeline hook, validation/fallback. -- Excludes: optional advanced tool APIs. -- Merge gate: sanitized input/output transform tests pass. - -PR-14+: Optional advanced capabilities -- Includes: allowlisted app tools, AI-assistant script tools, curated package policy. -- Merge gate: explicit security review and feature-flag rollout. - ---- - -## 10. Current Status - -Status: Revised staged plan (MVP-first, full-scope preserved). - -Recommended next action: -1. Approve Phase 0 scope and benchmarks. -2. Implement spike and record numbers. -3. Lock ABI before building full UI and migration layers. +- [ ] Render-time macros run through Python script path in production generation flow. +- [ ] Scripts directory external changes are synchronized reliably. +- [ ] Runtime boundary decision implemented and protected by tests. +- [ ] Legacy JS macro path removed (or explicitly retained with documented rationale). +- [ ] `npm test` and `npm run build` pass. diff --git a/drizzle/0005_short_sally_floyd.sql b/drizzle/0005_short_sally_floyd.sql new file mode 100644 index 0000000..c1c1a45 --- /dev/null +++ b/drizzle/0005_short_sally_floyd.sql @@ -0,0 +1,15 @@ +CREATE TABLE `scripts` ( + `id` text PRIMARY KEY NOT NULL, + `project_id` text NOT NULL, + `slug` text NOT NULL, + `title` text NOT NULL, + `kind` text DEFAULT 'utility' NOT NULL, + `entrypoint` text DEFAULT 'render' NOT NULL, + `enabled` integer DEFAULT true NOT NULL, + `version` integer DEFAULT 1 NOT NULL, + `file_path` text NOT NULL, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `scripts_project_slug_idx` ON `scripts` (`project_id`,`slug`); \ No newline at end of file diff --git a/drizzle/meta/0005_snapshot.json b/drizzle/meta/0005_snapshot.json new file mode 100644 index 0000000..5370a9e --- /dev/null +++ b/drizzle/meta/0005_snapshot.json @@ -0,0 +1,913 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b157a762-0743-4499-a635-16ac3fb5ee18", + "prevId": "46702982-9f8a-4c7e-8fb6-3270c3fbe120", + "tables": { + "chat_conversations": { + "name": "chat_conversations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_session_id": { + "name": "copilot_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "chat_messages": { + "name": "chat_messages", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "generated_file_hashes": { + "name": "generated_file_hashes", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "generated_file_hashes_project_path_idx": { + "name": "generated_file_hashes_project_path_idx", + "columns": [ + "project_id", + "relative_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "import_definitions": { + "name": "import_definitions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "wxr_file_path": { + "name": "wxr_file_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploads_folder_path": { + "name": "uploads_folder_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_analysis_result": { + "name": "last_analysis_result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "media": { + "name": "media", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "alt": { + "name": "alt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "caption": { + "name": "caption", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sidecar_path": { + "name": "sidecar_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_links": { + "name": "post_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_post_id": { + "name": "source_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_post_id": { + "name": "target_post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "link_text": { + "name": "link_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "post_media": { + "name": "post_media", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media_id": { + "name": "media_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "post_media_post_media_idx": { + "name": "post_media_post_media_idx", + "columns": [ + "post_id", + "media_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "posts": { + "name": "posts", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "excerpt": { + "name": "excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "categories": { + "name": "categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_title": { + "name": "published_title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_content": { + "name": "published_content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_tags": { + "name": "published_tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_categories": { + "name": "published_categories", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "published_excerpt": { + "name": "published_excerpt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "posts_project_slug_idx": { + "name": "posts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data_path": { + "name": "data_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scripts": { + "name": "scripts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'utility'" + }, + "entrypoint": { + "name": "entrypoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'render'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "scripts_project_slug_idx": { + "name": "scripts_project_slug_idx", + "columns": [ + "project_id", + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tags": { + "name": "tags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tags_project_name_idx": { + "name": "tags_project_name_idx", + "columns": [ + "project_id", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index bbc9190..5c1762b 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1771605253203, "tag": "0004_overjoyed_paper_doll", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1771792324840, + "tag": "0005_short_sally_floyd", + "breakpoints": true } ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index f0a31c4..f529c90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "liquidjs": "^10.24.0", "marked-react": "^3.0.2", "monaco-editor": "^0.55.1", + "pyodide": "^0.29.3", "react": "^19.2.4", "react-arborist": "^3.4.3", "react-dom": "^19.2.4", @@ -5263,6 +5264,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -12754,6 +12761,19 @@ "node": ">=6" } }, + "node_modules/pyodide": { + "version": "0.29.3", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.3.tgz", + "integrity": "sha512-22UBuhOJawj7vKUnS7/F3xK+515LJdjiMAHoCfuS6/PbHiOrSQVnYwDe+2sbVwiOZ3sMMexdXICew6NqOMQGgA==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", diff --git a/package.json b/package.json index 0e628d7..21a7eac 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:ui": "vitest --ui", + "bench:python-runtime": "node ./node_modules/tsx/dist/cli.mjs scripts/python-runtime-benchmark.ts", "lint": "eslint \"src/renderer/**/*.{ts,tsx}\" --max-warnings 0", "lint:i18n": "eslint \"src/renderer/**/*.{ts,tsx}\" --max-warnings 0", "db:generate": "node ./node_modules/drizzle-kit/bin.cjs generate", @@ -95,6 +96,7 @@ "liquidjs": "^10.24.0", "marked-react": "^3.0.2", "monaco-editor": "^0.55.1", + "pyodide": "^0.29.3", "react": "^19.2.4", "react-arborist": "^3.4.3", "react-dom": "^19.2.4", diff --git a/scripts/python-runtime-benchmark.ts b/scripts/python-runtime-benchmark.ts new file mode 100644 index 0000000..3c16882 --- /dev/null +++ b/scripts/python-runtime-benchmark.ts @@ -0,0 +1,28 @@ +import { runPythonRuntimeBenchmark } from '../src/renderer/python/pythonRuntimeBenchmark'; + +async function main(): Promise { + const iterationsArg = process.argv[2]; + const iterations = iterationsArg ? Number(iterationsArg) : 200; + + if (!Number.isInteger(iterations) || iterations <= 0) { + throw new Error('Iterations must be a positive integer'); + } + + const result = await runPythonRuntimeBenchmark({ iterations }); + + const output = { + measuredAt: new Date().toISOString(), + iterations, + coldStartMs: result.coldStartMs, + warmRunMs: result.warmRunMs, + repeatedMacro: result.repeatedMacro.stats, + }; + + console.log(JSON.stringify(output, null, 2)); +} + +void main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + console.error(`[python-runtime-benchmark] ${message}`); + process.exit(1); +}); diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index 5453e2e..be695c1 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -151,6 +151,24 @@ export const importDefinitions = sqliteTable('import_definitions', { updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(), }); +// Scripts table - stores metadata for Python scripts persisted in scripts/*.py +export const scripts = sqliteTable('scripts', { + id: text('id').primaryKey(), + projectId: text('project_id').notNull(), + slug: text('slug').notNull(), + title: text('title').notNull(), + kind: text('kind', { enum: ['macro', 'utility', 'transform'] }).notNull().default('utility'), + entrypoint: text('entrypoint').notNull().default('render'), + enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true), + version: integer('version').notNull().default(1), + filePath: text('file_path').notNull(), + createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), + updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(), +}, (table) => ({ + // Composite unique index: slug must be unique within each project + projectSlugIdx: uniqueIndex('scripts_project_slug_idx').on(table.projectId, table.slug), +})); + // Types for TypeScript export type Project = typeof projects.$inferSelect; export type NewProject = typeof projects.$inferInsert; @@ -174,3 +192,5 @@ export type ChatMessage = typeof chatMessages.$inferSelect; export type NewChatMessage = typeof chatMessages.$inferInsert; export type ImportDefinition = typeof importDefinitions.$inferSelect; export type NewImportDefinition = typeof importDefinitions.$inferInsert; +export type Script = typeof scripts.$inferSelect; +export type NewScript = typeof scripts.$inferInsert; diff --git a/src/main/engine/BlogmarkTransformService.ts b/src/main/engine/BlogmarkTransformService.ts new file mode 100644 index 0000000..7f65c66 --- /dev/null +++ b/src/main/engine/BlogmarkTransformService.ts @@ -0,0 +1,316 @@ +import { z } from 'zod'; +import { getScriptEngine } from './ScriptEngine'; + +const transformPostSchema = z.object({ + title: z.string().trim().min(1), + content: z.string().trim().min(1), + tags: z.array(z.string().trim().min(1)), + categories: z.array(z.string().trim().min(1)), +}); + +export type BlogmarkTransformedPost = z.infer; + +export interface BlogmarkTransformInput { + post: BlogmarkTransformedPost; + context: { + source: 'blogmark'; + url: string; + }; +} + +export interface BlogmarkTransformScriptRecord { + id: string; + slug: string; + title: string; + kind: 'macro' | 'utility' | 'transform'; + entrypoint: string; + enabled: boolean; + content: string; + updatedAt: Date | string; +} + +export interface BlogmarkTransformExecutor { + runTransform(script: BlogmarkTransformScriptRecord, input: BlogmarkTransformInput): Promise; +} + +export interface BlogmarkTransformScriptProvider { + getScripts(): Promise; +} + +export interface BlogmarkTransformError { + scriptId: string; + scriptSlug: string; + message: string; +} + +export interface BlogmarkTransformExecutionData { + output: unknown; + toasts: string[]; +} + +export interface BlogmarkTransformResult { + post: BlogmarkTransformedPost; + appliedScriptIds: string[]; + errors: BlogmarkTransformError[]; + toasts: string[]; +} + +const MAX_TOASTS_PER_SCRIPT = 5; +const MAX_TOASTS_TOTAL = 20; +const MAX_TOAST_LENGTH = 300; + +const scriptEngineBackedProvider: BlogmarkTransformScriptProvider = { + async getScripts() { + return getScriptEngine().getAllScripts(); + }, +}; + +function toTimestamp(value: Date | string): number { + if (value instanceof Date) { + return value.getTime(); + } + + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : 0; +} + +function normalizePost(value: unknown): BlogmarkTransformedPost | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return null; + } + + const valueRecord = value as Record; + const maybePost = valueRecord.post; + + const candidate = maybePost && typeof maybePost === 'object' && !Array.isArray(maybePost) + ? maybePost + : value; + + const parsed = transformPostSchema.safeParse(candidate); + if (!parsed.success) { + return null; + } + + return parsed.data; +} + +function normalizeToastMessage(value: unknown): string | null { + if (value === undefined || value === null) { + return null; + } + + const normalized = String(value).trim(); + if (normalized.length === 0) { + return null; + } + + return normalized.slice(0, MAX_TOAST_LENGTH); +} + +function toExecutionData(value: unknown): BlogmarkTransformExecutionData { + if (value && typeof value === 'object' && !Array.isArray(value)) { + const valueRecord = value as Record; + const toasts = Array.isArray(valueRecord.toasts) + ? valueRecord.toasts + .map((item) => normalizeToastMessage(item)) + .filter((item): item is string => item !== null) + : []; + + if (Object.prototype.hasOwnProperty.call(valueRecord, 'output')) { + return { + output: valueRecord.output, + toasts, + }; + } + + return { + output: value, + toasts, + }; + } + + return { + output: value, + toasts: [], + }; +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error && typeof error.message === 'string' && error.message.trim().length > 0) { + return error.message; + } + return String(error); +} + +class PythonBlogmarkTransformExecutor implements BlogmarkTransformExecutor { + private runtimePromise: Promise | null = null; + + async runTransform(script: BlogmarkTransformScriptRecord, input: BlogmarkTransformInput): Promise { + const runtime = await this.getRuntime(); + const toastMessages: string[] = []; + const pushToast = (message: unknown): void => { + if (toastMessages.length >= MAX_TOASTS_PER_SCRIPT) { + return; + } + + const normalizedMessage = normalizeToastMessage(message); + if (!normalizedMessage) { + return; + } + + toastMessages.push(normalizedMessage); + }; + + runtime.globals.set('__bds_push_toast', pushToast); + await runtime.runPythonAsync(` +def toast(message): + __bds_push_toast(str(message)) +`); + + await runtime.runPythonAsync(script.content); + + const requestedEntrypoint = this.resolveEntrypoint(script.entrypoint); + const payload = JSON.stringify(input); + runtime.globals.set('__bds_transform_payload_json', payload); + runtime.globals.set('__bds_transform_entrypoint', requestedEntrypoint); + + const rawResult = await runtime.runPythonAsync(` +import json +_payload = json.loads(__bds_transform_payload_json) +_entrypoint = __bds_transform_entrypoint +_transform_fn = globals().get(_entrypoint) +if _transform_fn is None or not callable(_transform_fn): + raise RuntimeError(f"Transform entrypoint '{_entrypoint}' is not callable") +_post = _payload.get("post") +if not isinstance(_post, dict): + raise RuntimeError("Transform payload is missing a valid 'post' object") +_context = _payload.get("context") +try: + _result = _transform_fn(_post, _context) +except TypeError: + _result = _transform_fn(_post) +if _result is None: + _result = _post +json.dumps(_result) +`); + + return { + output: JSON.parse(String(rawResult)), + toasts: toastMessages, + }; + } + + private resolveEntrypoint(value: string): string { + const nextEntrypoint = typeof value === 'string' ? value.trim() : ''; + if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(nextEntrypoint) && nextEntrypoint !== 'main') { + return nextEntrypoint; + } + + return 'transform'; + } + + private async getRuntime(): Promise { + if (!this.runtimePromise) { + this.runtimePromise = (async () => { + const pyodideModule = await import('pyodide'); + return pyodideModule.loadPyodide(); + })(); + } + + return this.runtimePromise; + } +} + +export class BlogmarkTransformService { + constructor( + private readonly dependencies: { + provider?: BlogmarkTransformScriptProvider; + executor?: BlogmarkTransformExecutor; + } = {}, + ) {} + + async applyTransforms(input: BlogmarkTransformInput): Promise { + const parsedInput = transformPostSchema.parse(input.post); + const transformInput: BlogmarkTransformInput = { + ...input, + post: parsedInput, + }; + + const provider = this.dependencies.provider ?? scriptEngineBackedProvider; + const executor = this.dependencies.executor ?? new PythonBlogmarkTransformExecutor(); + + const scripts = await provider.getScripts(); + const activeTransforms = scripts + .filter((script) => script.enabled && script.kind === 'transform') + .sort((left, right) => { + const byUpdatedAt = toTimestamp(left.updatedAt) - toTimestamp(right.updatedAt); + if (byUpdatedAt !== 0) { + return byUpdatedAt; + } + + const bySlug = left.slug.localeCompare(right.slug); + if (bySlug !== 0) { + return bySlug; + } + + return left.id.localeCompare(right.id); + }); + + let currentPost = transformInput.post; + const appliedScriptIds: string[] = []; + const errors: BlogmarkTransformError[] = []; + const toasts: string[] = []; + + for (const script of activeTransforms) { + try { + const execution = await executor.runTransform(script, { + ...transformInput, + post: currentPost, + }); + + const executionData = toExecutionData(execution); + const nextToasts = executionData.toasts + .map((message) => normalizeToastMessage(message)) + .filter((message): message is string => message !== null); + + if (nextToasts.length > 0 && toasts.length < MAX_TOASTS_TOTAL) { + const remaining = MAX_TOASTS_TOTAL - toasts.length; + toasts.push(...nextToasts.slice(0, remaining)); + } + + const normalizedPost = normalizePost(executionData.output); + if (!normalizedPost) { + throw new Error('Transform output validation failed'); + } + + currentPost = normalizedPost; + appliedScriptIds.push(script.id); + } catch (error) { + const message = toErrorMessage(error); + errors.push({ + scriptId: script.id, + scriptSlug: script.slug, + message, + }); + console.error(`[blogmark-transform] ${script.slug}: ${message}`); + } + } + + return { + post: currentPost, + appliedScriptIds, + errors, + toasts, + }; + } +} + +let blogmarkTransformServiceInstance: BlogmarkTransformService | null = null; + +export function getBlogmarkTransformService(): BlogmarkTransformService { + if (!blogmarkTransformServiceInstance) { + blogmarkTransformServiceInstance = new BlogmarkTransformService(); + } + + return blogmarkTransformServiceInstance; +} diff --git a/src/main/engine/ScriptEngine.ts b/src/main/engine/ScriptEngine.ts new file mode 100644 index 0000000..ed9b379 --- /dev/null +++ b/src/main/engine/ScriptEngine.ts @@ -0,0 +1,330 @@ +import { EventEmitter } from 'events'; +import { v4 as uuidv4 } from 'uuid'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { app } from 'electron'; +import { and, desc, eq } from 'drizzle-orm'; +import { getDatabase } from '../database'; +import { scripts, type NewScript, type Script } from '../database/schema'; + +export type ScriptKind = 'macro' | 'utility' | 'transform'; + +export interface ScriptData { + id: string; + projectId: string; + slug: string; + title: string; + kind: ScriptKind; + entrypoint: string; + enabled: boolean; + version: number; + filePath: string; + content: string; + createdAt: Date; + updatedAt: Date; +} + +export interface CreateScriptInput { + title: string; + kind: ScriptKind; + content: string; + slug?: string; + entrypoint?: string; + enabled?: boolean; +} + +export interface UpdateScriptInput { + title?: string; + kind?: ScriptKind; + content?: string; + slug?: string; + entrypoint?: string; + enabled?: boolean; +} + +export class ScriptEngine extends EventEmitter { + private currentProjectId = 'default'; + private dataDir: string | null = null; + + setProjectContext(projectId: string, dataDir?: string): void { + this.currentProjectId = projectId; + this.dataDir = dataDir || null; + } + + getProjectContext(): string { + return this.currentProjectId; + } + + async createScript(input: CreateScriptInput): Promise { + const now = new Date(); + const allScripts = await this.getAllScriptRows(); + const desiredSlug = this.normalizeSlug(input.slug || input.title || 'script'); + const uniqueSlug = this.ensureUniqueSlug(desiredSlug, allScripts); + const scriptId = uuidv4(); + const filePath = this.getScriptFilePath(uniqueSlug); + + const row: NewScript = { + id: scriptId, + projectId: this.currentProjectId, + slug: uniqueSlug, + title: input.title, + kind: input.kind, + entrypoint: input.entrypoint || 'render', + enabled: input.enabled ?? true, + version: 1, + filePath, + createdAt: now, + updatedAt: now, + }; + + await fs.mkdir(this.getScriptsDir(), { recursive: true }); + await fs.writeFile(filePath, this.serializeScriptFile(row as Script, input.content), 'utf-8'); + + await getDatabase().getLocal().insert(scripts).values(row); + + const created = await this.toScriptData(row as Script); + this.emit('scriptCreated', created); + return created; + } + + async updateScript(id: string, updates: UpdateScriptInput): Promise { + const existing = await this.getScriptRow(id); + if (!existing) { + return null; + } + + const allScripts = await this.getAllScriptRows(); + const desiredSlug = typeof updates.slug === 'string' + ? this.normalizeSlug(updates.slug) + : typeof updates.title === 'string' + ? this.normalizeSlug(updates.title) + : existing.slug; + const nextSlug = this.ensureUniqueSlug(desiredSlug, allScripts, existing.id); + const nextFilePath = this.getScriptFilePath(nextSlug); + const now = new Date(); + + if (existing.filePath !== nextFilePath) { + await fs.mkdir(this.getScriptsDir(), { recursive: true }); + await fs.rename(existing.filePath, nextFilePath); + } + + const nextTitle = updates.title ?? existing.title; + const nextKind = updates.kind ?? existing.kind; + const nextEntrypoint = updates.entrypoint ?? existing.entrypoint; + const nextEnabled = updates.enabled ?? existing.enabled; + const nextVersion = existing.version + 1; + const nextContent = typeof updates.content === 'string' + ? updates.content + : await this.readScriptBody(nextFilePath); + + const nextRow = { + ...existing, + title: nextTitle, + slug: nextSlug, + kind: nextKind, + entrypoint: nextEntrypoint, + enabled: nextEnabled, + filePath: nextFilePath, + version: nextVersion, + updatedAt: now, + }; + + await fs.writeFile(nextFilePath, this.serializeScriptFile(nextRow, nextContent), 'utf-8'); + + await getDatabase().getLocal() + .update(scripts) + .set({ + title: nextTitle, + slug: nextSlug, + kind: nextKind, + entrypoint: nextEntrypoint, + enabled: nextEnabled, + filePath: nextFilePath, + version: nextVersion, + updatedAt: now, + }) + .where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + + const updatedRow = await this.getScriptRow(existing.id); + if (!updatedRow) { + return null; + } + + const updated = await this.toScriptData(updatedRow); + this.emit('scriptUpdated', updated); + return updated; + } + + async deleteScript(id: string): Promise { + const existing = await this.getScriptRow(id); + if (!existing) { + return false; + } + + await getDatabase().getLocal() + .delete(scripts) + .where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId))); + + try { + await fs.unlink(existing.filePath); + } catch (error) { + const fsError = error as NodeJS.ErrnoException; + if (fsError.code !== 'ENOENT') { + throw error; + } + } + + this.emit('scriptDeleted', id); + return true; + } + + async getScript(id: string): Promise { + const row = await this.getScriptRow(id); + if (!row) { + return null; + } + return this.toScriptData(row); + } + + async getAllScripts(): Promise { + const rows = await this.getAllScriptRows(); + return Promise.all(rows.map((item) => this.toScriptData(item))); + } + + private async getScriptRow(id: string): Promise