feat(python): add queued worker runtime and configurable transform mode

This commit is contained in:
2026-02-23 22:26:54 +01:00
parent 8e8f099768
commit 838ea34ab7
21 changed files with 744 additions and 88 deletions

View File

@@ -10,6 +10,7 @@ When plan and code differ, code is the source of truth.
- [x] Pyodide dependency integrated.
- [x] Renderer worker runtime exists (`pythonRuntime.worker.ts`) with ready/error/stdout/run protocol.
- [x] Runtime timeout watchdog + reset/recovery implemented in `PythonRuntimeManager`.
- [x] Renderer runtime request queueing implemented (concurrent calls are serialized in manager).
- [x] ABI v1 schemas and validation for macro context/result implemented (`abiV1.ts`).
- [x] Benchmark harness implemented (`npm run bench:python-runtime -- <iterations>`).
- [x] Script persistence model implemented (`scripts` DB table + `scripts/*.py` files).
@@ -18,15 +19,16 @@ When plan and code differ, code is the source of truth.
- [x] Preload + shared API typings for scripts implemented.
- [x] Renderer scripts UX implemented (sidebar list, editor, save, run, delete).
- [x] Script syntax check + entrypoint discovery integrated in editor UX.
- [x] Blogmark transform pipeline executes Python transform scripts (`kind='transform'`).
- [x] Blogmark transform pipeline executes Python transform scripts (`kind='transform'`) via a queued worker runtime by default.
- [x] Project preference `pythonRuntimeMode` added with Settings → Technology section.
## Confirmed Deviations from Original Plan
These are current realities and should be treated as authoritative unless we explicitly decide to change them.
1. **Transform script runtime location differs**
- Original plan: untrusted Python runs in renderer worker only.
- Actual implementation: Blogmark transform scripts run in **main process** Pyodide (`BlogmarkTransformService`).
1. **Transform runtime is now configurable (project-level)**
- Default: `webworker` (worker-thread based Python runtime with queued requests).
- Optional fallback: `main-thread` legacy execution mode.
2. **Render-time macro migration has not happened yet**
- Original plan: all render-time macros become Python-backed.
@@ -42,12 +44,11 @@ These are current realities and should be treated as authoritative unless we exp
## Remaining Work Only
## 1) Decide and enforce Python runtime boundary (P0)
## 1) Python runtime boundary (P0) — Implemented
- [ ] Decide if `transform` scripts should stay in main process or move to renderer worker.
- [ ] If staying in main process: add explicit timeout/kill/recovery safeguards equivalent to worker watchdog behavior.
- [ ] If moving to worker: route transform execution through typed IPC/worker bridge and remove main-process execution path.
- [ ] Document final security model in this file after decision.
- [x] Worker model introduced for Blogmark transform execution with queued communication.
- [x] Runtime mode made project-configurable via Settings → Technology (`pythonRuntimeMode`).
- [x] Legacy main-thread mode retained as explicit fallback option.
## 2) Add scripts file-system rebuild/sync (P1)
@@ -89,6 +90,6 @@ These are current realities and should be treated as authoritative unless we exp
- [ ] Render-time macros run through Python script path in production generation flow.
- [ ] Scripts directory external changes are synchronized reliably.
- [ ] Runtime boundary decision implemented and protected by tests.
- [x] Runtime boundary decision implemented and protected by tests.
- [ ] Legacy JS macro path removed (or explicitly retained with documented rationale).
- [ ] `npm test` and `npm run build` pass.

View File

@@ -0,0 +1,261 @@
import * as path from 'path';
import { Worker } from 'worker_threads';
interface WorkerRunTransformRequest {
type: 'runTransform';
requestId: string;
scriptContent: string;
entrypoint: string;
payloadJson: string;
}
interface WorkerReadyMessage {
type: 'ready';
}
interface WorkerResultMessage {
type: 'transformResult';
requestId: string;
output: unknown;
toasts: string[];
}
interface WorkerErrorMessage {
type: 'transformError';
requestId: string;
error: string;
}
interface WorkerFatalErrorMessage {
type: 'error';
error: string;
}
type WorkerResponseMessage = WorkerReadyMessage | WorkerResultMessage | WorkerErrorMessage | WorkerFatalErrorMessage;
interface QueuedRequest {
request: WorkerRunTransformRequest;
timeoutMs: number;
resolve: (value: { output: unknown; toasts: string[] }) => void;
reject: (error: Error) => void;
}
interface ActiveRequest extends QueuedRequest {
timeoutId: ReturnType<typeof setTimeout> | null;
}
export class BlogmarkPythonWorkerRuntime {
private worker: Worker | null = null;
private workerReady = false;
private workerStartPromise: Promise<void> | null = null;
private workerStartResolve: (() => void) | null = null;
private workerStartReject: ((error: Error) => void) | null = null;
private activeRequest: ActiveRequest | null = null;
private queue: QueuedRequest[] = [];
private requestCounter = 0;
async executeTransform(params: {
scriptContent: string;
entrypoint: string;
payloadJson: string;
timeoutMs?: number;
}): Promise<{ output: unknown; toasts: string[] }> {
const requestId = this.nextRequestId();
const timeoutMs = params.timeoutMs ?? 5000;
return new Promise<{ output: unknown; toasts: string[] }>((resolve, reject) => {
this.queue.push({
request: {
type: 'runTransform',
requestId,
scriptContent: params.scriptContent,
entrypoint: params.entrypoint,
payloadJson: params.payloadJson,
},
timeoutMs,
resolve,
reject,
});
this.dispatchNext().catch((error) => {
reject(error instanceof Error ? error : new Error(String(error)));
});
});
}
dispose(): void {
this.rejectStartPromise(new Error('Python worker runtime disposed'));
this.rejectActiveAndQueue(new Error('Python worker runtime disposed'));
this.resetWorker();
}
private async dispatchNext(): Promise<void> {
if (this.activeRequest || this.queue.length === 0) {
return;
}
await this.ensureWorkerStarted();
const nextRequest = this.queue.shift();
if (!nextRequest) {
return;
}
const timeoutId = setTimeout(() => {
if (!this.activeRequest || this.activeRequest.request.requestId !== nextRequest.request.requestId) {
return;
}
const timeoutError = new Error(`Python transform timed out after ${nextRequest.timeoutMs}ms`);
this.activeRequest.reject(timeoutError);
this.activeRequest = null;
this.resetWorker();
void this.dispatchNext();
}, nextRequest.timeoutMs);
this.activeRequest = {
...nextRequest,
timeoutId,
};
this.worker?.postMessage(nextRequest.request);
}
private async ensureWorkerStarted(): Promise<void> {
if (this.worker && this.workerReady) {
return;
}
if (this.workerStartPromise) {
return this.workerStartPromise;
}
const workerPath = path.join(__dirname, 'blogmarkPython.worker.js');
this.worker = new Worker(workerPath);
this.workerReady = false;
this.worker.on('message', (message: WorkerResponseMessage) => {
this.handleWorkerMessage(message);
});
this.worker.on('error', (error) => {
this.handleWorkerCrash(error instanceof Error ? error : new Error(String(error)));
});
this.worker.on('exit', (code) => {
if (code !== 0) {
this.handleWorkerCrash(new Error(`Python worker exited with code ${code}`));
}
});
this.workerStartPromise = new Promise<void>((resolve, reject) => {
this.workerStartResolve = resolve;
this.workerStartReject = reject;
});
return this.workerStartPromise;
}
private handleWorkerMessage(message: WorkerResponseMessage): void {
if (message.type === 'ready') {
this.workerReady = true;
this.resolveStartPromise();
return;
}
if (message.type === 'error') {
this.handleWorkerCrash(new Error(message.error));
return;
}
const active = this.activeRequest;
if (!active) {
return;
}
if (active.request.requestId !== message.requestId) {
return;
}
if (active.timeoutId) {
clearTimeout(active.timeoutId);
}
this.activeRequest = null;
if (message.type === 'transformResult') {
active.resolve({ output: message.output, toasts: message.toasts });
} else {
active.reject(new Error(message.error));
}
void this.dispatchNext();
}
private handleWorkerCrash(error: Error): void {
this.rejectStartPromise(error);
this.rejectActiveAndQueue(error);
this.resetWorker();
}
private rejectActiveAndQueue(error: Error): void {
if (this.activeRequest) {
if (this.activeRequest.timeoutId) {
clearTimeout(this.activeRequest.timeoutId);
}
this.activeRequest.reject(error);
this.activeRequest = null;
}
while (this.queue.length > 0) {
const queued = this.queue.shift();
queued?.reject(error);
}
}
private resolveStartPromise(): void {
if (this.workerStartResolve) {
this.workerStartResolve();
}
this.workerStartResolve = null;
this.workerStartReject = null;
this.workerStartPromise = null;
}
private rejectStartPromise(error: Error): void {
if (this.workerStartReject) {
this.workerStartReject(error);
}
this.workerStartResolve = null;
this.workerStartReject = null;
this.workerStartPromise = null;
}
private resetWorker(): void {
if (this.worker) {
this.worker.removeAllListeners();
this.worker.terminate();
}
this.worker = null;
this.workerReady = false;
this.workerStartPromise = null;
this.workerStartResolve = null;
this.workerStartReject = null;
}
private nextRequestId(): string {
this.requestCounter += 1;
return `blogmark-py-${this.requestCounter}`;
}
}
let blogmarkPythonWorkerRuntimeInstance: BlogmarkPythonWorkerRuntime | null = null;
export function getBlogmarkPythonWorkerRuntime(): BlogmarkPythonWorkerRuntime {
if (!blogmarkPythonWorkerRuntimeInstance) {
blogmarkPythonWorkerRuntimeInstance = new BlogmarkPythonWorkerRuntime();
}
return blogmarkPythonWorkerRuntimeInstance;
}

View File

@@ -1,5 +1,7 @@
import { z } from 'zod';
import { getScriptEngine } from './ScriptEngine';
import { getMetaEngine } from './MetaEngine';
import { getBlogmarkPythonWorkerRuntime } from './BlogmarkPythonWorkerRuntime';
const transformPostSchema = z.object({
title: z.string().trim().min(1),
@@ -55,6 +57,8 @@ export interface BlogmarkTransformResult {
toasts: string[];
}
export type PythonRuntimeMode = 'webworker' | 'main-thread';
const MAX_TOASTS_PER_SCRIPT = 5;
const MAX_TOASTS_TOTAL = 20;
const MAX_TOAST_LENGTH = 300;
@@ -142,6 +146,28 @@ function toErrorMessage(error: unknown): string {
return String(error);
}
function resolveTransformEntrypoint(value: string): string {
const nextEntrypoint = typeof value === 'string' ? value.trim() : '';
if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(nextEntrypoint) && nextEntrypoint !== 'main') {
return nextEntrypoint;
}
return 'transform';
}
function resolvePythonRuntimeMode(value: unknown): PythonRuntimeMode {
if (value === 'main-thread') {
return 'main-thread';
}
return 'webworker';
}
async function getConfiguredPythonRuntimeMode(): Promise<PythonRuntimeMode> {
const metadata = await getMetaEngine().getProjectMetadata();
return resolvePythonRuntimeMode((metadata as { pythonRuntimeMode?: unknown } | null)?.pythonRuntimeMode);
}
class PythonBlogmarkTransformExecutor implements BlogmarkTransformExecutor {
private runtimePromise: Promise<any> | null = null;
@@ -169,7 +195,7 @@ def toast(message):
await runtime.runPythonAsync(script.content);
const requestedEntrypoint = this.resolveEntrypoint(script.entrypoint);
const requestedEntrypoint = resolveTransformEntrypoint(script.entrypoint);
const payload = JSON.stringify(input);
runtime.globals.set('__bds_transform_payload_json', payload);
runtime.globals.set('__bds_transform_entrypoint', requestedEntrypoint);
@@ -200,15 +226,6 @@ json.dumps(_result)
};
}
private resolveEntrypoint(value: string): string {
const nextEntrypoint = typeof value === 'string' ? value.trim() : '';
if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(nextEntrypoint) && nextEntrypoint !== 'main') {
return nextEntrypoint;
}
return 'transform';
}
private async getRuntime(): Promise<any> {
if (!this.runtimePromise) {
this.runtimePromise = (async () => {
@@ -221,11 +238,26 @@ json.dumps(_result)
}
}
class PythonWorkerBlogmarkTransformExecutor implements BlogmarkTransformExecutor {
async runTransform(script: BlogmarkTransformScriptRecord, input: BlogmarkTransformInput): Promise<unknown> {
return getBlogmarkPythonWorkerRuntime().executeTransform({
scriptContent: script.content,
entrypoint: resolveTransformEntrypoint(script.entrypoint),
payloadJson: JSON.stringify(input),
});
}
}
const mainThreadExecutor = new PythonBlogmarkTransformExecutor();
const workerExecutor = new PythonWorkerBlogmarkTransformExecutor();
export class BlogmarkTransformService {
constructor(
private readonly dependencies: {
provider?: BlogmarkTransformScriptProvider;
executor?: BlogmarkTransformExecutor;
resolvePythonRuntimeMode?: () => Promise<PythonRuntimeMode>;
executors?: Partial<Record<PythonRuntimeMode, BlogmarkTransformExecutor>>;
} = {},
) {}
@@ -237,7 +269,7 @@ export class BlogmarkTransformService {
};
const provider = this.dependencies.provider ?? scriptEngineBackedProvider;
const executor = this.dependencies.executor ?? new PythonBlogmarkTransformExecutor();
const executor = this.dependencies.executor ?? await this.resolveExecutorForConfiguredRuntime();
const scripts = await provider.getScripts();
const activeTransforms = scripts
@@ -303,6 +335,18 @@ export class BlogmarkTransformService {
toasts,
};
}
private async resolveExecutorForConfiguredRuntime(): Promise<BlogmarkTransformExecutor> {
const resolveMode = this.dependencies.resolvePythonRuntimeMode ?? getConfiguredPythonRuntimeMode;
const mode = await resolveMode();
const executors = this.dependencies.executors ?? {};
if (mode === 'main-thread') {
return executors['main-thread'] ?? mainThreadExecutor;
}
return executors.webworker ?? workerExecutor;
}
}
let blogmarkTransformServiceInstance: BlogmarkTransformService | null = null;

View File

@@ -24,6 +24,7 @@ export interface ProjectMetadata {
defaultAuthor?: string; // Default author for new posts and media
maxPostsPerPage?: number; // Preview/server maximum posts per page (default 50)
blogmarkCategory?: string; // Category used for externally captured bookmark posts
pythonRuntimeMode?: 'webworker' | 'main-thread'; // Runtime mode for Python script execution
picoTheme?: PicoThemeName; // Selected Pico CSS theme for preview/rendering
categoryMetadata?: Record<string, CategoryMetadata>; // Per-category metadata for UI/rendering
categorySettings?: Record<string, CategoryRenderSettings>; // Per-category list rendering preferences
@@ -85,6 +86,7 @@ function normalizeProjectMetadata(metadata: ProjectMetadata): ProjectMetadata {
const blogmarkCategory = typeof metadata.blogmarkCategory === 'string'
? normalizeNonEmptyTaxonomyTerm(metadata.blogmarkCategory) ?? undefined
: undefined;
const pythonRuntimeMode = metadata.pythonRuntimeMode === 'main-thread' ? 'main-thread' : 'webworker';
const picoTheme = sanitizePicoTheme(metadata.picoTheme);
const categoryMetadata = normalizeCategoryMetadata(metadata.categoryMetadata ?? metadata.categorySettings);
return {
@@ -92,6 +94,7 @@ function normalizeProjectMetadata(metadata: ProjectMetadata): ProjectMetadata {
publicUrl,
maxPostsPerPage,
blogmarkCategory,
pythonRuntimeMode,
picoTheme,
categoryMetadata,
categorySettings: undefined,
@@ -306,6 +309,7 @@ export class MetaEngine extends EventEmitter {
defaultAuthor: normalizedUpdates.defaultAuthor,
maxPostsPerPage: normalizedUpdates.maxPostsPerPage,
blogmarkCategory: normalizedUpdates.blogmarkCategory,
pythonRuntimeMode: normalizedUpdates.pythonRuntimeMode,
picoTheme: normalizedUpdates.picoTheme,
categoryMetadata: normalizedUpdates.categoryMetadata,
});

View File

@@ -0,0 +1,150 @@
import { parentPort } from 'worker_threads';
interface WorkerRunTransformRequest {
type: 'runTransform';
requestId: string;
scriptContent: string;
entrypoint: string;
payloadJson: string;
}
interface WorkerReadyMessage {
type: 'ready';
}
interface WorkerResultMessage {
type: 'transformResult';
requestId: string;
output: unknown;
toasts: string[];
}
interface WorkerErrorMessage {
type: 'transformError';
requestId: string;
error: string;
}
interface WorkerFatalErrorMessage {
type: 'error';
error: string;
}
type WorkerResponseMessage = WorkerReadyMessage | WorkerResultMessage | WorkerErrorMessage | WorkerFatalErrorMessage;
type PyodideRuntime = {
globals: {
set: (name: string, value: unknown) => void;
} | any;
runPythonAsync: (code: string) => Promise<unknown>;
};
const MAX_TOASTS_PER_SCRIPT = 5;
const MAX_TOAST_LENGTH = 300;
let runtimePromise: Promise<PyodideRuntime> | null = null;
function postMessage(message: WorkerResponseMessage): void {
parentPort?.postMessage(message);
}
function normalizeToastMessage(value: unknown): string | null {
if (value === undefined || value === null) {
return null;
}
const normalized = String(value).trim();
if (normalized.length === 0) {
return null;
}
return normalized.slice(0, MAX_TOAST_LENGTH);
}
async function getRuntime(): Promise<PyodideRuntime> {
if (!runtimePromise) {
runtimePromise = (async () => {
const pyodideModule = await import('pyodide');
return (await pyodideModule.loadPyodide()) as unknown as PyodideRuntime;
})();
}
return runtimePromise;
}
async function runTransform(request: WorkerRunTransformRequest): Promise<void> {
try {
const runtime = await getRuntime();
const toastMessages: string[] = [];
const pushToast = (message: unknown): void => {
if (toastMessages.length >= MAX_TOASTS_PER_SCRIPT) {
return;
}
const normalized = normalizeToastMessage(message);
if (!normalized) {
return;
}
toastMessages.push(normalized);
};
runtime.globals.set('__bds_push_toast', pushToast);
await runtime.runPythonAsync(`
def toast(message):
__bds_push_toast(str(message))
`);
await runtime.runPythonAsync(request.scriptContent);
runtime.globals.set('__bds_transform_payload_json', request.payloadJson);
runtime.globals.set('__bds_transform_entrypoint', request.entrypoint);
const rawResult = await runtime.runPythonAsync(`
import json
_payload = json.loads(__bds_transform_payload_json)
_entrypoint = __bds_transform_entrypoint
_transform_fn = globals().get(_entrypoint)
if _transform_fn is None or not callable(_transform_fn):
raise RuntimeError(f"Transform entrypoint '{_entrypoint}' is not callable")
_post = _payload.get("post")
if not isinstance(_post, dict):
raise RuntimeError("Transform payload is missing a valid 'post' object")
_context = _payload.get("context")
try:
_result = _transform_fn(_post, _context)
except TypeError:
_result = _transform_fn(_post)
if _result is None:
_result = _post
json.dumps(_result)
`);
postMessage({
type: 'transformResult',
requestId: request.requestId,
output: JSON.parse(String(rawResult)),
toasts: toastMessages,
});
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
postMessage({ type: 'transformError', requestId: request.requestId, error: message });
}
}
parentPort?.on('message', (message: WorkerRunTransformRequest) => {
if (message.type !== 'runTransform') {
return;
}
void runTransform(message);
});
void getRuntime()
.then(() => {
postMessage({ type: 'ready' });
})
.catch((error) => {
const message = error instanceof Error ? error.message : String(error);
postMessage({ type: 'error', error: message });
});

View File

@@ -1006,7 +1006,7 @@ export function registerIpcHandlers(): void {
return engine.getProjectMetadata();
});
safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; picoTheme?: import('../shared/picoThemes').PicoThemeName; categoryMetadata?: Record<string, { renderInLists: boolean; showTitle: boolean; title: string }>; categorySettings?: Record<string, { renderInLists: boolean; showTitle: boolean }> }) => {
safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('../shared/picoThemes').PicoThemeName; categoryMetadata?: Record<string, { renderInLists: boolean; showTitle: boolean; title: string }>; categorySettings?: Record<string, { renderInLists: boolean; showTitle: boolean }> }) => {
const engine = getMetaEngine();
await ensureMetaContext(engine);
await engine.updateProjectMetadata(updates);

View File

@@ -172,7 +172,7 @@ export const electronAPI: ElectronAPI = {
syncOnStartup: () => ipcRenderer.invoke('meta:syncOnStartup'),
getProjectMetadata: () => ipcRenderer.invoke('meta:getProjectMetadata'),
setProjectMetadata: (metadata: { name: string; description?: string }) => ipcRenderer.invoke('meta:setProjectMetadata', metadata),
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; picoTheme?: import('./shared/picoThemes').PicoThemeName; categoryMetadata?: Record<string, { renderInLists: boolean; showTitle: boolean; title: string }>; categorySettings?: Record<string, { renderInLists: boolean; showTitle: boolean }> }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates),
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('./shared/picoThemes').PicoThemeName; categoryMetadata?: Record<string, { renderInLists: boolean; showTitle: boolean; title: string }>; categorySettings?: Record<string, { renderInLists: boolean; showTitle: boolean }> }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates),
},
// Tag Management (advanced tag operations)

View File

@@ -43,6 +43,7 @@ export interface ProjectMetadata {
defaultAuthor?: string;
maxPostsPerPage?: number;
blogmarkCategory?: string;
pythonRuntimeMode?: 'webworker' | 'main-thread';
picoTheme?: import('./picoThemes').PicoThemeName;
categoryMetadata?: Record<string, CategoryMetadata>;
categorySettings?: Record<string, CategoryRenderSettings>;
@@ -619,7 +620,7 @@ export interface ElectronAPI {
syncOnStartup: () => Promise<{ tags: string[]; categories: string[]; projectMetadata: ProjectMetadata | null }>;
getProjectMetadata: () => Promise<ProjectMetadata | null>;
setProjectMetadata: (metadata: { name: string; description?: string }) => Promise<ProjectMetadata | null>;
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; picoTheme?: import('./picoThemes').PicoThemeName; categoryMetadata?: Record<string, CategoryMetadata>; categorySettings?: Record<string, CategoryRenderSettings> }) => Promise<ProjectMetadata | null>;
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; publicUrl?: string; mainLanguage?: string; defaultAuthor?: string; maxPostsPerPage?: number; blogmarkCategory?: string; pythonRuntimeMode?: 'webworker' | 'main-thread'; picoTheme?: import('./picoThemes').PicoThemeName; categoryMetadata?: Record<string, CategoryMetadata>; categorySettings?: Record<string, CategoryRenderSettings> }) => Promise<ProjectMetadata | null>;
};
tags: {
getAll: () => Promise<TagData[]>;

View File

@@ -10,7 +10,7 @@ import {
import './SettingsView.css';
// Export category IDs for sidebar navigation
export type SettingsCategory = 'project' | 'editor' | 'content' | 'ai' | 'publishing' | 'data';
export type SettingsCategory = 'project' | 'editor' | 'content' | 'ai' | 'technology' | 'publishing' | 'data';
// Scroll to a settings section by category ID
export const scrollToSettingsSection = (category: SettingsCategory) => {
@@ -150,6 +150,7 @@ export const SettingsView: React.FC = () => {
const [projectDefaultAuthor, setProjectDefaultAuthor] = useState('');
const [projectMaxPostsPerPage, setProjectMaxPostsPerPage] = useState(50);
const [projectBlogmarkCategory, setProjectBlogmarkCategory] = useState('article');
const [projectPythonRuntimeMode, setProjectPythonRuntimeMode] = useState<'webworker' | 'main-thread'>('webworker');
// Post categories management
const [postCategories, setPostCategories] = useState<string[]>(DEFAULT_POST_CATEGORIES);
@@ -208,6 +209,9 @@ export const SettingsView: React.FC = () => {
const incomingBlogmarkCategory = normalizeBlogmarkCategory((metadata as { blogmarkCategory?: unknown } | null)?.blogmarkCategory);
setProjectBlogmarkCategory(incomingBlogmarkCategory || 'article');
const incomingPythonRuntimeMode = (metadata as { pythonRuntimeMode?: unknown } | null)?.pythonRuntimeMode;
setProjectPythonRuntimeMode(incomingPythonRuntimeMode === 'main-thread' ? 'main-thread' : 'webworker');
const incomingCategoryMetadata = (metadata as any)?.categoryMetadata as Record<string, CategoryMetadata> | undefined;
const incomingLegacyCategorySettings = (metadata as any)?.categorySettings as Record<string, { renderInLists: boolean; showTitle: boolean }> | undefined;
setCategoryMetadata((current) => {
@@ -342,6 +346,7 @@ export const SettingsView: React.FC = () => {
defaultAuthor: projectDefaultAuthor.trim() || undefined,
maxPostsPerPage: Math.min(500, Math.max(1, Math.floor(projectMaxPostsPerPage || 50))),
blogmarkCategory: normalizeBlogmarkCategory(projectBlogmarkCategory) || undefined,
pythonRuntimeMode: projectPythonRuntimeMode,
categoryMetadata,
});
}
@@ -389,6 +394,7 @@ export const SettingsView: React.FC = () => {
const editorKeywords = ['editor', 'mode', 'wysiwyg', 'markdown', 'preview', 'visual'];
const contentKeywords = ['content', 'categories', 'post', 'article', 'picture', 'aside', 'page'];
const aiKeywords = ['ai', 'assistant', 'chat', 'model', 'prompt', 'system', 'api', 'key', 'claude', 'gpt', 'opencode'];
const technologyKeywords = ['technology', 'python', 'runtime', 'worker', 'webworker', 'main thread', 'execution'];
const publishingKeywords = ['publishing', 'ftp', 'ssh', 'deploy', 'server', 'host', 'upload'];
const dataKeywords = ['data', 'database', 'rebuild', 'maintenance', 'posts', 'media', 'links', 'folder', 'filesystem'];
@@ -1023,6 +1029,30 @@ export const SettingsView: React.FC = () => {
</SettingSection>
);
const renderTechnologySettings = () => (
<SettingSection
id="settings-section-technology"
title={t('settings.technology.title')}
description={t('settings.technology.description')}
hidden={!sectionHasMatches(technologyKeywords)}
>
<SettingRow
id="project-python-runtime-mode"
label={t('settings.technology.pythonRuntimeModeLabel')}
description={t('settings.technology.pythonRuntimeModeDescription')}
>
<select
id="project-python-runtime-mode"
value={projectPythonRuntimeMode}
onChange={(event) => setProjectPythonRuntimeMode(event.target.value as 'webworker' | 'main-thread')}
>
<option value="webworker">{t('settings.technology.pythonRuntimeMode.webworker')}</option>
<option value="main-thread">{t('settings.technology.pythonRuntimeMode.mainThread')}</option>
</select>
</SettingRow>
</SettingSection>
);
const renderPublishingSettings = () => (
<>
<SettingSection
@@ -1290,6 +1320,7 @@ export const SettingsView: React.FC = () => {
sectionHasMatches(editorKeywords) ||
sectionHasMatches(contentKeywords) ||
sectionHasMatches(aiKeywords) ||
sectionHasMatches(technologyKeywords) ||
sectionHasMatches(publishingKeywords) ||
sectionHasMatches(dataKeywords);
@@ -1325,6 +1356,7 @@ export const SettingsView: React.FC = () => {
{renderEditorSettings()}
{renderContentSettings()}
{renderAISettings()}
{renderTechnologySettings()}
{renderPublishingSettings()}
{renderDataSettings()}
</>

View File

@@ -1261,7 +1261,7 @@ const SettingsNav: React.FC = () => {
const { tabs, activeTabId, openTab } = useAppStore();
const [activeSection, setActiveSection] = useState<SettingsCategory | null>(() => {
const persisted = getPersistedSidebarSection('settings');
if (persisted === 'project' || persisted === 'editor' || persisted === 'content' || persisted === 'ai' || persisted === 'publishing' || persisted === 'data') {
if (persisted === 'project' || persisted === 'editor' || persisted === 'content' || persisted === 'ai' || persisted === 'technology' || persisted === 'publishing' || persisted === 'data') {
return persisted;
}
return null;
@@ -1322,6 +1322,13 @@ const SettingsNav: React.FC = () => {
<span className="settings-nav-entry-icon">🤖</span>
<span>{t('sidebar.nav.ai')}</span>
</button>
<button
className={`settings-nav-entry ${activeSection === 'technology' ? 'active' : ''}`}
onClick={() => handleNavClick('technology')}
>
<span className="settings-nav-entry-icon"></span>
<span>{t('sidebar.nav.technology')}</span>
</button>
<button
className={`settings-nav-entry ${activeSection === 'publishing' ? 'active' : ''}`}
onClick={() => handleNavClick('publishing')}

View File

@@ -127,6 +127,12 @@
"settings.content.showTitles": "Titel anzeigen",
"settings.ai.title": "KI-Assistent",
"settings.ai.noModels": "Keine Modelle verfügbar",
"settings.technology.title": "Technologie",
"settings.technology.description": "Konfiguriere das Laufzeitverhalten für die Python-Skriptausführung.",
"settings.technology.pythonRuntimeModeLabel": "Python-Laufzeitmodus",
"settings.technology.pythonRuntimeModeDescription": "Lege fest, wo Python-Skripte für Transformationspipelines ausgeführt werden.",
"settings.technology.pythonRuntimeMode.webworker": "Web Worker (empfohlen)",
"settings.technology.pythonRuntimeMode.mainThread": "Hauptthread (Legacy)",
"settings.publishing.ftpTitle": "FTP-Veröffentlichung",
"settings.publishing.sshTitle": "SSH-Veröffentlichung",
"settings.data.title": "Datenbankwartung",
@@ -421,6 +427,7 @@
"sidebar.nav.editor": "Texteditor",
"sidebar.nav.content": "Inhalt",
"sidebar.nav.ai": "KI-Assistent",
"sidebar.nav.technology": "Technologie",
"sidebar.nav.publishing": "Veröffentlichung",
"sidebar.nav.data": "Daten",
"sidebar.nav.style": "Stil",

View File

@@ -127,6 +127,12 @@
"settings.content.showTitles": "Show titles",
"settings.ai.title": "AI Assistant",
"settings.ai.noModels": "No models available",
"settings.technology.title": "Technology",
"settings.technology.description": "Configure runtime behavior for Python script execution.",
"settings.technology.pythonRuntimeModeLabel": "Python Runtime Mode",
"settings.technology.pythonRuntimeModeDescription": "Choose where Python scripts execute for transform pipelines.",
"settings.technology.pythonRuntimeMode.webworker": "Web Worker (Recommended)",
"settings.technology.pythonRuntimeMode.mainThread": "Main Thread (Legacy)",
"settings.publishing.ftpTitle": "FTP Publishing",
"settings.publishing.sshTitle": "SSH Publishing",
"settings.data.title": "Database Maintenance",
@@ -421,6 +427,7 @@
"sidebar.nav.editor": "Editor",
"sidebar.nav.content": "Content",
"sidebar.nav.ai": "AI Assistant",
"sidebar.nav.technology": "Technology",
"sidebar.nav.publishing": "Publishing",
"sidebar.nav.data": "Data",
"sidebar.nav.style": "Style",

View File

@@ -127,6 +127,12 @@
"settings.content.showTitles": "Mostrar títulos",
"settings.ai.title": "Asistente IA",
"settings.ai.noModels": "No hay modelos disponibles",
"settings.technology.title": "Tecnología",
"settings.technology.description": "Configura el comportamiento de ejecución para scripts de Python.",
"settings.technology.pythonRuntimeModeLabel": "Modo de ejecución de Python",
"settings.technology.pythonRuntimeModeDescription": "Elige dónde se ejecutan los scripts de Python para los flujos de transformación.",
"settings.technology.pythonRuntimeMode.webworker": "Web Worker (recomendado)",
"settings.technology.pythonRuntimeMode.mainThread": "Hilo principal (heredado)",
"settings.publishing.ftpTitle": "Publicación FTP",
"settings.publishing.sshTitle": "Publicación SSH",
"settings.data.title": "Mantenimiento de base de datos",
@@ -421,6 +427,7 @@
"sidebar.nav.editor": "Editor",
"sidebar.nav.content": "Contenido",
"sidebar.nav.ai": "Asistente IA",
"sidebar.nav.technology": "Tecnología",
"sidebar.nav.publishing": "Publicación",
"sidebar.nav.data": "Datos",
"sidebar.nav.style": "Estilo",

View File

@@ -127,6 +127,12 @@
"settings.content.showTitles": "Afficher les titres",
"settings.ai.title": "Assistant IA",
"settings.ai.noModels": "Aucun modèle disponible",
"settings.technology.title": "Technologie",
"settings.technology.description": "Configurez le comportement dexécution des scripts Python.",
"settings.technology.pythonRuntimeModeLabel": "Mode dexécution Python",
"settings.technology.pythonRuntimeModeDescription": "Choisissez où les scripts Python sexécutent pour les pipelines de transformation.",
"settings.technology.pythonRuntimeMode.webworker": "Web Worker (recommandé)",
"settings.technology.pythonRuntimeMode.mainThread": "Thread principal (hérité)",
"settings.publishing.ftpTitle": "Publication FTP",
"settings.publishing.sshTitle": "Publication SSH",
"settings.data.title": "Maintenance de la base de données",
@@ -421,6 +427,7 @@
"sidebar.nav.editor": "Éditeur",
"sidebar.nav.content": "Contenu",
"sidebar.nav.ai": "Assistant IA",
"sidebar.nav.technology": "Technologie",
"sidebar.nav.publishing": "Publication",
"sidebar.nav.data": "Données",
"sidebar.nav.style": "Style",

View File

@@ -127,6 +127,12 @@
"settings.content.showTitles": "Mostra titoli",
"settings.ai.title": "Assistente IA",
"settings.ai.noModels": "Nessun modello disponibile",
"settings.technology.title": "Tecnologia",
"settings.technology.description": "Configura il comportamento di runtime per l'esecuzione degli script Python.",
"settings.technology.pythonRuntimeModeLabel": "Modalità runtime Python",
"settings.technology.pythonRuntimeModeDescription": "Scegli dove eseguire gli script Python per le pipeline di trasformazione.",
"settings.technology.pythonRuntimeMode.webworker": "Web Worker (consigliato)",
"settings.technology.pythonRuntimeMode.mainThread": "Thread principale (legacy)",
"settings.publishing.ftpTitle": "Pubblicazione FTP",
"settings.publishing.sshTitle": "Pubblicazione SSH",
"settings.data.title": "Manutenzione database",
@@ -421,6 +427,7 @@
"sidebar.nav.editor": "Editor",
"sidebar.nav.content": "Contenuto",
"sidebar.nav.ai": "Assistente IA",
"sidebar.nav.technology": "Tecnologia",
"sidebar.nav.publishing": "Pubblicazione",
"sidebar.nav.data": "Dati",
"sidebar.nav.style": "Stile",

View File

@@ -54,6 +54,8 @@ export class PythonRuntimeManager {
private initializeDeferred: InitializeDeferred | null = null;
private ready = false;
private pendingRuns = new Map<string, PendingRun>();
private requestQueue: PythonWorkerRequest[] = [];
private activeRequestId: string | null = null;
private requestCounter = 0;
constructor(private readonly workerFactory: WorkerFactory = createPythonRuntimeWorker) {}
@@ -123,7 +125,7 @@ export class PythonRuntimeManager {
entrypoint: options?.entrypoint,
};
this.worker!.postMessage(message);
this.enqueueRequest(message);
});
}
@@ -162,7 +164,7 @@ export class PythonRuntimeManager {
cacheKey: options?.cacheKey,
};
this.worker!.postMessage(message);
this.enqueueRequest(message);
});
}
@@ -198,7 +200,7 @@ export class PythonRuntimeManager {
cacheKey: options?.cacheKey,
};
this.worker!.postMessage(message);
this.enqueueRequest(message);
});
}
@@ -234,7 +236,7 @@ export class PythonRuntimeManager {
cacheKey: options?.cacheKey,
};
this.worker!.postMessage(message);
this.enqueueRequest(message);
});
}
@@ -262,6 +264,10 @@ export class PythonRuntimeManager {
const pendingRun = this.pendingRuns.get(payload.requestId);
if (!pendingRun) {
if (this.activeRequestId === payload.requestId && payload.type !== 'stdout') {
this.activeRequestId = null;
this.dispatchNextRequest();
}
return;
}
@@ -278,33 +284,40 @@ export class PythonRuntimeManager {
if (payload.type === 'runResult') {
if (pendingRun.kind !== 'run') {
pendingRun.reject(new Error('Invalid response type for pending macro request'));
this.finishRequest(payload.requestId);
return;
}
pendingRun.resolve({ result: payload.result, stdout: pendingRun.stdout });
this.finishRequest(payload.requestId);
return;
}
if (payload.type === 'entrypoints') {
if (pendingRun.kind !== 'inspect-entrypoints') {
pendingRun.reject(new Error('Invalid response type for pending run request'));
this.finishRequest(payload.requestId);
return;
}
pendingRun.resolve(payload.entrypoints);
this.finishRequest(payload.requestId);
return;
}
if (payload.type === 'syntaxResult') {
if (pendingRun.kind !== 'syntax-check') {
pendingRun.reject(new Error('Invalid response type for pending syntax check request'));
this.finishRequest(payload.requestId);
return;
}
pendingRun.resolve({ errors: payload.errors });
this.finishRequest(payload.requestId);
return;
}
if (payload.type === 'macroResult') {
if (pendingRun.kind !== 'macro-v1') {
pendingRun.reject(new Error('Invalid response type for pending run request'));
this.finishRequest(payload.requestId);
return;
}
@@ -314,10 +327,12 @@ export class PythonRuntimeManager {
} catch (error) {
pendingRun.reject(error instanceof Error ? error : new Error(String(error)));
}
this.finishRequest(payload.requestId);
return;
}
pendingRun.reject(new Error(payload.error));
this.finishRequest(payload.requestId);
}
private handleWorkerError(error: Error): void {
@@ -334,6 +349,8 @@ export class PythonRuntimeManager {
}
this.pendingRuns.clear();
this.requestQueue = [];
this.activeRequestId = null;
this.worker?.terminate();
this.worker = null;
this.initializingPromise = null;
@@ -356,12 +373,50 @@ export class PythonRuntimeManager {
}
this.pendingRuns.clear();
this.requestQueue = [];
this.activeRequestId = null;
this.worker?.terminate();
this.worker = null;
this.initializingPromise = null;
this.ready = false;
}
private enqueueRequest(request: PythonWorkerRequest): void {
if (!this.worker || !this.ready) {
this.requestQueue.push(request);
return;
}
if (this.activeRequestId !== null) {
this.requestQueue.push(request);
return;
}
this.activeRequestId = request.requestId;
this.worker.postMessage(request);
}
private dispatchNextRequest(): void {
if (!this.worker || !this.ready || this.activeRequestId !== null || this.requestQueue.length === 0) {
return;
}
const nextRequest = this.requestQueue.shift();
if (!nextRequest) {
return;
}
this.activeRequestId = nextRequest.requestId;
this.worker.postMessage(nextRequest);
}
private finishRequest(requestId: string): void {
if (this.activeRequestId === requestId) {
this.activeRequestId = null;
}
this.dispatchNextRequest();
}
private nextRequestId(): string {
this.requestCounter += 1;
return `req-${this.requestCounter}`;

View File

@@ -63,7 +63,10 @@ describe('BlogmarkTransformService (Pyodide integration)', () => {
getScripts: async () => [createTransformScript()],
};
const service = new BlogmarkTransformService({ provider });
const service = new BlogmarkTransformService({
provider,
resolvePythonRuntimeMode: async () => 'main-thread',
});
const result = await service.applyTransforms(createInput());

View File

@@ -247,67 +247,53 @@ describe('BlogmarkTransformService', () => {
expect(result.toasts).toEqual(['Step finished', 'Step finished']);
});
it('invokes python transform entrypoint with post payload shape', async () => {
const globalsStore = new Map<string, unknown>();
const runPythonAsync = vi.fn(async (code: string) => {
if (code.includes('json.dumps(_result)')) {
const payload = JSON.parse(String(globalsStore.get('__bds_transform_payload_json')));
if (code.includes('_transform_fn(_payload)')) {
return JSON.stringify(payload);
}
return JSON.stringify({
...payload.post,
title: 'Normalized',
categories: ['spielelog', 'asides'],
tags: ['inbox', 'spielen'],
});
}
return null;
});
vi.doMock('pyodide', () => ({
loadPyodide: vi.fn(async () => ({
globals: {
set: (key: string, value: unknown) => {
globalsStore.set(key, value);
},
},
runPythonAsync,
})),
}));
const provider: BlogmarkTransformScriptProvider = {
getScripts: vi.fn(async () => [
createScript({
id: 'pyodide-transform',
slug: 'pyodide-transform',
title: 'Pyodide Transform',
kind: 'transform',
entrypoint: 'normalize_blogmark',
content: 'def normalize_blogmark(post):\n return post',
}),
]),
it('uses webworker executor when runtime mode resolves to webworker', async () => {
const webworkerExecutor: BlogmarkTransformExecutor = {
runTransform: vi.fn(async (_script, input) => ({ output: input.post, toasts: [] })),
};
const mainThreadExecutor: BlogmarkTransformExecutor = {
runTransform: vi.fn(async (_script, input) => ({ output: input.post, toasts: [] })),
};
const service = new BlogmarkTransformService({ provider });
const service = new BlogmarkTransformService({
provider: {
getScripts: async () => [createScript({ id: 'worker-script', slug: 'worker-script' })],
},
resolvePythonRuntimeMode: async () => 'webworker',
executors: {
webworker: webworkerExecutor,
'main-thread': mainThreadExecutor,
},
});
const result = await service.applyTransforms(createInput());
await service.applyTransforms(createInput());
const transformInvocationCode = runPythonAsync.mock.calls
.map((call) => call[0])
.find((code) => typeof code === 'string' && String(code).includes('json.dumps(_result)'));
expect(webworkerExecutor.runTransform).toHaveBeenCalledTimes(1);
expect(mainThreadExecutor.runTransform).not.toHaveBeenCalled();
});
expect(result.post.title).toBe('Normalized');
expect(result.post.categories).toEqual(['spielelog', 'asides']);
expect(result.post.tags).toEqual(['inbox', 'spielen']);
expect(transformInvocationCode).toBeDefined();
expect(String(transformInvocationCode)).not.toContain('import inspect');
expect(String(transformInvocationCode)).toContain('\ntry:\n');
expect(String(transformInvocationCode)).toContain('\nexcept TypeError:\n');
expect(String(transformInvocationCode)).not.toContain('\n try:\n');
expect(String(transformInvocationCode)).not.toContain('\n except TypeError:\n');
it('uses main-thread executor when runtime mode resolves to main-thread', async () => {
const webworkerExecutor: BlogmarkTransformExecutor = {
runTransform: vi.fn(async (_script, input) => ({ output: input.post, toasts: [] })),
};
const mainThreadExecutor: BlogmarkTransformExecutor = {
runTransform: vi.fn(async (_script, input) => ({ output: input.post, toasts: [] })),
};
const service = new BlogmarkTransformService({
provider: {
getScripts: async () => [createScript({ id: 'main-script', slug: 'main-script' })],
},
resolvePythonRuntimeMode: async () => 'main-thread',
executors: {
webworker: webworkerExecutor,
'main-thread': mainThreadExecutor,
},
});
await service.applyTransforms(createInput());
expect(mainThreadExecutor.runTransform).toHaveBeenCalledTimes(1);
expect(webworkerExecutor.runTransform).not.toHaveBeenCalled();
});
});

View File

@@ -739,6 +739,30 @@ describe('MetaEngine', () => {
expect((metadata as any)?.blogmarkCategory).toBe('article');
});
it('should set and get pythonRuntimeMode in project metadata', async () => {
await metaEngine.setProjectMetadata({
name: 'My Blog',
pythonRuntimeMode: 'main-thread',
} as any);
const metadata = await metaEngine.getProjectMetadata();
expect((metadata as any)?.pythonRuntimeMode).toBe('main-thread');
});
it('should persist pythonRuntimeMode to filesystem', async () => {
await metaEngine.setProjectMetadata({
name: 'Runtime Mode Project',
pythonRuntimeMode: 'webworker',
} as any);
const metaDir = metaEngine.getMetaDir();
const projectPath = normalizePath(`${metaDir}/project.json`);
const content = mockFiles.get(projectPath);
const parsed = JSON.parse(content!);
expect(parsed.pythonRuntimeMode).toBe('webworker');
});
it('should persist blogmarkCategory to filesystem', async () => {
await metaEngine.setProjectMetadata({
name: 'Test Project',

View File

@@ -117,6 +117,33 @@ describe('SettingsView Diff Preferences', () => {
);
});
it('includes python runtime mode in metadata save payload', async () => {
(window as any).electronAPI.meta.getProjectMetadata = vi.fn().mockResolvedValue({
maxPostsPerPage: 75,
publicUrl: 'https://example.com',
pythonRuntimeMode: 'main-thread',
categorySettings: {
article: { renderInLists: true, showTitle: true },
picture: { renderInLists: true, showTitle: true },
aside: { renderInLists: true, showTitle: false },
page: { renderInLists: false, showTitle: true },
},
});
render(<SettingsView />);
await screen.findByDisplayValue('Main Thread (Legacy)');
const saveButton = screen.getByRole('button', { name: /save project settings/i });
fireEvent.click(saveButton);
await new Promise((resolve) => setTimeout(resolve, 0));
expect((window as any).electronAPI.meta.updateProjectMetadata).toHaveBeenCalledWith(
expect.objectContaining({ pythonRuntimeMode: 'main-thread' })
);
});
it('renders category settings checkboxes with required defaults', async () => {
render(<SettingsView />);

View File

@@ -191,6 +191,32 @@ describe('PythonRuntimeManager', () => {
await expect(runPromise).rejects.toThrow('boom');
});
it('queues concurrent execute calls and sends the next request only after completion', async () => {
const worker = new MockWorker();
const manager = new PythonRuntimeManager(() => worker as unknown as Worker);
const initPromise = manager.initialize();
worker.emitMessage({ type: 'ready' });
await initPromise;
const firstRun = manager.execute('1 + 1');
const secondRun = manager.execute('2 + 2');
await Promise.resolve();
expect(worker.postedMessages).toHaveLength(1);
const firstRequest = worker.postedMessages[0] as { requestId: string };
worker.emitMessage({ type: 'runResult', requestId: firstRequest.requestId, result: '2' });
await expect(firstRun).resolves.toEqual({ result: '2', stdout: '' });
await Promise.resolve();
expect(worker.postedMessages).toHaveLength(2);
const secondRequest = worker.postedMessages[1] as { requestId: string };
worker.emitMessage({ type: 'runResult', requestId: secondRequest.requestId, result: '4' });
await expect(secondRun).resolves.toEqual({ result: '4', stdout: '' });
});
it('terminates timed out worker and recovers with a new worker', async () => {
const workers: MockWorker[] = [];
const manager = new PythonRuntimeManager(() => {