hugo/bDS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: python script sync db - files

Browse Source
This commit is contained in:
hugo
2026-02-23 22:52:34 +01:00
parent 838ea34ab7
commit 18835a395d
16 changed files with 766 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/main/engine/GitEngine.ts
View File

@@ -140,6 +140,14 @@ export interface GitPostFileChange {
previousPath?: string;
}
export type GitScriptFileChangeStatus = 'added' | 'modified' | 'deleted' | 'renamed';
export interface GitScriptFileChange {
status: GitScriptFileChangeStatus;
path: string;
previousPath?: string;
}
type GitProvider = 'unknown' | 'github' | 'gitlab' | 'gitea-forgejo';
let gitEngineInstance: GitEngine | null = null;
@@ -526,7 +534,12 @@ export class GitEngine {
return this.markdownExtensions.has(extension);
}
private parseNameStatusOutput(raw: string): GitPostFileChange[] {
private isScriptsPythonPath(value: string): boolean {
const normalized = this.normalizeRepoRelativePath(value);
return normalized.startsWith('scripts/') && path.extname(normalized).toLowerCase() === '.py';
}
private parseNameStatusOutput(raw: string, pathMatcher: (value: string) => boolean): GitPostFileChange[] {
const tokens = raw.split('\0').filter((token) => token.length > 0);
const changes: GitPostFileChange[] = [];
@@ -543,7 +556,7 @@ export class GitEngine {
const previousPath = this.normalizeRepoRelativePath(previousPathRaw);
const pathValue = this.normalizeRepoRelativePath(nextPathRaw);
if (this.isPostsMarkdownPath(previousPath) || this.isPostsMarkdownPath(pathValue)) {
if (pathMatcher(previousPath) || pathMatcher(pathValue)) {
changes.push({
status: 'renamed',
path: pathValue,
@@ -555,7 +568,7 @@ export class GitEngine {
const filePathRaw = tokens[index++] ?? '';
const filePath = this.normalizeRepoRelativePath(filePathRaw);
if (!this.isPostsMarkdownPath(filePath)) {
if (!pathMatcher(filePath)) {
continue;
}
@@ -1338,13 +1351,40 @@ export class GitEngine {
try {
const output = await git.raw(args);
return this.parseNameStatusOutput(output);
return this.parseNameStatusOutput(output, (value) => this.isPostsMarkdownPath(value));
} catch (error) {
const message = error instanceof Error ? error.message : String(error ?? '');
if (this.isSpawnBadFileDescriptorError(message)) {
try {
const output = await this.runGitCli(projectPath, args);
return this.parseNameStatusOutput(output);
return this.parseNameStatusOutput(output, (value) => this.isPostsMarkdownPath(value));
} catch {
return [];
}
}
return [];
}
}
async getChangedScriptFilesBetween(projectPath: string, fromCommit: string, toCommit: string): Promise<GitScriptFileChange[]> {
const fromRef = fromCommit.trim();
const toRef = toCommit.trim();
if (!fromRef || !toRef || fromRef === toRef) {
return [];
}
const git = this.createNonInteractiveGit(projectPath);
const args = ['diff', '--name-status', '--find-renames', '-z', `${fromRef}..${toRef}`, '--', 'scripts'];
try {
const output = await git.raw(args);
return this.parseNameStatusOutput(output, (value) => this.isScriptsPythonPath(value));
} catch (error) {
const message = error instanceof Error ? error.message : String(error ?? '');
if (this.isSpawnBadFileDescriptorError(message)) {
try {
const output = await this.runGitCli(projectPath, args);
return this.parseNameStatusOutput(output, (value) => this.isScriptsPythonPath(value));
} catch {
return [];
}

416
src/main/engine/ScriptEngine.ts
View File

@@ -42,6 +42,37 @@ export interface UpdateScriptInput {
enabled?: boolean;
}
export type GitScriptFileChangeStatus = 'added' | 'modified' | 'deleted' | 'renamed';
export interface GitScriptFileChange {
status: GitScriptFileChangeStatus;
path: string;
previousPath?: string;
}
export interface ScriptReconcileResult {
created: number;
updated: number;
deleted: number;
processedFiles: number;
}
interface ParsedScriptFile {
metadata: {
id?: string;
projectId?: string;
slug?: string;
title?: string;
kind?: string;
entrypoint?: string;
enabled?: boolean;
version?: number;
createdAt?: string;
updatedAt?: string;
};
body: string;
}
export class ScriptEngine extends EventEmitter {
private currentProjectId = 'default';
private dataDir: string | null = null;
@@ -191,6 +222,205 @@ export class ScriptEngine extends EventEmitter {
return Promise.all(rows.map((item) => this.toScriptData(item)));
}
async rebuildDatabaseFromFiles(): Promise<void> {
const db = getDatabase().getLocal();
const scriptsDir = this.getScriptsDir();
await db.delete(scripts).where(eq(scripts.projectId, this.currentProjectId));
const pythonFiles = await this.scanScriptFiles(scriptsDir);
if (pythonFiles.length === 0) {
this.emit('scriptsRebuilt');
return;
}
const usedIds = new Set<string>();
const insertedRows: Script[] = [];
for (const filePath of pythonFiles) {
const parsed = await this.readScriptFileWithMetadata(filePath);
if (!parsed) {
continue;
}
const desiredSlug = this.normalizeSlug(parsed.metadata.slug || path.basename(filePath, '.py'));
const slug = this.ensureUniqueSlug(desiredSlug, insertedRows);
const desiredId = typeof parsed.metadata.id === 'string' && parsed.metadata.id.trim().length > 0
? parsed.metadata.id.trim()
: uuidv4();
const id = usedIds.has(desiredId) ? uuidv4() : desiredId;
const now = new Date();
const row: NewScript = {
id,
projectId: this.currentProjectId,
slug,
title: this.normalizeTitle(parsed.metadata.title, slug),
kind: this.normalizeKind(parsed.metadata.kind),
entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint),
enabled: this.normalizeEnabled(parsed.metadata.enabled),
version: this.normalizeVersion(parsed.metadata.version),
filePath,
createdAt: this.normalizeDate(parsed.metadata.createdAt, now),
updatedAt: this.normalizeDate(parsed.metadata.updatedAt, now),
};
await db.insert(scripts).values(row);
insertedRows.push(row as Script);
usedIds.add(id);
}
this.emit('scriptsRebuilt');
}
async reconcileScriptsFromGitChanges(projectPath: string, changes: GitScriptFileChange[]): Promise<ScriptReconcileResult> {
const db = getDatabase().getLocal();
const normalizedProjectPath = path.resolve(projectPath);
const relevantChanges = changes.filter((change) => {
if (!this.isPythonScriptPath(change.path)) {
return false;
}
if (change.status === 'renamed' && change.previousPath && !this.isPythonScriptPath(change.previousPath) && !this.isPythonScriptPath(change.path)) {
return false;
}
return true;
});
if (relevantChanges.length === 0) {
return { created: 0, updated: 0, deleted: 0, processedFiles: 0 };
}
const scriptRows = await this.getAllScriptRows();
const scriptsByPath = new Map<string, Script>();
for (const row of scriptRows) {
scriptsByPath.set(this.normalizePathForCompare(row.filePath), row);
}
let created = 0;
let updated = 0;
let deleted = 0;
let processedFiles = 0;
for (const change of relevantChanges) {
const absolutePath = this.normalizePathForCompare(path.resolve(normalizedProjectPath, change.path));
const previousAbsolutePath = change.previousPath
? this.normalizePathForCompare(path.resolve(normalizedProjectPath, change.previousPath))
: null;
if (change.status === 'deleted') {
const existing = scriptsByPath.get(absolutePath);
if (!existing) {
continue;
}
await db.delete(scripts).where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId)));
scriptsByPath.delete(absolutePath);
this.emit('scriptDeleted', existing.id);
deleted += 1;
processedFiles += 1;
continue;
}
let existing = previousAbsolutePath
? (scriptsByPath.get(previousAbsolutePath) || scriptsByPath.get(absolutePath))
: scriptsByPath.get(absolutePath);
const parsed = await this.readScriptFileWithMetadata(absolutePath);
if (!parsed) {
continue;
}
const allRows = await this.getAllScriptRows();
const parsedId = typeof parsed.metadata.id === 'string' ? parsed.metadata.id.trim() : '';
if (!existing && parsedId.length > 0) {
const byId = allRows.find((row) => row.id === parsedId);
if (byId) {
existing = byId;
}
}
const desiredSlug = this.normalizeSlug(parsed.metadata.slug || path.basename(absolutePath, '.py'));
const slug = this.ensureUniqueSlug(desiredSlug, allRows, existing?.id);
if (existing) {
const updateNow = new Date();
const nextRow = {
title: this.normalizeTitle(parsed.metadata.title, slug, existing.title),
slug,
kind: this.normalizeKind(parsed.metadata.kind, existing.kind),
entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint, existing.entrypoint),
enabled: this.normalizeEnabled(parsed.metadata.enabled, existing.enabled),
version: this.normalizeVersion(parsed.metadata.version, existing.version),
filePath: absolutePath,
createdAt: this.normalizeDate(parsed.metadata.createdAt, existing.createdAt),
updatedAt: this.normalizeDate(parsed.metadata.updatedAt, updateNow),
};
await db.update(scripts)
.set(nextRow)
.where(and(eq(scripts.id, existing.id), eq(scripts.projectId, this.currentProjectId)));
const updatedRow = await this.getScriptRow(existing.id);
if (updatedRow) {
const updatedScript = await this.toScriptData(updatedRow);
this.emit('scriptUpdated', updatedScript);
}
if (previousAbsolutePath) {
scriptsByPath.delete(previousAbsolutePath);
}
scriptsByPath.set(absolutePath, {
...existing,
...nextRow,
});
updated += 1;
processedFiles += 1;
continue;
}
const desiredId = typeof parsed.metadata.id === 'string' && parsed.metadata.id.trim().length > 0
? parsed.metadata.id.trim()
: uuidv4();
const idExists = allRows.some((row) => row.id === desiredId);
const rowId = idExists ? uuidv4() : desiredId;
const now = new Date();
const newRow: NewScript = {
id: rowId,
projectId: this.currentProjectId,
slug,
title: this.normalizeTitle(parsed.metadata.title, slug),
kind: this.normalizeKind(parsed.metadata.kind),
entrypoint: this.normalizeEntrypoint(parsed.metadata.entrypoint),
enabled: this.normalizeEnabled(parsed.metadata.enabled),
version: this.normalizeVersion(parsed.metadata.version),
filePath: absolutePath,
createdAt: this.normalizeDate(parsed.metadata.createdAt, now),
updatedAt: this.normalizeDate(parsed.metadata.updatedAt, now),
};
await db.insert(scripts).values(newRow);
const createdRow = await this.getScriptRow(newRow.id);
if (createdRow) {
const createdScript = await this.toScriptData(createdRow);
this.emit('scriptCreated', createdScript);
}
scriptsByPath.set(absolutePath, newRow as Script);
created += 1;
processedFiles += 1;
}
return {
created,
updated,
deleted,
processedFiles,
};
}
private async getScriptRow(id: string): Promise<Script | null> {
const rows = await this.getAllScriptRows();
return rows.find((item) => item.id === id) || null;
@@ -240,6 +470,15 @@ export class ScriptEngine extends EventEmitter {
return path.join(this.getScriptsDir(), `${slug}.py`);
}
private normalizePathForCompare(filePath: string): string {
return path.resolve(filePath).replace(/\\/g, '/');
}
private isPythonScriptPath(value: string): boolean {
const normalized = value.replace(/\\/g, '/').replace(/^\.\//, '');
return normalized.startsWith('scripts/') && path.extname(normalized).toLowerCase() === '.py';
}
private normalizeSlug(value: string): string {
const normalized = value
.toLowerCase()
@@ -306,6 +545,183 @@ export class ScriptEngine extends EventEmitter {
return rawContent.replace(frontmatterDocstringPattern, '');
}
private parseScriptFile(rawContent: string): ParsedScriptFile {
const frontmatterDocstringPattern = /^(?:"""|''')\r?\n---\r?\n([\s\S]*?)\r?\n---\r?\n(?:"""|''')\r?\n?/;
const match = rawContent.match(frontmatterDocstringPattern);
if (!match) {
return {
metadata: {},
body: rawContent,
};
}
const metadataLines = (match[1] || '').split(/\r?\n/);
const metadata: ParsedScriptFile['metadata'] = {};
for (const rawLine of metadataLines) {
const line = rawLine.trim();
if (!line || line.startsWith('#')) {
continue;
}
const separatorIndex = line.indexOf(':');
if (separatorIndex <= 0) {
continue;
}
const key = line.slice(0, separatorIndex).trim();
const valueRaw = line.slice(separatorIndex + 1).trim();
const value = this.parseYamlScalar(valueRaw);
if (key === 'enabled') {
if (typeof value === 'boolean') {
metadata.enabled = value;
}
continue;
}
if (key === 'version') {
const parsed = Number(value);
if (Number.isFinite(parsed)) {
metadata.version = parsed;
}
continue;
}
if (
key === 'id' ||
key === 'projectId' ||
key === 'slug' ||
key === 'title' ||
key === 'kind' ||
key === 'entrypoint' ||
key === 'createdAt' ||
key === 'updatedAt'
) {
if (typeof value === 'string') {
metadata[key] = value;
}
}
}
return {
metadata,
body: rawContent.replace(frontmatterDocstringPattern, ''),
};
}
private parseYamlScalar(valueRaw: string): string | number | boolean {
if ((valueRaw.startsWith('"') && valueRaw.endsWith('"')) || (valueRaw.startsWith("'") && valueRaw.endsWith("'"))) {
return valueRaw.slice(1, -1)
.replace(/\\"/g, '"')
.replace(/\\\\/g, '\\');
}
if (valueRaw === 'true') {
return true;
}
if (valueRaw === 'false') {
return false;
}
const numeric = Number(valueRaw);
if (!Number.isNaN(numeric)) {
return numeric;
}
return valueRaw;
}
private normalizeKind(kind: string | undefined, fallback: ScriptKind = 'utility'): ScriptKind {
if (kind === 'macro' || kind === 'utility' || kind === 'transform') {
return kind;
}
return fallback;
}
private normalizeEntrypoint(entrypoint: string | undefined, fallback = 'render'): string {
if (typeof entrypoint === 'string' && entrypoint.trim().length > 0) {
return entrypoint.trim();
}
return fallback;
}
private normalizeEnabled(enabled: boolean | undefined, fallback = true): boolean {
if (typeof enabled === 'boolean') {
return enabled;
}
return fallback;
}
private normalizeVersion(version: number | undefined, fallback = 1): number {
if (typeof version === 'number' && Number.isFinite(version) && version > 0) {
return Math.floor(version);
}
return fallback;
}
private normalizeDate(value: string | undefined, fallback: Date): Date {
if (typeof value === 'string') {
const parsed = new Date(value);
if (!Number.isNaN(parsed.getTime())) {
return parsed;
}
}
return fallback;
}
private normalizeTitle(title: string | undefined, slug: string, fallback?: string): string {
if (typeof title === 'string' && title.trim().length > 0) {
return title.trim();
}
if (typeof fallback === 'string' && fallback.trim().length > 0) {
return fallback.trim();
}
return slug;
}
private async scanScriptFiles(dir: string): Promise<string[]> {
const results: string[] = [];
const scan = async (currentDir: string): Promise<void> => {
let entries: Array<{ name: string; isDirectory(): boolean; isFile(): boolean }> = [];
try {
entries = await fs.readdir(currentDir, { withFileTypes: true }) as Array<{ name: string; isDirectory(): boolean; isFile(): boolean }>;
} catch {
return;
}
for (const entry of entries) {
const fullPath = path.join(currentDir, entry.name);
if (entry.isDirectory()) {
await scan(fullPath);
continue;
}
if (entry.isFile() && path.extname(entry.name).toLowerCase() === '.py') {
results.push(fullPath);
}
}
};
await scan(dir);
return results;
}
private async readScriptFileWithMetadata(filePath: string): Promise<ParsedScriptFile | null> {
try {
const rawContent = await fs.readFile(filePath, 'utf-8');
return this.parseScriptFile(rawContent);
} catch (error) {
const fsError = error as NodeJS.ErrnoException;
if (fsError.code !== 'ENOENT') {
throw error;
}
return null;
}
}
private async readScriptBody(filePath: string): Promise<string> {
try {
const rawContent = await fs.readFile(filePath, 'utf-8');

32
src/main/ipc/handlers.ts
View File

@@ -185,8 +185,11 @@ export function registerIpcHandlers(): void {
return pullResult;
}
const changedPostFiles = await engine.getChangedPostFilesBetween(projectPath, beforeHead, afterHead);
if (changedPostFiles.length === 0) {
const [changedPostFiles, changedScriptFiles] = await Promise.all([
engine.getChangedPostFilesBetween(projectPath, beforeHead, afterHead),
engine.getChangedScriptFilesBetween(projectPath, beforeHead, afterHead),
]);
if (changedPostFiles.length === 0 && changedScriptFiles.length === 0) {
return pullResult;
}
@@ -194,15 +197,24 @@ export function registerIpcHandlers(): void {
const projectEngine = getProjectEngine();
const project = await projectEngine.getActiveProject();
const postEngine = getPostEngine();
const scriptEngine = getScriptEngine();
if (project) {
const dataDir = projectEngine.getDataDir(project.id, project.dataPath);
postEngine.setProjectContext(project.id, dataDir);
scriptEngine.setProjectContext(project.id, dataDir);
}
await postEngine.reconcilePublishedPostsFromGitChanges(projectPath, changedPostFiles);
await Promise.all([
changedPostFiles.length > 0
? postEngine.reconcilePublishedPostsFromGitChanges(projectPath, changedPostFiles)
: Promise.resolve(),
changedScriptFiles.length > 0
? scriptEngine.reconcileScriptsFromGitChanges(projectPath, changedScriptFiles)
: Promise.resolve(),
]);
} catch (error) {
console.error('Failed to reconcile published posts after git pull:', error);
console.error('Failed to reconcile published posts/scripts after git pull:', error);
}
return pullResult;
@@ -755,6 +767,18 @@ export function registerIpcHandlers(): void {
return engine.getAllScripts();
});
safeHandle('scripts:rebuildFromFiles', async () => {
const projectEngine = getProjectEngine();
const project = await projectEngine.getActiveProject();
const engine = getScriptEngine();
if (project) {
const dataDir = projectEngine.getDataDir(project.id, project.dataPath);
engine.setProjectContext(project.id, dataDir);
}
await engine.rebuildDatabaseFromFiles();
return true;
});
// ============ Task Handlers ============
safeHandle('tasks:getAll', async () => {

1
src/main/preload.ts
View File

@@ -108,6 +108,7 @@ export const electronAPI: ElectronAPI = {
delete: (id: string) => ipcRenderer.invoke('scripts:delete', id),
get: (id: string) => ipcRenderer.invoke('scripts:get', id),
getAll: () => ipcRenderer.invoke('scripts:getAll'),
rebuildFromFiles: () => ipcRenderer.invoke('scripts:rebuildFromFiles'),
},
// Post-Media Links

1
src/main/shared/electronApi.ts
View File

@@ -566,6 +566,7 @@ export interface ElectronAPI {
delete: (id: string) => Promise<boolean>;
get: (id: string) => Promise<ScriptData | null>;
getAll: () => Promise<ScriptData[]>;
rebuildFromFiles: () => Promise<void>;
};
postMedia: {
link: (postId: string, mediaId: string) => Promise<MediaLinkData>;