hugo/bDS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: rebuild from files now works

Browse Source
This commit is contained in:
hugo
2026-02-10 22:34:06 +01:00
parent e4cf0d333f
commit 7e4457c15d
5 changed files with 216 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/main/database/connection.ts
View File

@@ -137,7 +137,7 @@ export class DatabaseConnection {
id TEXT PRIMARY KEY,
project_id TEXT NOT NULL DEFAULT 'default',
title TEXT NOT NULL,
slug TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL,
excerpt TEXT,
status TEXT NOT NULL DEFAULT 'draft',
author TEXT,
@@ -211,6 +211,7 @@ export class DatabaseConnection {
CREATE INDEX IF NOT EXISTS idx_sync_log_status ON sync_log(status);
CREATE INDEX IF NOT EXISTS idx_post_links_source ON post_links(source_post_id);
CREATE INDEX IF NOT EXISTS idx_post_links_target ON post_links(target_post_id);
CREATE UNIQUE INDEX IF NOT EXISTS posts_project_slug_idx ON posts(project_id, slug);
`);
// Check if project_id column exists in posts table, add if missing (migration)
@@ -267,6 +268,83 @@ export class DatabaseConnection {
await this.localClient.execute("ALTER TABLE posts ADD COLUMN content TEXT");
}
// Migration: Update slug unique constraint to be project-scoped
// SQLite doesn't allow dropping column-level UNIQUE constraints, so we must recreate the table
// Check if the posts table has a column-level UNIQUE on slug (from the table definition)
const tableInfo = await this.localClient.execute(
"SELECT sql FROM sqlite_master WHERE type='table' AND name='posts'"
);
const tableSql = tableInfo.rows[0]?.sql as string || '';
const hasColumnLevelUnique = tableSql.includes('slug TEXT NOT NULL UNIQUE') ||
tableSql.includes('slug TEXT UNIQUE') ||
/slug\s+TEXT[^,]*UNIQUE/i.test(tableSql);
if (hasColumnLevelUnique) {
console.log('Migrating posts table to remove column-level UNIQUE constraint on slug...');
// Create new table without the UNIQUE constraint
await this.localClient.execute(`
CREATE TABLE IF NOT EXISTS posts_new (
id TEXT PRIMARY KEY,
project_id TEXT NOT NULL DEFAULT 'default',
title TEXT NOT NULL,
slug TEXT NOT NULL,
excerpt TEXT,
content TEXT,
status TEXT NOT NULL DEFAULT 'draft',
author TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
published_at INTEGER,
file_path TEXT NOT NULL DEFAULT '',
sync_status TEXT NOT NULL DEFAULT 'pending',
synced_at INTEGER,
checksum TEXT,
tags TEXT,
categories TEXT,
published_title TEXT,
published_content TEXT,
published_tags TEXT,
published_categories TEXT,
published_excerpt TEXT
)
`);
// Copy data
await this.localClient.execute(`
INSERT INTO posts_new
SELECT id, project_id, title, slug, excerpt, content, status, author,
created_at, updated_at, published_at, file_path, sync_status,
synced_at, checksum, tags, categories, published_title,
published_content, published_tags, published_categories, published_excerpt
FROM posts
`);
// Drop old table and rename new one
await this.localClient.execute('DROP TABLE posts');
await this.localClient.execute('ALTER TABLE posts_new RENAME TO posts');
// Recreate indexes
await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_slug ON posts(slug)');
await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_status ON posts(status)');
await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_sync_status ON posts(sync_status)');
await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_created_at ON posts(created_at)');
await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_project_id ON posts(project_id)');
await this.localClient.execute('CREATE UNIQUE INDEX IF NOT EXISTS posts_project_slug_idx ON posts(project_id, slug)');
console.log('Posts table migration complete');
} else {
// Just ensure the composite unique index exists
const compositeSlugIndex = await this.localClient.execute(
"SELECT name FROM sqlite_master WHERE type='index' AND name='posts_project_slug_idx' AND tbl_name='posts'"
);
if (compositeSlugIndex.rows.length === 0) {
await this.localClient.execute(
"CREATE UNIQUE INDEX posts_project_slug_idx ON posts(project_id, slug)"
);
}
}
// Create FTS5 virtual table for full-text search
await this.localClient.execute(`
CREATE VIRTUAL TABLE IF NOT EXISTS posts_fts USING fts5(

9
src/main/database/schema.ts
View File

@@ -1,4 +1,4 @@
import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core';
import { sqliteTable, text, integer, uniqueIndex } from 'drizzle-orm/sqlite-core';
// Projects table - stores blog projects/websites
export const projects = sqliteTable('projects', {
@@ -20,7 +20,7 @@ export const posts = sqliteTable('posts', {
projectId: text('project_id').notNull(),
id: text('id').primaryKey(),
title: text('title').notNull(),
slug: text('slug').notNull().unique(),
slug: text('slug').notNull(),
excerpt: text('excerpt'),
content: text('content'), // Draft body text (null/empty when published — content is in the file)
status: text('status', { enum: ['draft', 'published', 'archived'] }).notNull().default('draft'),
@@ -40,7 +40,10 @@ export const posts = sqliteTable('posts', {
publishedTags: text('published_tags'),
publishedCategories: text('published_categories'),
publishedExcerpt: text('published_excerpt'),
});
}, (table) => ({
// Composite unique index: slug must be unique within each project
projectSlugIdx: uniqueIndex('posts_project_slug_idx').on(table.projectId, table.slug),
}));
// Media table - stores metadata for images and other media
export const media = sqliteTable('media', {

79
src/main/engine/MediaEngine.ts
View File

@@ -238,6 +238,14 @@ export class MediaEngine extends EventEmitter {
private async readSidecarFile(sidecarPath: string): Promise<MediaMetadata | null> {
try {
// Check if file exists first to avoid noisy errors
try {
await fs.access(sidecarPath);
} catch {
// File doesn't exist - this is expected when DB has stale paths
return null;
}
const content = await fs.readFile(sidecarPath, 'utf-8');
const lines = content.split('\n');
@@ -309,7 +317,7 @@ export class MediaEngine extends EventEmitter {
return metadata as MediaMetadata;
} catch (error) {
console.error(`Failed to read sidecar file: ${sidecarPath}`, error);
console.error(`Failed to parse sidecar file: ${sidecarPath}`, error);
return null;
}
}
@@ -535,7 +543,16 @@ export class MediaEngine extends EventEmitter {
execute: async (onProgress) => {
const db = getDatabase().getLocal();
onProgress(0, 'Scanning media directory...');
onProgress(0, 'Deleting existing media for project...');
// Delete all media for the current project - clean slate rebuild
const existingMedia = await db.select({ id: media.id }).from(media).where(eq(media.projectId, this.currentProjectId)).all();
if (existingMedia.length > 0) {
await db.delete(media).where(eq(media.projectId, this.currentProjectId));
console.log(`Deleted ${existingMedia.length} existing media record(s) for project ${this.currentProjectId}`);
}
onProgress(5, 'Scanning media directory...');
// Recursively find all .meta files in the media directory tree
const metaFiles: string[] = [];
@@ -580,44 +597,26 @@ export class MediaEngine extends EventEmitter {
const checksum = this.calculateChecksum(buffer);
const filename = path.basename(mediaFilePath);
const existing = await db.select().from(media).where(eq(media.id, metadata.id)).get();
if (existing) {
await db.update(media)
.set({
originalName: metadata.originalName,
mimeType: metadata.mimeType,
size: stats.size,
width: metadata.width,
height: metadata.height,
alt: metadata.alt,
caption: metadata.caption,
updatedAt: new Date(metadata.updatedAt),
checksum,
tags: JSON.stringify(metadata.tags),
})
.where(eq(media.id, metadata.id));
} else {
await db.insert(media).values({
id: metadata.id,
projectId: this.currentProjectId,
filename,
originalName: metadata.originalName,
mimeType: metadata.mimeType,
size: stats.size,
width: metadata.width,
height: metadata.height,
alt: metadata.alt,
caption: metadata.caption,
filePath: mediaFilePath,
sidecarPath,
createdAt: new Date(metadata.createdAt),
updatedAt: new Date(metadata.updatedAt),
syncStatus: 'pending',
checksum,
tags: JSON.stringify(metadata.tags),
});
}
// Insert fresh - we deleted all records at the start
await db.insert(media).values({
id: metadata.id,
projectId: this.currentProjectId,
filename,
originalName: metadata.originalName,
mimeType: metadata.mimeType,
size: stats.size,
width: metadata.width,
height: metadata.height,
alt: metadata.alt,
caption: metadata.caption,
filePath: mediaFilePath,
sidecarPath,
createdAt: new Date(metadata.createdAt),
updatedAt: new Date(metadata.updatedAt),
syncStatus: 'pending',
checksum,
tags: JSON.stringify(metadata.tags),
});
} catch (error) {
console.error(`Media file not found for sidecar: ${sidecarPath}`, error);
}

97
src/main/engine/PostEngine.ts
View File

@@ -4,7 +4,7 @@ import * as fs from 'fs/promises';
import * as path from 'path';
import * as crypto from 'crypto';
import matter from 'gray-matter';
import { eq, and, desc, gte, lte, like } from 'drizzle-orm';
import { eq, and, desc, gte, lte, like, inArray } from 'drizzle-orm';
import { app } from 'electron';
import { getDatabase } from '../database';
import { posts, Post, NewPost, postLinks } from '../database/schema';
@@ -193,6 +193,14 @@ export class PostEngine extends EventEmitter {
private async readPostFile(filePath: string): Promise<PostData | null> {
try {
// Check if file exists first to avoid noisy errors
try {
await fs.access(filePath);
} catch {
// File doesn't exist - this is expected when DB has stale paths
return null;
}
const content = await fs.readFile(filePath, 'utf-8');
const { data, content: body } = matter(content);
const metadata = data as PostMetadata;
@@ -213,7 +221,7 @@ export class PostEngine extends EventEmitter {
categories: metadata.categories || [],
};
} catch (error) {
console.error(`Failed to read post file: ${filePath}`, error);
console.error(`Failed to parse post file: ${filePath}`, error);
return null;
}
}
@@ -834,7 +842,27 @@ export class PostEngine extends EventEmitter {
const db = getDatabase().getLocal();
const client = getDatabase().getLocalClient();
onProgress(0, 'Scanning posts directory...');
onProgress(0, 'Deleting existing posts for project...');
// Delete all posts for the current project - clean slate rebuild
const existingPosts = await db.select({ id: posts.id }).from(posts).where(eq(posts.projectId, this.currentProjectId)).all();
if (existingPosts.length > 0) {
const postIds = existingPosts.map(p => p.id);
// Delete FTS entries first
if (client) {
for (const post of existingPosts) {
await client.execute({ sql: 'DELETE FROM posts_fts WHERE id = ?', args: [post.id] });
}
}
// Delete post links where source or target is in the posts being deleted
await db.delete(postLinks).where(inArray(postLinks.sourcePostId, postIds));
await db.delete(postLinks).where(inArray(postLinks.targetPostId, postIds));
// Delete posts
await db.delete(posts).where(eq(posts.projectId, this.currentProjectId));
console.log(`Deleted ${existingPosts.length} existing post(s) for project ${this.currentProjectId}`);
}
onProgress(5, 'Scanning posts directory...');
// Recursively find all .md files in the posts directory tree
const mdFiles: string[] = [];
@@ -863,6 +891,9 @@ export class PostEngine extends EventEmitter {
onProgress(10, `Found ${mdFiles.length} post files`);
// Track slugs to detect duplicates
const insertedSlugs = new Map<string, string>(); // slug -> filePath
for (let i = 0; i < mdFiles.length; i++) {
const filePath = mdFiles[i];
const fileName = path.basename(filePath);
@@ -872,33 +903,22 @@ export class PostEngine extends EventEmitter {
const postData = await this.readPostFile(filePath);
if (postData) {
const existing = await db.select().from(posts).where(eq(posts.id, postData.id)).get();
const checksum = this.calculateChecksum(postData.content);
try {
const projectId = postData.projectId || this.currentProjectId;
const slugKey = `${projectId}:${postData.slug}`;
if (existing) {
// Only update if no active draft content in DB (don't overwrite edits)
if (!existing.content) {
await db.update(posts)
.set({
title: postData.title,
slug: postData.slug,
excerpt: postData.excerpt,
content: null, // Content lives in the file, not DB
status: 'published', // Files on disk = published
author: postData.author,
updatedAt: postData.updatedAt,
publishedAt: postData.publishedAt,
filePath,
checksum,
tags: JSON.stringify(postData.tags),
categories: JSON.stringify(postData.categories),
})
.where(eq(posts.id, postData.id));
// Check for duplicate slugs
if (insertedSlugs.has(slugKey)) {
console.error(`Duplicate slug "${postData.slug}" found. File "${filePath}" duplicates "${insertedSlugs.get(slugKey)}". Skipping.`);
continue;
}
} else {
const checksum = this.calculateChecksum(postData.content);
// Insert fresh - we deleted all records at the start
await db.insert(posts).values({
id: postData.id,
projectId: postData.projectId || this.currentProjectId,
projectId,
title: postData.title,
slug: postData.slug,
excerpt: postData.excerpt,
@@ -914,15 +934,24 @@ export class PostEngine extends EventEmitter {
tags: JSON.stringify(postData.tags),
categories: JSON.stringify(postData.categories),
});
}
// Update FTS index (use file content for search)
if (client) {
await client.execute({ sql: 'DELETE FROM posts_fts WHERE id = ?', args: [postData.id] });
await client.execute({
sql: 'INSERT INTO posts_fts (id, title, content, excerpt, tags, categories) VALUES (?, ?, ?, ?, ?, ?)',
args: [postData.id, postData.title, postData.content, postData.excerpt || '', postData.tags.join(' '), postData.categories.join(' ')],
});
insertedSlugs.set(slugKey, filePath);
// Update FTS index (use file content for search)
if (client) {
await client.execute({ sql: 'DELETE FROM posts_fts WHERE id = ?', args: [postData.id] });
await client.execute({
sql: 'INSERT INTO posts_fts (id, title, content, excerpt, tags, categories) VALUES (?, ?, ?, ?, ?, ?)',
args: [postData.id, postData.title, postData.content, postData.excerpt || '', postData.tags.join(' '), postData.categories.join(' ')],
});
}
} catch (error: any) {
// Handle constraint violations and other errors gracefully
if (error?.code === 'SQLITE_CONSTRAINT_UNIQUE') {
console.error(`Failed to insert post "${postData.title}" from ${filePath}: Unique constraint violation (likely slug conflict)`);
} else {
console.error(`Failed to process post from ${filePath}:`, error);
}
}
}
}

26
tests/engine/PostEngine.test.ts
View File

@@ -1241,6 +1241,9 @@ const code = 'example';
mockDirent('other.txt'),
] as any);
// Mock access to allow file reads
vi.mocked(fs.access).mockResolvedValue(undefined);
// Mock readFile to return valid post content
vi.mocked(fs.readFile).mockImplementation(async (filePath: any) => {
if (filePath.includes('post1.md')) {
@@ -1313,11 +1316,14 @@ Content 2`;
expect(fs.mkdir).toHaveBeenCalled();
});
it('should update existing posts in database', async () => {
it('should delete all existing posts before inserting fresh from files', async () => {
const fs = await import('fs/promises');
vi.mocked(fs.readdir).mockResolvedValueOnce([mockDirent('existing.md')] as any);
// Mock access to allow file reads
vi.mocked(fs.access).mockResolvedValue(undefined);
vi.mocked(fs.readFile).mockResolvedValueOnce(`---
id: existing-id
projectId: default
@@ -1331,22 +1337,22 @@ categories: []
---
Updated content`);
// Mock that post exists in database
// Mock that project has existing posts to delete
vi.mocked(mockLocalDb.select).mockImplementation(() => {
const chain = createSelectChain();
chain.where = vi.fn().mockReturnValue({
...chain,
get: vi.fn().mockResolvedValue({
id: 'existing-id',
title: 'Old Title',
}),
all: vi.fn().mockResolvedValue([{ id: 'existing-id' }]),
get: vi.fn().mockResolvedValue(null),
});
return chain;
});
await postEngine.rebuildDatabaseFromFiles();
expect(mockLocalDb.update).toHaveBeenCalled();
// Should delete existing posts first, then insert fresh
expect(mockLocalDb.delete).toHaveBeenCalled();
expect(mockLocalDb.insert).toHaveBeenCalled();
});
it('should insert new posts not in database', async () => {
@@ -1354,6 +1360,9 @@ Updated content`);
vi.mocked(fs.readdir).mockResolvedValueOnce([mockDirent('new-post.md')] as any);
// Mock access to allow file reads
vi.mocked(fs.access).mockResolvedValue(undefined);
vi.mocked(fs.readFile).mockResolvedValueOnce(`---
id: new-post-id
projectId: default
@@ -1387,6 +1396,9 @@ New content`);
vi.mocked(fs.readdir).mockResolvedValueOnce([mockDirent('fts-test.md')] as any);
// Mock access to allow file reads
vi.mocked(fs.access).mockResolvedValue(undefined);
vi.mocked(fs.readFile).mockResolvedValueOnce(`---
id: fts-test-id
projectId: default