diff --git a/src/main/database/connection.ts b/src/main/database/connection.ts index eb4fcd3..b05dec9 100644 --- a/src/main/database/connection.ts +++ b/src/main/database/connection.ts @@ -137,7 +137,7 @@ export class DatabaseConnection { id TEXT PRIMARY KEY, project_id TEXT NOT NULL DEFAULT 'default', title TEXT NOT NULL, - slug TEXT NOT NULL UNIQUE, + slug TEXT NOT NULL, excerpt TEXT, status TEXT NOT NULL DEFAULT 'draft', author TEXT, @@ -211,6 +211,7 @@ export class DatabaseConnection { CREATE INDEX IF NOT EXISTS idx_sync_log_status ON sync_log(status); CREATE INDEX IF NOT EXISTS idx_post_links_source ON post_links(source_post_id); CREATE INDEX IF NOT EXISTS idx_post_links_target ON post_links(target_post_id); + CREATE UNIQUE INDEX IF NOT EXISTS posts_project_slug_idx ON posts(project_id, slug); `); // Check if project_id column exists in posts table, add if missing (migration) @@ -267,6 +268,83 @@ export class DatabaseConnection { await this.localClient.execute("ALTER TABLE posts ADD COLUMN content TEXT"); } + // Migration: Update slug unique constraint to be project-scoped + // SQLite doesn't allow dropping column-level UNIQUE constraints, so we must recreate the table + // Check if the posts table has a column-level UNIQUE on slug (from the table definition) + const tableInfo = await this.localClient.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='posts'" + ); + const tableSql = tableInfo.rows[0]?.sql as string || ''; + const hasColumnLevelUnique = tableSql.includes('slug TEXT NOT NULL UNIQUE') || + tableSql.includes('slug TEXT UNIQUE') || + /slug\s+TEXT[^,]*UNIQUE/i.test(tableSql); + + if (hasColumnLevelUnique) { + console.log('Migrating posts table to remove column-level UNIQUE constraint on slug...'); + + // Create new table without the UNIQUE constraint + await this.localClient.execute(` + CREATE TABLE IF NOT EXISTS posts_new ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL DEFAULT 'default', + title TEXT NOT NULL, + slug TEXT NOT NULL, + excerpt TEXT, + content TEXT, + status TEXT NOT NULL DEFAULT 'draft', + author TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + published_at INTEGER, + file_path TEXT NOT NULL DEFAULT '', + sync_status TEXT NOT NULL DEFAULT 'pending', + synced_at INTEGER, + checksum TEXT, + tags TEXT, + categories TEXT, + published_title TEXT, + published_content TEXT, + published_tags TEXT, + published_categories TEXT, + published_excerpt TEXT + ) + `); + + // Copy data + await this.localClient.execute(` + INSERT INTO posts_new + SELECT id, project_id, title, slug, excerpt, content, status, author, + created_at, updated_at, published_at, file_path, sync_status, + synced_at, checksum, tags, categories, published_title, + published_content, published_tags, published_categories, published_excerpt + FROM posts + `); + + // Drop old table and rename new one + await this.localClient.execute('DROP TABLE posts'); + await this.localClient.execute('ALTER TABLE posts_new RENAME TO posts'); + + // Recreate indexes + await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_slug ON posts(slug)'); + await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_status ON posts(status)'); + await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_sync_status ON posts(sync_status)'); + await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_created_at ON posts(created_at)'); + await this.localClient.execute('CREATE INDEX IF NOT EXISTS idx_posts_project_id ON posts(project_id)'); + await this.localClient.execute('CREATE UNIQUE INDEX IF NOT EXISTS posts_project_slug_idx ON posts(project_id, slug)'); + + console.log('Posts table migration complete'); + } else { + // Just ensure the composite unique index exists + const compositeSlugIndex = await this.localClient.execute( + "SELECT name FROM sqlite_master WHERE type='index' AND name='posts_project_slug_idx' AND tbl_name='posts'" + ); + if (compositeSlugIndex.rows.length === 0) { + await this.localClient.execute( + "CREATE UNIQUE INDEX posts_project_slug_idx ON posts(project_id, slug)" + ); + } + } + // Create FTS5 virtual table for full-text search await this.localClient.execute(` CREATE VIRTUAL TABLE IF NOT EXISTS posts_fts USING fts5( diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index 0770766..1509324 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core'; +import { sqliteTable, text, integer, uniqueIndex } from 'drizzle-orm/sqlite-core'; // Projects table - stores blog projects/websites export const projects = sqliteTable('projects', { @@ -20,7 +20,7 @@ export const posts = sqliteTable('posts', { projectId: text('project_id').notNull(), id: text('id').primaryKey(), title: text('title').notNull(), - slug: text('slug').notNull().unique(), + slug: text('slug').notNull(), excerpt: text('excerpt'), content: text('content'), // Draft body text (null/empty when published — content is in the file) status: text('status', { enum: ['draft', 'published', 'archived'] }).notNull().default('draft'), @@ -40,7 +40,10 @@ export const posts = sqliteTable('posts', { publishedTags: text('published_tags'), publishedCategories: text('published_categories'), publishedExcerpt: text('published_excerpt'), -}); +}, (table) => ({ + // Composite unique index: slug must be unique within each project + projectSlugIdx: uniqueIndex('posts_project_slug_idx').on(table.projectId, table.slug), +})); // Media table - stores metadata for images and other media export const media = sqliteTable('media', { diff --git a/src/main/engine/MediaEngine.ts b/src/main/engine/MediaEngine.ts index 273a0e5..052281d 100644 --- a/src/main/engine/MediaEngine.ts +++ b/src/main/engine/MediaEngine.ts @@ -238,6 +238,14 @@ export class MediaEngine extends EventEmitter { private async readSidecarFile(sidecarPath: string): Promise { try { + // Check if file exists first to avoid noisy errors + try { + await fs.access(sidecarPath); + } catch { + // File doesn't exist - this is expected when DB has stale paths + return null; + } + const content = await fs.readFile(sidecarPath, 'utf-8'); const lines = content.split('\n'); @@ -309,7 +317,7 @@ export class MediaEngine extends EventEmitter { return metadata as MediaMetadata; } catch (error) { - console.error(`Failed to read sidecar file: ${sidecarPath}`, error); + console.error(`Failed to parse sidecar file: ${sidecarPath}`, error); return null; } } @@ -535,7 +543,16 @@ export class MediaEngine extends EventEmitter { execute: async (onProgress) => { const db = getDatabase().getLocal(); - onProgress(0, 'Scanning media directory...'); + onProgress(0, 'Deleting existing media for project...'); + + // Delete all media for the current project - clean slate rebuild + const existingMedia = await db.select({ id: media.id }).from(media).where(eq(media.projectId, this.currentProjectId)).all(); + if (existingMedia.length > 0) { + await db.delete(media).where(eq(media.projectId, this.currentProjectId)); + console.log(`Deleted ${existingMedia.length} existing media record(s) for project ${this.currentProjectId}`); + } + + onProgress(5, 'Scanning media directory...'); // Recursively find all .meta files in the media directory tree const metaFiles: string[] = []; @@ -580,44 +597,26 @@ export class MediaEngine extends EventEmitter { const checksum = this.calculateChecksum(buffer); const filename = path.basename(mediaFilePath); - const existing = await db.select().from(media).where(eq(media.id, metadata.id)).get(); - - if (existing) { - await db.update(media) - .set({ - originalName: metadata.originalName, - mimeType: metadata.mimeType, - size: stats.size, - width: metadata.width, - height: metadata.height, - alt: metadata.alt, - caption: metadata.caption, - updatedAt: new Date(metadata.updatedAt), - checksum, - tags: JSON.stringify(metadata.tags), - }) - .where(eq(media.id, metadata.id)); - } else { - await db.insert(media).values({ - id: metadata.id, - projectId: this.currentProjectId, - filename, - originalName: metadata.originalName, - mimeType: metadata.mimeType, - size: stats.size, - width: metadata.width, - height: metadata.height, - alt: metadata.alt, - caption: metadata.caption, - filePath: mediaFilePath, - sidecarPath, - createdAt: new Date(metadata.createdAt), - updatedAt: new Date(metadata.updatedAt), - syncStatus: 'pending', - checksum, - tags: JSON.stringify(metadata.tags), - }); - } + // Insert fresh - we deleted all records at the start + await db.insert(media).values({ + id: metadata.id, + projectId: this.currentProjectId, + filename, + originalName: metadata.originalName, + mimeType: metadata.mimeType, + size: stats.size, + width: metadata.width, + height: metadata.height, + alt: metadata.alt, + caption: metadata.caption, + filePath: mediaFilePath, + sidecarPath, + createdAt: new Date(metadata.createdAt), + updatedAt: new Date(metadata.updatedAt), + syncStatus: 'pending', + checksum, + tags: JSON.stringify(metadata.tags), + }); } catch (error) { console.error(`Media file not found for sidecar: ${sidecarPath}`, error); } diff --git a/src/main/engine/PostEngine.ts b/src/main/engine/PostEngine.ts index 265d0f8..1abc780 100644 --- a/src/main/engine/PostEngine.ts +++ b/src/main/engine/PostEngine.ts @@ -4,7 +4,7 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import * as crypto from 'crypto'; import matter from 'gray-matter'; -import { eq, and, desc, gte, lte, like } from 'drizzle-orm'; +import { eq, and, desc, gte, lte, like, inArray } from 'drizzle-orm'; import { app } from 'electron'; import { getDatabase } from '../database'; import { posts, Post, NewPost, postLinks } from '../database/schema'; @@ -193,6 +193,14 @@ export class PostEngine extends EventEmitter { private async readPostFile(filePath: string): Promise { try { + // Check if file exists first to avoid noisy errors + try { + await fs.access(filePath); + } catch { + // File doesn't exist - this is expected when DB has stale paths + return null; + } + const content = await fs.readFile(filePath, 'utf-8'); const { data, content: body } = matter(content); const metadata = data as PostMetadata; @@ -213,7 +221,7 @@ export class PostEngine extends EventEmitter { categories: metadata.categories || [], }; } catch (error) { - console.error(`Failed to read post file: ${filePath}`, error); + console.error(`Failed to parse post file: ${filePath}`, error); return null; } } @@ -833,9 +841,29 @@ export class PostEngine extends EventEmitter { execute: async (onProgress) => { const db = getDatabase().getLocal(); const client = getDatabase().getLocalClient(); - - onProgress(0, 'Scanning posts directory...'); - + + onProgress(0, 'Deleting existing posts for project...'); + + // Delete all posts for the current project - clean slate rebuild + const existingPosts = await db.select({ id: posts.id }).from(posts).where(eq(posts.projectId, this.currentProjectId)).all(); + if (existingPosts.length > 0) { + const postIds = existingPosts.map(p => p.id); + // Delete FTS entries first + if (client) { + for (const post of existingPosts) { + await client.execute({ sql: 'DELETE FROM posts_fts WHERE id = ?', args: [post.id] }); + } + } + // Delete post links where source or target is in the posts being deleted + await db.delete(postLinks).where(inArray(postLinks.sourcePostId, postIds)); + await db.delete(postLinks).where(inArray(postLinks.targetPostId, postIds)); + // Delete posts + await db.delete(posts).where(eq(posts.projectId, this.currentProjectId)); + console.log(`Deleted ${existingPosts.length} existing post(s) for project ${this.currentProjectId}`); + } + + onProgress(5, 'Scanning posts directory...'); + // Recursively find all .md files in the posts directory tree const mdFiles: string[] = []; const scanDir = async (dir: string) => { @@ -860,45 +888,37 @@ export class PostEngine extends EventEmitter { // Already exists } await scanDir(postsBaseDir); - + onProgress(10, `Found ${mdFiles.length} post files`); - + + // Track slugs to detect duplicates + const insertedSlugs = new Map(); // slug -> filePath + for (let i = 0; i < mdFiles.length; i++) { const filePath = mdFiles[i]; const fileName = path.basename(filePath); - + onProgress(10 + (80 * (i / mdFiles.length)), `Processing ${fileName}...`); - + const postData = await this.readPostFile(filePath); - + if (postData) { - const existing = await db.select().from(posts).where(eq(posts.id, postData.id)).get(); - const checksum = this.calculateChecksum(postData.content); - - if (existing) { - // Only update if no active draft content in DB (don't overwrite edits) - if (!existing.content) { - await db.update(posts) - .set({ - title: postData.title, - slug: postData.slug, - excerpt: postData.excerpt, - content: null, // Content lives in the file, not DB - status: 'published', // Files on disk = published - author: postData.author, - updatedAt: postData.updatedAt, - publishedAt: postData.publishedAt, - filePath, - checksum, - tags: JSON.stringify(postData.tags), - categories: JSON.stringify(postData.categories), - }) - .where(eq(posts.id, postData.id)); + try { + const projectId = postData.projectId || this.currentProjectId; + const slugKey = `${projectId}:${postData.slug}`; + + // Check for duplicate slugs + if (insertedSlugs.has(slugKey)) { + console.error(`Duplicate slug "${postData.slug}" found. File "${filePath}" duplicates "${insertedSlugs.get(slugKey)}". Skipping.`); + continue; } - } else { + + const checksum = this.calculateChecksum(postData.content); + + // Insert fresh - we deleted all records at the start await db.insert(posts).values({ id: postData.id, - projectId: postData.projectId || this.currentProjectId, + projectId, title: postData.title, slug: postData.slug, excerpt: postData.excerpt, @@ -914,24 +934,33 @@ export class PostEngine extends EventEmitter { tags: JSON.stringify(postData.tags), categories: JSON.stringify(postData.categories), }); - } - // Update FTS index (use file content for search) - if (client) { - await client.execute({ sql: 'DELETE FROM posts_fts WHERE id = ?', args: [postData.id] }); - await client.execute({ - sql: 'INSERT INTO posts_fts (id, title, content, excerpt, tags, categories) VALUES (?, ?, ?, ?, ?, ?)', - args: [postData.id, postData.title, postData.content, postData.excerpt || '', postData.tags.join(' '), postData.categories.join(' ')], - }); + insertedSlugs.set(slugKey, filePath); + + // Update FTS index (use file content for search) + if (client) { + await client.execute({ sql: 'DELETE FROM posts_fts WHERE id = ?', args: [postData.id] }); + await client.execute({ + sql: 'INSERT INTO posts_fts (id, title, content, excerpt, tags, categories) VALUES (?, ?, ?, ?, ?, ?)', + args: [postData.id, postData.title, postData.content, postData.excerpt || '', postData.tags.join(' '), postData.categories.join(' ')], + }); + } + } catch (error: any) { + // Handle constraint violations and other errors gracefully + if (error?.code === 'SQLITE_CONSTRAINT_UNIQUE') { + console.error(`Failed to insert post "${postData.title}" from ${filePath}: Unique constraint violation (likely slug conflict)`); + } else { + console.error(`Failed to process post from ${filePath}:`, error); + } } } } - + onProgress(100, 'Database rebuild complete'); this.emit('databaseRebuilt'); }, }; - + await taskManager.runTask(task); } diff --git a/tests/engine/PostEngine.test.ts b/tests/engine/PostEngine.test.ts index cd03187..faa0b0f 100644 --- a/tests/engine/PostEngine.test.ts +++ b/tests/engine/PostEngine.test.ts @@ -1240,6 +1240,9 @@ const code = 'example'; mockDirent('post2.md'), mockDirent('other.txt'), ] as any); + + // Mock access to allow file reads + vi.mocked(fs.access).mockResolvedValue(undefined); // Mock readFile to return valid post content vi.mocked(fs.readFile).mockImplementation(async (filePath: any) => { @@ -1313,10 +1316,13 @@ Content 2`; expect(fs.mkdir).toHaveBeenCalled(); }); - it('should update existing posts in database', async () => { + it('should delete all existing posts before inserting fresh from files', async () => { const fs = await import('fs/promises'); vi.mocked(fs.readdir).mockResolvedValueOnce([mockDirent('existing.md')] as any); + + // Mock access to allow file reads + vi.mocked(fs.access).mockResolvedValue(undefined); vi.mocked(fs.readFile).mockResolvedValueOnce(`--- id: existing-id @@ -1331,28 +1337,31 @@ categories: [] --- Updated content`); - // Mock that post exists in database + // Mock that project has existing posts to delete vi.mocked(mockLocalDb.select).mockImplementation(() => { const chain = createSelectChain(); chain.where = vi.fn().mockReturnValue({ ...chain, - get: vi.fn().mockResolvedValue({ - id: 'existing-id', - title: 'Old Title', - }), + all: vi.fn().mockResolvedValue([{ id: 'existing-id' }]), + get: vi.fn().mockResolvedValue(null), }); return chain; }); await postEngine.rebuildDatabaseFromFiles(); - expect(mockLocalDb.update).toHaveBeenCalled(); + // Should delete existing posts first, then insert fresh + expect(mockLocalDb.delete).toHaveBeenCalled(); + expect(mockLocalDb.insert).toHaveBeenCalled(); }); it('should insert new posts not in database', async () => { const fs = await import('fs/promises'); vi.mocked(fs.readdir).mockResolvedValueOnce([mockDirent('new-post.md')] as any); + + // Mock access to allow file reads + vi.mocked(fs.access).mockResolvedValue(undefined); vi.mocked(fs.readFile).mockResolvedValueOnce(`--- id: new-post-id @@ -1386,6 +1395,9 @@ New content`); const fs = await import('fs/promises'); vi.mocked(fs.readdir).mockResolvedValueOnce([mockDirent('fts-test.md')] as any); + + // Mock access to allow file reads + vi.mocked(fs.access).mockResolvedValue(undefined); vi.mocked(fs.readFile).mockResolvedValueOnce(`--- id: fts-test-id