From 6bbf13dd41d212240caa791bece45593decb0b39 Mon Sep 17 00:00:00 2001 From: hugo Date: Tue, 10 Feb 2026 22:48:13 +0100 Subject: [PATCH] fix: better handling of many posts --- src/main/engine/PostEngine.ts | 63 +++++++++++++++++-- src/main/engine/index.ts | 2 +- src/main/ipc/handlers.ts | 18 +++++- src/main/main.ts | 64 ++++++++++++++++++- src/main/preload.ts | 4 +- src/renderer/App.tsx | 9 +-- src/renderer/components/Editor/Editor.tsx | 68 +++++++++++++++++++-- src/renderer/components/Sidebar/Sidebar.css | 28 +++++++++ src/renderer/components/Sidebar/Sidebar.tsx | 34 ++++++++++- src/renderer/store/appStore.ts | 19 +++++- 10 files changed, 285 insertions(+), 24 deletions(-) diff --git a/src/main/engine/PostEngine.ts b/src/main/engine/PostEngine.ts index 1abc780..17d7110 100644 --- a/src/main/engine/PostEngine.ts +++ b/src/main/engine/PostEngine.ts @@ -60,6 +60,17 @@ export interface PostFilter { month?: number; } +export interface PaginatedResult { + items: T[]; + hasMore: boolean; + total: number; +} + +export interface PaginationOptions { + limit?: number; + offset?: number; +} + export class PostEngine extends EventEmitter { private currentProjectId: string = 'default'; @@ -455,7 +466,49 @@ export class PostEngine extends EventEmitter { return this.dbRowToPostData(dbPost, ''); } - async getAllPosts(): Promise { + async getAllPosts(options?: PaginationOptions): Promise> { + const db = getDatabase().getLocal(); + const limit = options?.limit ?? 500; + const offset = options?.offset ?? 0; + + // Get total count for hasMore calculation + const countResult = await db + .select({ count: posts.id }) + .from(posts) + .where(eq(posts.projectId, this.currentProjectId)) + .all(); + const total = countResult.length; + + const dbPosts = await db + .select() + .from(posts) + .where(eq(posts.projectId, this.currentProjectId)) + .orderBy(desc(posts.createdAt)) + .limit(limit) + .offset(offset) + .all(); + + const items: PostData[] = []; + + for (const dbPost of dbPosts) { + const postData = await this.getPost(dbPost.id); + if (postData) { + items.push(postData); + } + } + + return { + items, + hasMore: offset + items.length < total, + total, + }; + } + + /** + * Internal method to get all posts without pagination. + * Used by methods that need to iterate over all posts (search, tags, categories, etc.) + */ + private async getAllPostsUnpaginated(): Promise { const db = getDatabase().getLocal(); const dbPosts = await db .select() @@ -574,7 +627,7 @@ export class PostEngine extends EventEmitter { args: [query], }); - const projectPosts = await this.getAllPosts(); + const projectPosts = await this.getAllPostsUnpaginated(); const projectPostIds = new Set(projectPosts.map(p => p.id)); return result.rows @@ -594,7 +647,7 @@ export class PostEngine extends EventEmitter { } async getAvailableTags(): Promise { - const allPosts = await this.getAllPosts(); + const allPosts = await this.getAllPostsUnpaginated(); const tags = new Set(); for (const post of allPosts) { for (const tag of post.tags) { @@ -605,7 +658,7 @@ export class PostEngine extends EventEmitter { } async getAvailableCategories(): Promise { - const allPosts = await this.getAllPosts(); + const allPosts = await this.getAllPostsUnpaginated(); const categories = new Set(); for (const post of allPosts) { for (const cat of post.categories) { @@ -616,7 +669,7 @@ export class PostEngine extends EventEmitter { } async getPostsByYearMonth(): Promise<{ year: number; month: number; count: number }[]> { - const allPosts = await this.getAllPosts(); + const allPosts = await this.getAllPostsUnpaginated(); const counts = new Map(); for (const post of allPosts) { diff --git a/src/main/engine/index.ts b/src/main/engine/index.ts index c1179ed..24fadcd 100644 --- a/src/main/engine/index.ts +++ b/src/main/engine/index.ts @@ -1,5 +1,5 @@ export { TaskManager, taskManager, type Task, type TaskProgress, type TaskStatus } from './TaskManager'; -export { PostEngine, getPostEngine, type PostData, type PostFilter, type SearchResult } from './PostEngine'; +export { PostEngine, getPostEngine, type PostData, type PostFilter, type SearchResult, type PaginatedResult, type PaginationOptions } from './PostEngine'; export { MediaEngine, getMediaEngine, type MediaData } from './MediaEngine'; export { SyncEngine, getSyncEngine, type SyncConfig, type SyncResult, type SyncDirection, type SyncStatus } from './SyncEngine'; export { ProjectEngine, getProjectEngine, type ProjectData } from './ProjectEngine'; diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.ts index 1be3b04..602faee 100644 --- a/src/main/ipc/handlers.ts +++ b/src/main/ipc/handlers.ts @@ -1,6 +1,6 @@ import { ipcMain, dialog, shell } from 'electron'; import { eq } from 'drizzle-orm'; -import { getPostEngine, PostData, PostFilter } from '../engine/PostEngine'; +import { getPostEngine, PostData, PostFilter, PaginationOptions } from '../engine/PostEngine'; import { getMediaEngine, MediaData } from '../engine/MediaEngine'; import { getSyncEngine, SyncConfig, SyncDirection } from '../engine/SyncEngine'; import { getDropboxSyncEngine, DropboxSyncConfig, ConflictResolution } from '../engine/DropboxSyncEngine'; @@ -99,9 +99,9 @@ export function registerIpcHandlers(): void { return engine.getPost(id); }); - ipcMain.handle('posts:getAll', async () => { + ipcMain.handle('posts:getAll', async (_, options?: PaginationOptions) => { const engine = getPostEngine(); - return engine.getAllPosts(); + return engine.getAllPosts(options); }); ipcMain.handle('posts:getByStatus', async (_, status: 'draft' | 'published' | 'archived') => { @@ -231,6 +231,18 @@ export function registerIpcHandlers(): void { return engine.getMedia(id); }); + ipcMain.handle('media:getUrl', async (_, id: string) => { + // Returns the bds-media:// protocol URL for a media item + return `bds-media://${id}`; + }); + + ipcMain.handle('media:getFilePath', async (_, id: string) => { + // Returns the actual file path for a media item (for debugging/advanced use) + const db = getDatabase().getLocal(); + const mediaItem = await db.select().from(media).where(eq(media.id, id)).get(); + return mediaItem?.filePath ?? null; + }); + ipcMain.handle('media:getAll', async () => { const engine = getMediaEngine(); return engine.getAllMedia(); diff --git a/src/main/main.ts b/src/main/main.ts index e611716..70232f5 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -1,14 +1,29 @@ -import { app, BrowserWindow, Menu, MenuItemConstructorOptions, ipcMain } from 'electron'; +import { app, BrowserWindow, Menu, MenuItemConstructorOptions, ipcMain, protocol, net } from 'electron'; import * as path from 'path'; import * as fs from 'fs'; import { getDatabase } from './database'; import { registerIpcHandlers } from './ipc'; +import { media } from './database/schema'; +import { eq } from 'drizzle-orm'; let mainWindow: BrowserWindow | null = null; // Check if dev server is likely running (only in development) const isDev = process.env.NODE_ENV === 'development'; +// Register custom protocol scheme as privileged (must be done before app is ready) +protocol.registerSchemesAsPrivileged([ + { + scheme: 'bds-media', + privileges: { + standard: true, + secure: true, + supportFetchAPI: true, + corsEnabled: true, + }, + }, +]); + function createWindow(): void { mainWindow = new BrowserWindow({ width: 1400, @@ -298,6 +313,53 @@ async function initialize(): Promise { const db = getDatabase(); await db.initializeLocal(); + // Register custom protocol for serving media files + // URLs like bds-media://media-id will be resolved to the actual file + protocol.handle('bds-media', async (request) => { + try { + const url = new URL(request.url); + const mediaIdentifier = url.hostname; // bds-media://media-id or bds-media://filename.jpg + + const database = getDatabase().getLocal(); + + // First, try to find by ID (most common case) + let mediaItem = await database + .select() + .from(media) + .where(eq(media.id, mediaIdentifier)) + .get(); + + // If not found by ID, try by filename + if (!mediaItem) { + mediaItem = await database + .select() + .from(media) + .where(eq(media.filename, mediaIdentifier)) + .get(); + } + + // If still not found, try by original name + if (!mediaItem) { + mediaItem = await database + .select() + .from(media) + .where(eq(media.originalName, mediaIdentifier)) + .get(); + } + + if (mediaItem && mediaItem.filePath) { + // Use net.fetch to get the file - this handles the file protocol properly + return net.fetch(`file://${mediaItem.filePath}`); + } + + // Return a 404 response if media not found + return new Response('Media not found', { status: 404 }); + } catch (error) { + console.error('Error serving media:', error); + return new Response('Internal server error', { status: 500 }); + } + }); + // Register IPC handlers registerIpcHandlers(); } diff --git a/src/main/preload.ts b/src/main/preload.ts index a8346fc..774b213 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -20,7 +20,7 @@ contextBridge.exposeInMainWorld('electronAPI', { update: (id: string, data: unknown) => ipcRenderer.invoke('posts:update', id, data), delete: (id: string) => ipcRenderer.invoke('posts:delete', id), get: (id: string) => ipcRenderer.invoke('posts:get', id), - getAll: () => ipcRenderer.invoke('posts:getAll'), + getAll: (options?: { limit?: number; offset?: number }) => ipcRenderer.invoke('posts:getAll', options), getByStatus: (status: string) => ipcRenderer.invoke('posts:getByStatus', status), publish: (id: string) => ipcRenderer.invoke('posts:publish', id), unpublish: (id: string) => ipcRenderer.invoke('posts:unpublish', id), @@ -46,6 +46,8 @@ contextBridge.exposeInMainWorld('electronAPI', { update: (id: string, data: unknown) => ipcRenderer.invoke('media:update', id, data), delete: (id: string) => ipcRenderer.invoke('media:delete', id), get: (id: string) => ipcRenderer.invoke('media:get', id), + getUrl: (id: string) => ipcRenderer.invoke('media:getUrl', id), + getFilePath: (id: string) => ipcRenderer.invoke('media:getFilePath', id), getAll: () => ipcRenderer.invoke('media:getAll'), rebuildFromFiles: () => ipcRenderer.invoke('media:rebuildFromFiles'), getThumbnail: (id: string, size?: 'small' | 'medium' | 'large') => ipcRenderer.invoke('media:getThumbnail', id, size), diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 3a3b5a5..c8ad4b8 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -33,10 +33,11 @@ const App: React.FC = () => { // First, get active project to set the correct context in backend engines await window.electronAPI?.projects.getActive(); - // Load posts (now with correct project context) - const posts = await window.electronAPI?.posts.getAll(); - if (posts) { - setPosts(posts as PostData[]); + // Load posts (now with correct project context, limited to 500) + const postsResult = await window.electronAPI?.posts.getAll({ limit: 500, offset: 0 }); + if (postsResult) { + const { items, hasMore, total } = postsResult as { items: PostData[]; hasMore: boolean; total: number }; + setPosts(items, hasMore, total); } // Load media diff --git a/src/renderer/components/Editor/Editor.tsx b/src/renderer/components/Editor/Editor.tsx index 6c53932..6603b51 100644 --- a/src/renderer/components/Editor/Editor.tsx +++ b/src/renderer/components/Editor/Editor.tsx @@ -1,6 +1,6 @@ -import React, { useState, useEffect, useCallback, useRef } from 'react'; +import React, { useState, useEffect, useCallback, useRef, useMemo } from 'react'; import MonacoEditor from '@monaco-editor/react'; -import { useAppStore, PostData, EditorMode } from '../../store'; +import { useAppStore, PostData, EditorMode, MediaData } from '../../store'; import { showToast } from '../Toast'; import { WysiwygEditor } from '../WysiwygEditor'; import { Lightbox, useMarkdownImages } from '../Lightbox'; @@ -9,6 +9,60 @@ import { ErrorModal } from '../ErrorModal'; import { SettingsView } from '../SettingsView'; import './Editor.css'; +/** + * Resolves media references in markdown content to bds-media:// URLs + * Matches images by: + * 1. Media ID in the path (e.g., /media/2025/01/{id}.jpg) + * 2. Original filename (e.g., image.jpg) + * 3. Filename pattern (e.g., {id}.jpg) + */ +const resolveMediaUrls = (content: string, mediaList: MediaData[]): string => { + if (!content || mediaList.length === 0) return content; + + // Build lookup maps for efficient matching + const byId = new Map(); + const byOriginalName = new Map(); + const byFilename = new Map(); + + for (const m of mediaList) { + byId.set(m.id, m.id); + byOriginalName.set(m.originalName.toLowerCase(), m.id); + byFilename.set(m.filename.toLowerCase(), m.id); + } + + // Replace image URLs in markdown + return content.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, src) => { + // Skip if already using bds-media protocol or external URLs + if (src.startsWith('bds-media://') || src.startsWith('http://') || src.startsWith('https://')) { + return match; + } + + // Extract the filename from the path + const filename = src.split('/').pop() || ''; + const filenameWithoutExt = filename.replace(/\.[^.]+$/, ''); + const filenameLower = filename.toLowerCase(); + + // Try to match by: + // 1. UUID in path (the file is named by ID) + if (byId.has(filenameWithoutExt)) { + return `![${alt}](bds-media://${filenameWithoutExt})`; + } + + // 2. Filename lookup + if (byFilename.has(filenameLower)) { + return `![${alt}](bds-media://${byFilename.get(filenameLower)})`; + } + + // 3. Original name lookup + if (byOriginalName.has(filenameLower)) { + return `![${alt}](bds-media://${byOriginalName.get(filenameLower)})`; + } + + // No match found, return original + return match; + }); +}; + // Simple markdown to HTML converter for preview const markdownToHtml = (markdown: string): string => { return markdown @@ -52,6 +106,7 @@ const PostEditor: React.FC = ({ post }) => { preferredEditorMode, setPreferredEditorMode, showErrorModal, + media, } = useAppStore(); const [title, setTitle] = useState(post.title); @@ -88,8 +143,11 @@ const PostEditor: React.FC = ({ post }) => { } }, []); - // Extract images from content for lightbox - const images = useMarkdownImages(content); + // Resolve media URLs in content for display + const resolvedContent = useMemo(() => resolveMediaUrls(content, media), [content, media]); + + // Extract images from resolved content for lightbox + const images = useMarkdownImages(resolvedContent); // Track latest values for auto-save on unmount/switch const pendingChangesRef = useRef<{ @@ -505,7 +563,7 @@ const PostEditor: React.FC = ({ post }) => {
)} diff --git a/src/renderer/components/Sidebar/Sidebar.css b/src/renderer/components/Sidebar/Sidebar.css index 71b8e87..d274201 100644 --- a/src/renderer/components/Sidebar/Sidebar.css +++ b/src/renderer/components/Sidebar/Sidebar.css @@ -145,6 +145,34 @@ margin-bottom: 16px; } +/* Load More Button */ +.sidebar-load-more { + padding: 12px 16px; + display: flex; + justify-content: center; +} + +.load-more-button { + width: 100%; + padding: 8px 16px; + background-color: var(--vscode-button-secondaryBackground); + color: var(--vscode-button-secondaryForeground); + border: none; + border-radius: 4px; + font-size: 12px; + cursor: pointer; + transition: background-color 0.2s; +} + +.load-more-button:hover:not(:disabled) { + background-color: var(--vscode-button-secondaryHoverBackground); +} + +.load-more-button:disabled { + opacity: 0.6; + cursor: not-allowed; +} + /* Media Grid */ .media-grid { display: grid; diff --git a/src/renderer/components/Sidebar/Sidebar.tsx b/src/renderer/components/Sidebar/Sidebar.tsx index bb5edbe..5dd5503 100644 --- a/src/renderer/components/Sidebar/Sidebar.tsx +++ b/src/renderer/components/Sidebar/Sidebar.tsx @@ -222,7 +222,7 @@ const SearchBox: React.FC = ({ onSearch }) => { }; const PostsList: React.FC = () => { - const { posts, selectedPostId, setSelectedPost } = useAppStore(); + const { posts, selectedPostId, setSelectedPost, hasMorePosts, totalPosts, appendPosts } = useAppStore(); // Filter state const [searchQuery, setSearchQuery] = useState(''); @@ -235,6 +235,7 @@ const PostsList: React.FC = () => { const [availableCategories, setAvailableCategories] = useState([]); const [showFilters, setShowFilters] = useState(false); const [filteredPosts, setFilteredPosts] = useState(null); + const [isLoadingMore, setIsLoadingMore] = useState(false); // Load available tags and categories useEffect(() => { @@ -339,6 +340,24 @@ const PostsList: React.FC = () => { } }; + const handleLoadMore = async () => { + if (isLoadingMore || !hasMorePosts) return; + + setIsLoadingMore(true); + try { + const postsResult = await window.electronAPI?.posts.getAll({ limit: 500, offset: posts.length }); + if (postsResult) { + const { items, hasMore } = postsResult as { items: PostData[]; hasMore: boolean }; + appendPosts(items, hasMore); + } + } catch (error) { + console.error('Failed to load more posts:', error); + showToast.error('Failed to load more posts'); + } finally { + setIsLoadingMore(false); + } + }; + // Determine which posts to display const displayPosts = searchResults ?? filteredPosts ?? posts; const isFiltered = searchResults !== null || filteredPosts !== null; @@ -510,6 +529,19 @@ const PostsList: React.FC = () => {
)} + + {/* Load More button - only show when not filtering and has more posts */} + {!isFiltered && hasMorePosts && ( +
+ +
+ )} ); }; diff --git a/src/renderer/store/appStore.ts b/src/renderer/store/appStore.ts index df9154a..a942b13 100644 --- a/src/renderer/store/appStore.ts +++ b/src/renderer/store/appStore.ts @@ -82,6 +82,10 @@ interface AppState { media: MediaData[]; tasks: TaskProgress[]; + // Pagination + hasMorePosts: boolean; + totalPosts: number; + // Track which posts have unsaved changes (by post ID) dirtyPosts: Set; @@ -112,7 +116,8 @@ interface AppState { setSelectedMedia: (id: string | null) => void; setPreferredEditorMode: (mode: EditorMode) => void; - setPosts: (posts: PostData[]) => void; + setPosts: (posts: PostData[], hasMore?: boolean, total?: number) => void; + appendPosts: (posts: PostData[], hasMore: boolean) => void; addPost: (post: PostData) => void; updatePost: (id: string, post: Partial) => void; removePost: (id: string) => void; @@ -162,6 +167,10 @@ export const useAppStore = create()( media: [], tasks: [], + // Pagination + hasMorePosts: false, + totalPosts: 0, + // Dirty posts tracking dirtyPosts: new Set(), @@ -197,8 +206,12 @@ export const useAppStore = create()( setPreferredEditorMode: (mode) => set({ preferredEditorMode: mode }), // Post Actions - setPosts: (posts) => set({ posts }), - addPost: (post) => set((state) => ({ posts: [...state.posts, post] })), + setPosts: (posts, hasMore = false, total = 0) => set({ posts, hasMorePosts: hasMore, totalPosts: total }), + appendPosts: (newPosts, hasMore) => set((state) => ({ + posts: [...state.posts, ...newPosts], + hasMorePosts: hasMore, + })), + addPost: (post) => set((state) => ({ posts: [post, ...state.posts], totalPosts: state.totalPosts + 1 })), updatePost: (id, updatedPost) => set((state) => ({ posts: state.posts.map((p) => (p.id === id ? { ...p, ...updatedPost } : p)), })),