fix: better handling of many posts

src/main/engine/PostEngine.ts

@@ -60,6 +60,17 @@ export interface PostFilter {
   month?: number;
 }
 
+export interface PaginatedResult<T> {
+  items: T[];
+  hasMore: boolean;
+  total: number;
+}
+
+export interface PaginationOptions {
+  limit?: number;
+  offset?: number;
+}
+
 export class PostEngine extends EventEmitter {
   private currentProjectId: string = 'default';
 
@@ -455,7 +466,49 @@ export class PostEngine extends EventEmitter {
     return this.dbRowToPostData(dbPost, '');
   }
 
-  async getAllPosts(): Promise<PostData[]> {
+  async getAllPosts(options?: PaginationOptions): Promise<PaginatedResult<PostData>> {
+    const db = getDatabase().getLocal();
+    const limit = options?.limit ?? 500;
+    const offset = options?.offset ?? 0;
+
+    // Get total count for hasMore calculation
+    const countResult = await db
+      .select({ count: posts.id })
+      .from(posts)
+      .where(eq(posts.projectId, this.currentProjectId))
+      .all();
+    const total = countResult.length;
+
+    const dbPosts = await db
+      .select()
+      .from(posts)
+      .where(eq(posts.projectId, this.currentProjectId))
+      .orderBy(desc(posts.createdAt))
+      .limit(limit)
+      .offset(offset)
+      .all();
+    
+    const items: PostData[] = [];
+    
+    for (const dbPost of dbPosts) {
+      const postData = await this.getPost(dbPost.id);
+      if (postData) {
+        items.push(postData);
+      }
+    }
+
+    return {
+      items,
+      hasMore: offset + items.length < total,
+      total,
+    };
+  }
+
+  /**
+   * Internal method to get all posts without pagination.
+   * Used by methods that need to iterate over all posts (search, tags, categories, etc.)
+   */
+  private async getAllPostsUnpaginated(): Promise<PostData[]> {
     const db = getDatabase().getLocal();
     const dbPosts = await db
       .select()
@@ -574,7 +627,7 @@ export class PostEngine extends EventEmitter {
         args: [query],
       });
 
-      const projectPosts = await this.getAllPosts();
+      const projectPosts = await this.getAllPostsUnpaginated();
       const projectPostIds = new Set(projectPosts.map(p => p.id));
 
       return result.rows
@@ -594,7 +647,7 @@ export class PostEngine extends EventEmitter {
   }
 
   async getAvailableTags(): Promise<string[]> {
-    const allPosts = await this.getAllPosts();
+    const allPosts = await this.getAllPostsUnpaginated();
     const tags = new Set<string>();
     for (const post of allPosts) {
       for (const tag of post.tags) {
@@ -605,7 +658,7 @@ export class PostEngine extends EventEmitter {
   }
 
   async getAvailableCategories(): Promise<string[]> {
-    const allPosts = await this.getAllPosts();
+    const allPosts = await this.getAllPostsUnpaginated();
     const categories = new Set<string>();
     for (const post of allPosts) {
       for (const cat of post.categories) {
@@ -616,7 +669,7 @@ export class PostEngine extends EventEmitter {
   }
 
   async getPostsByYearMonth(): Promise<{ year: number; month: number; count: number }[]> {
-    const allPosts = await this.getAllPosts();
+    const allPosts = await this.getAllPostsUnpaginated();
     const counts = new Map<string, { year: number; month: number; count: number }>();
 
     for (const post of allPosts) {

src/main/engine/index.ts

@@ -1,5 +1,5 @@
 export { TaskManager, taskManager, type Task, type TaskProgress, type TaskStatus } from './TaskManager';
-export { PostEngine, getPostEngine, type PostData, type PostFilter, type SearchResult } from './PostEngine';
+export { PostEngine, getPostEngine, type PostData, type PostFilter, type SearchResult, type PaginatedResult, type PaginationOptions } from './PostEngine';
 export { MediaEngine, getMediaEngine, type MediaData } from './MediaEngine';
 export { SyncEngine, getSyncEngine, type SyncConfig, type SyncResult, type SyncDirection, type SyncStatus } from './SyncEngine';
 export { ProjectEngine, getProjectEngine, type ProjectData } from './ProjectEngine';

src/main/ipc/handlers.ts

@@ -1,6 +1,6 @@
 import { ipcMain, dialog, shell } from 'electron';
 import { eq } from 'drizzle-orm';
-import { getPostEngine, PostData, PostFilter } from '../engine/PostEngine';
+import { getPostEngine, PostData, PostFilter, PaginationOptions } from '../engine/PostEngine';
 import { getMediaEngine, MediaData } from '../engine/MediaEngine';
 import { getSyncEngine, SyncConfig, SyncDirection } from '../engine/SyncEngine';
 import { getDropboxSyncEngine, DropboxSyncConfig, ConflictResolution } from '../engine/DropboxSyncEngine';
@@ -99,9 +99,9 @@ export function registerIpcHandlers(): void {
     return engine.getPost(id);
   });
 
-  ipcMain.handle('posts:getAll', async () => {
+  ipcMain.handle('posts:getAll', async (_, options?: PaginationOptions) => {
     const engine = getPostEngine();
-    return engine.getAllPosts();
+    return engine.getAllPosts(options);
   });
 
   ipcMain.handle('posts:getByStatus', async (_, status: 'draft' | 'published' | 'archived') => {
@@ -231,6 +231,18 @@ export function registerIpcHandlers(): void {
     return engine.getMedia(id);
   });
 
+  ipcMain.handle('media:getUrl', async (_, id: string) => {
+    // Returns the bds-media:// protocol URL for a media item
+    return `bds-media://${id}`;
+  });
+
+  ipcMain.handle('media:getFilePath', async (_, id: string) => {
+    // Returns the actual file path for a media item (for debugging/advanced use)
+    const db = getDatabase().getLocal();
+    const mediaItem = await db.select().from(media).where(eq(media.id, id)).get();
+    return mediaItem?.filePath ?? null;
+  });
+
   ipcMain.handle('media:getAll', async () => {
     const engine = getMediaEngine();
     return engine.getAllMedia();

src/main/main.ts

@@ -1,14 +1,29 @@
-import { app, BrowserWindow, Menu, MenuItemConstructorOptions, ipcMain } from 'electron';
+import { app, BrowserWindow, Menu, MenuItemConstructorOptions, ipcMain, protocol, net } from 'electron';
 import * as path from 'path';
 import * as fs from 'fs';
 import { getDatabase } from './database';
 import { registerIpcHandlers } from './ipc';
+import { media } from './database/schema';
+import { eq } from 'drizzle-orm';
 
 let mainWindow: BrowserWindow | null = null;
 
 // Check if dev server is likely running (only in development)
 const isDev = process.env.NODE_ENV === 'development';
 
+// Register custom protocol scheme as privileged (must be done before app is ready)
+protocol.registerSchemesAsPrivileged([
+  {
+    scheme: 'bds-media',
+    privileges: {
+      standard: true,
+      secure: true,
+      supportFetchAPI: true,
+      corsEnabled: true,
+    },
+  },
+]);
+
 function createWindow(): void {
   mainWindow = new BrowserWindow({
     width: 1400,
@@ -298,6 +313,53 @@ async function initialize(): Promise<void> {
   const db = getDatabase();
   await db.initializeLocal();
 
+  // Register custom protocol for serving media files
+  // URLs like bds-media://media-id will be resolved to the actual file
+  protocol.handle('bds-media', async (request) => {
+    try {
+      const url = new URL(request.url);
+      const mediaIdentifier = url.hostname; // bds-media://media-id or bds-media://filename.jpg
+      
+      const database = getDatabase().getLocal();
+      
+      // First, try to find by ID (most common case)
+      let mediaItem = await database
+        .select()
+        .from(media)
+        .where(eq(media.id, mediaIdentifier))
+        .get();
+      
+      // If not found by ID, try by filename
+      if (!mediaItem) {
+        mediaItem = await database
+          .select()
+          .from(media)
+          .where(eq(media.filename, mediaIdentifier))
+          .get();
+      }
+      
+      // If still not found, try by original name
+      if (!mediaItem) {
+        mediaItem = await database
+          .select()
+          .from(media)
+          .where(eq(media.originalName, mediaIdentifier))
+          .get();
+      }
+      
+      if (mediaItem && mediaItem.filePath) {
+        // Use net.fetch to get the file - this handles the file protocol properly
+        return net.fetch(`file://${mediaItem.filePath}`);
+      }
+      
+      // Return a 404 response if media not found
+      return new Response('Media not found', { status: 404 });
+    } catch (error) {
+      console.error('Error serving media:', error);
+      return new Response('Internal server error', { status: 500 });
+    }
+  });
+
   // Register IPC handlers
   registerIpcHandlers();
 }

src/main/preload.ts

@@ -20,7 +20,7 @@ contextBridge.exposeInMainWorld('electronAPI', {
     update: (id: string, data: unknown) => ipcRenderer.invoke('posts:update', id, data),
     delete: (id: string) => ipcRenderer.invoke('posts:delete', id),
     get: (id: string) => ipcRenderer.invoke('posts:get', id),
-    getAll: () => ipcRenderer.invoke('posts:getAll'),
+    getAll: (options?: { limit?: number; offset?: number }) => ipcRenderer.invoke('posts:getAll', options),
     getByStatus: (status: string) => ipcRenderer.invoke('posts:getByStatus', status),
     publish: (id: string) => ipcRenderer.invoke('posts:publish', id),
     unpublish: (id: string) => ipcRenderer.invoke('posts:unpublish', id),
@@ -46,6 +46,8 @@ contextBridge.exposeInMainWorld('electronAPI', {
     update: (id: string, data: unknown) => ipcRenderer.invoke('media:update', id, data),
     delete: (id: string) => ipcRenderer.invoke('media:delete', id),
     get: (id: string) => ipcRenderer.invoke('media:get', id),
+    getUrl: (id: string) => ipcRenderer.invoke('media:getUrl', id),
+    getFilePath: (id: string) => ipcRenderer.invoke('media:getFilePath', id),
     getAll: () => ipcRenderer.invoke('media:getAll'),
     rebuildFromFiles: () => ipcRenderer.invoke('media:rebuildFromFiles'),
     getThumbnail: (id: string, size?: 'small' | 'medium' | 'large') => ipcRenderer.invoke('media:getThumbnail', id, size),

src/renderer/App.tsx

@@ -33,10 +33,11 @@ const App: React.FC = () => {
         // First, get active project to set the correct context in backend engines
         await window.electronAPI?.projects.getActive();
         
-        // Load posts (now with correct project context)
-        const posts = await window.electronAPI?.posts.getAll();
-        if (posts) {
-          setPosts(posts as PostData[]);
+        // Load posts (now with correct project context, limited to 500)
+        const postsResult = await window.electronAPI?.posts.getAll({ limit: 500, offset: 0 });
+        if (postsResult) {
+          const { items, hasMore, total } = postsResult as { items: PostData[]; hasMore: boolean; total: number };
+          setPosts(items, hasMore, total);
         }
 
         // Load media

src/renderer/components/Editor/Editor.tsx

@@ -1,6 +1,6 @@
-import React, { useState, useEffect, useCallback, useRef } from 'react';
+import React, { useState, useEffect, useCallback, useRef, useMemo } from 'react';
 import MonacoEditor from '@monaco-editor/react';
-import { useAppStore, PostData, EditorMode } from '../../store';
+import { useAppStore, PostData, EditorMode, MediaData } from '../../store';
 import { showToast } from '../Toast';
 import { WysiwygEditor } from '../WysiwygEditor';
 import { Lightbox, useMarkdownImages } from '../Lightbox';
@@ -9,6 +9,60 @@ import { ErrorModal } from '../ErrorModal';
 import { SettingsView } from '../SettingsView';
 import './Editor.css';
 
+/**
+ * Resolves media references in markdown content to bds-media:// URLs
+ * Matches images by:
+ * 1. Media ID in the path (e.g., /media/2025/01/{id}.jpg)
+ * 2. Original filename (e.g., image.jpg)
+ * 3. Filename pattern (e.g., {id}.jpg)
+ */
+const resolveMediaUrls = (content: string, mediaList: MediaData[]): string => {
+  if (!content || mediaList.length === 0) return content;
+  
+  // Build lookup maps for efficient matching
+  const byId = new Map<string, string>();
+  const byOriginalName = new Map<string, string>();
+  const byFilename = new Map<string, string>();
+  
+  for (const m of mediaList) {
+    byId.set(m.id, m.id);
+    byOriginalName.set(m.originalName.toLowerCase(), m.id);
+    byFilename.set(m.filename.toLowerCase(), m.id);
+  }
+  
+  // Replace image URLs in markdown
+  return content.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, src) => {
+    // Skip if already using bds-media protocol or external URLs
+    if (src.startsWith('bds-media://') || src.startsWith('http://') || src.startsWith('https://')) {
+      return match;
+    }
+    
+    // Extract the filename from the path
+    const filename = src.split('/').pop() || '';
+    const filenameWithoutExt = filename.replace(/\.[^.]+$/, '');
+    const filenameLower = filename.toLowerCase();
+    
+    // Try to match by:
+    // 1. UUID in path (the file is named by ID)
+    if (byId.has(filenameWithoutExt)) {
+      return `![${alt}](bds-media://${filenameWithoutExt})`;
+    }
+    
+    // 2. Filename lookup
+    if (byFilename.has(filenameLower)) {
+      return `![${alt}](bds-media://${byFilename.get(filenameLower)})`;
+    }
+    
+    // 3. Original name lookup
+    if (byOriginalName.has(filenameLower)) {
+      return `![${alt}](bds-media://${byOriginalName.get(filenameLower)})`;
+    }
+    
+    // No match found, return original
+    return match;
+  });
+};
+
 // Simple markdown to HTML converter for preview
 const markdownToHtml = (markdown: string): string => {
   return markdown
@@ -52,6 +106,7 @@ const PostEditor: React.FC<PostEditorProps> = ({ post }) => {
     preferredEditorMode,
     setPreferredEditorMode,
     showErrorModal,
+    media,
   } = useAppStore();
   
   const [title, setTitle] = useState(post.title);
@@ -88,8 +143,11 @@ const PostEditor: React.FC<PostEditorProps> = ({ post }) => {
     }
   }, []);
 
-  // Extract images from content for lightbox
-  const images = useMarkdownImages(content);
+  // Resolve media URLs in content for display
+  const resolvedContent = useMemo(() => resolveMediaUrls(content, media), [content, media]);
+
+  // Extract images from resolved content for lightbox
+  const images = useMarkdownImages(resolvedContent);
 
   // Track latest values for auto-save on unmount/switch
   const pendingChangesRef = useRef<{
@@ -505,7 +563,7 @@ const PostEditor: React.FC<PostEditorProps> = ({ post }) => {
             <div className="editor-preview markdown-body">
               <div
                 className="preview-content"
-                dangerouslySetInnerHTML={{ __html: markdownToHtml(content) }}
+                dangerouslySetInnerHTML={{ __html: markdownToHtml(resolvedContent) }}
               />
             </div>
           )}

src/renderer/components/Sidebar/Sidebar.css

@@ -145,6 +145,34 @@
   margin-bottom: 16px;
 }
 
+/* Load More Button */
+.sidebar-load-more {
+  padding: 12px 16px;
+  display: flex;
+  justify-content: center;
+}
+
+.load-more-button {
+  width: 100%;
+  padding: 8px 16px;
+  background-color: var(--vscode-button-secondaryBackground);
+  color: var(--vscode-button-secondaryForeground);
+  border: none;
+  border-radius: 4px;
+  font-size: 12px;
+  cursor: pointer;
+  transition: background-color 0.2s;
+}
+
+.load-more-button:hover:not(:disabled) {
+  background-color: var(--vscode-button-secondaryHoverBackground);
+}
+
+.load-more-button:disabled {
+  opacity: 0.6;
+  cursor: not-allowed;
+}
+
 /* Media Grid */
 .media-grid {
   display: grid;

src/renderer/components/Sidebar/Sidebar.tsx

@@ -222,7 +222,7 @@ const SearchBox: React.FC<SearchBoxProps> = ({ onSearch }) => {
 };
 
 const PostsList: React.FC = () => {
-  const { posts, selectedPostId, setSelectedPost } = useAppStore();
+  const { posts, selectedPostId, setSelectedPost, hasMorePosts, totalPosts, appendPosts } = useAppStore();
   
   // Filter state
   const [searchQuery, setSearchQuery] = useState('');
@@ -235,6 +235,7 @@ const PostsList: React.FC = () => {
   const [availableCategories, setAvailableCategories] = useState<string[]>([]);
   const [showFilters, setShowFilters] = useState(false);
   const [filteredPosts, setFilteredPosts] = useState<PostData[] | null>(null);
+  const [isLoadingMore, setIsLoadingMore] = useState(false);
 
   // Load available tags and categories
   useEffect(() => {
@@ -339,6 +340,24 @@ const PostsList: React.FC = () => {
     }
   };
 
+  const handleLoadMore = async () => {
+    if (isLoadingMore || !hasMorePosts) return;
+    
+    setIsLoadingMore(true);
+    try {
+      const postsResult = await window.electronAPI?.posts.getAll({ limit: 500, offset: posts.length });
+      if (postsResult) {
+        const { items, hasMore } = postsResult as { items: PostData[]; hasMore: boolean };
+        appendPosts(items, hasMore);
+      }
+    } catch (error) {
+      console.error('Failed to load more posts:', error);
+      showToast.error('Failed to load more posts');
+    } finally {
+      setIsLoadingMore(false);
+    }
+  };
+
   // Determine which posts to display
   const displayPosts = searchResults ?? filteredPosts ?? posts;
   const isFiltered = searchResults !== null || filteredPosts !== null;
@@ -510,6 +529,19 @@ const PostsList: React.FC = () => {
           <button onClick={clearAllFilters}>Clear filters</button>
         </div>
       )}
+
+      {/* Load More button - only show when not filtering and has more posts */}
+      {!isFiltered && hasMorePosts && (
+        <div className="sidebar-load-more">
+          <button 
+            onClick={handleLoadMore} 
+            disabled={isLoadingMore}
+            className="load-more-button"
+          >
+            {isLoadingMore ? 'Loading...' : `Load more (${posts.length} of ${totalPosts})`}
+          </button>
+        </div>
+      )}
     </div>
   );
 };

src/renderer/store/appStore.ts

@@ -82,6 +82,10 @@ interface AppState {
   media: MediaData[];
   tasks: TaskProgress[];
   
+  // Pagination
+  hasMorePosts: boolean;
+  totalPosts: number;
+  
   // Track which posts have unsaved changes (by post ID)
   dirtyPosts: Set<string>;
   
@@ -112,7 +116,8 @@ interface AppState {
   setSelectedMedia: (id: string | null) => void;
   setPreferredEditorMode: (mode: EditorMode) => void;
   
-  setPosts: (posts: PostData[]) => void;
+  setPosts: (posts: PostData[], hasMore?: boolean, total?: number) => void;
+  appendPosts: (posts: PostData[], hasMore: boolean) => void;
   addPost: (post: PostData) => void;
   updatePost: (id: string, post: Partial<PostData>) => void;
   removePost: (id: string) => void;
@@ -162,6 +167,10 @@ export const useAppStore = create<AppState>()(
       media: [],
       tasks: [],
       
+      // Pagination
+      hasMorePosts: false,
+      totalPosts: 0,
+      
       // Dirty posts tracking
       dirtyPosts: new Set<string>(),
       
@@ -197,8 +206,12 @@ export const useAppStore = create<AppState>()(
       setPreferredEditorMode: (mode) => set({ preferredEditorMode: mode }),
       
       // Post Actions
-      setPosts: (posts) => set({ posts }),
-      addPost: (post) => set((state) => ({ posts: [...state.posts, post] })),
+      setPosts: (posts, hasMore = false, total = 0) => set({ posts, hasMorePosts: hasMore, totalPosts: total }),
+      appendPosts: (newPosts, hasMore) => set((state) => ({
+        posts: [...state.posts, ...newPosts],
+        hasMorePosts: hasMore,
+      })),
+      addPost: (post) => set((state) => ({ posts: [post, ...state.posts], totalPosts: state.totalPosts + 1 })),
       updatePost: (id, updatedPost) => set((state) => ({
         posts: state.posts.map((p) => (p.id === id ? { ...p, ...updatedPost } : p)),
       })),