From 642c6f52947975d4d94454ef568a2e90e9882385 Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 13 Feb 2026 22:48:15 +0100 Subject: [PATCH] feat: AI Quickaction to generate caption and alt text for images --- src/main/engine/MetaEngine.ts | 1 + src/main/engine/OpenCodeManager.ts | 129 ++++++++++++++++++ src/main/engine/PostEngine.ts | 68 ++++----- src/main/ipc/chatHandlers.ts | 13 ++ src/main/ipc/handlers.ts | 2 +- src/main/preload.ts | 5 +- src/renderer/components/Editor/Editor.css | 77 +++++++++++ src/renderer/components/Editor/Editor.tsx | 87 ++++++++++++ .../components/SettingsView/SettingsView.tsx | 48 ++++++- src/renderer/types/electron.d.ts | 6 +- 10 files changed, 388 insertions(+), 48 deletions(-) diff --git a/src/main/engine/MetaEngine.ts b/src/main/engine/MetaEngine.ts index 6a11850..2a37d63 100644 --- a/src/main/engine/MetaEngine.ts +++ b/src/main/engine/MetaEngine.ts @@ -13,6 +13,7 @@ export interface ProjectMetadata { name: string; description?: string; dataPath?: string; // Custom path for project data + mainLanguage?: string; // Main language for AI-generated content (ISO code, e.g., 'en', 'de', 'es') } /** diff --git a/src/main/engine/OpenCodeManager.ts b/src/main/engine/OpenCodeManager.ts index fe5680b..df0e91e 100644 --- a/src/main/engine/OpenCodeManager.ts +++ b/src/main/engine/OpenCodeManager.ts @@ -1330,6 +1330,135 @@ Remember: Only suggest mappings from NEW items to EXISTING items. Consider langu } } + /** + * Analyze a media image and generate alt text and caption using AI + * This is a one-shot request that looks at the image and suggests metadata + */ + async analyzeMediaImage(mediaId: string, language: string = 'en'): Promise<{ + success: boolean; + alt?: string; + caption?: string; + error?: string; + }> { + if (!this.apiKey) { + return { success: false, error: 'API key not configured. Please set your OpenCode API key in Settings.' }; + } + + // Get media metadata + const mediaItem = await this.mediaEngine.getMedia(mediaId); + if (!mediaItem) { + return { success: false, error: 'Media item not found' }; + } + + // Verify it's an image + if (!mediaItem.mimeType.startsWith('image/')) { + return { success: false, error: `Cannot analyze this file type: ${mediaItem.mimeType}. Only images are supported.` }; + } + + // Get the large thumbnail for better quality analysis (or medium as fallback) + let dataUrl = await this.mediaEngine.getThumbnailDataUrl(mediaId, 'large'); + if (!dataUrl) { + dataUrl = await this.mediaEngine.getThumbnailDataUrl(mediaId, 'medium'); + } + if (!dataUrl) { + return { success: false, error: 'Image thumbnail not available. Try regenerating thumbnails from Settings.' }; + } + + // Extract base64 data (remove data:image/webp;base64, prefix) + const base64Data = dataUrl.replace(/^data:image\/\w+;base64,/, ''); + + // Map language code to full name for clearer instructions + const languageNames: Record = { + en: 'English', de: 'German', es: 'Spanish', fr: 'French', it: 'Italian', + pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ru: 'Russian', ja: 'Japanese', + zh: 'Chinese', ko: 'Korean', ar: 'Arabic', hi: 'Hindi', tr: 'Turkish', + sv: 'Swedish', da: 'Danish', no: 'Norwegian', fi: 'Finnish', cs: 'Czech', + }; + const languageName = languageNames[language] || language; + + const systemPrompt = `Generate concise alt text and caption for this image in ${languageName}. + +ALT: Brief, objective description for screen readers (5-15 words). No "Image of" prefix. +CAPTION: Short, engaging blog caption (5-20 words). + +Respond with JSON only: {"alt": "...", "caption": "..."}`; + + try { + // Using Claude Sonnet 4.5 for best image analysis + const modelId = 'claude-sonnet-4-5'; + + const body = { + model: modelId, + max_tokens: 200, + system: systemPrompt, + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/webp', + data: base64Data, + }, + }, + { + type: 'text', + text: 'Analyze and respond with JSON.', + }, + ], + }, + ], + }; + + const response = await this.httpRequest(ZEN_ANTHROPIC_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': this.apiKey, + 'Authorization': `Bearer ${this.apiKey}`, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify(body), + }); + + if (response.statusCode !== 200) { + console.error('[OpenCodeManager] Image analysis failed:', response.body); + const errorMsg = this.parseErrorResponse(response); + return { success: false, error: errorMsg }; + } + + const data = JSON.parse(response.body); + + // Extract text from Anthropic response + let responseText = ''; + for (const block of data.content || []) { + if (block.type === 'text') { + responseText += block.text; + } + } + + // Parse the JSON response + const jsonMatch = responseText.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + console.error('[OpenCodeManager] No JSON found in image analysis response:', responseText); + return { success: false, error: 'Invalid response format from AI' }; + } + + const result = JSON.parse(jsonMatch[0]); + + return { + success: true, + alt: result.alt || undefined, + caption: result.caption || undefined, + }; + } catch (error) { + console.error('[OpenCodeManager] Error analyzing media image:', error); + return { success: false, error: (error as Error).message }; + } + } + private httpRequest( urlStr: string, options: { diff --git a/src/main/engine/PostEngine.ts b/src/main/engine/PostEngine.ts index d4a5529..f66e62c 100644 --- a/src/main/engine/PostEngine.ts +++ b/src/main/engine/PostEngine.ts @@ -542,14 +542,12 @@ export class PostEngine extends EventEmitter { .offset(offset) .all(); - const items: PostData[] = []; - - for (const dbPost of dbPosts) { - const postData = await this.getPost(dbPost.id); - if (postData) { - items.push(postData); - } - } + // For listing, we don't need to load content from filesystem. + // Use DB content for drafts, empty string for published posts. + // This avoids expensive filesystem reads for each post. + const items: PostData[] = dbPosts.map(dbPost => + this.dbRowToPostData(dbPost, dbPost.content || '') + ); return { items, @@ -571,16 +569,9 @@ export class PostEngine extends EventEmitter { .orderBy(desc(posts.createdAt)) .all(); - const result: PostData[] = []; - - for (const dbPost of dbPosts) { - const postData = await this.getPost(dbPost.id); - if (postData) { - result.push(postData); - } - } - - return result; + // Use DB content for drafts, empty string for published posts. + // This avoids expensive filesystem reads. + return dbPosts.map(dbPost => this.dbRowToPostData(dbPost, dbPost.content || '')); } async getPostsByStatus(status: 'draft' | 'published' | 'archived'): Promise { @@ -595,16 +586,9 @@ export class PostEngine extends EventEmitter { .orderBy(desc(posts.createdAt)) .all(); - const result: PostData[] = []; - - for (const dbPost of dbPosts) { - const postData = await this.getPost(dbPost.id); - if (postData) { - result.push(postData); - } - } - - return result; + // Use DB content for drafts, empty string for published posts. + // This avoids expensive filesystem reads. + return dbPosts.map(dbPost => this.dbRowToPostData(dbPost, dbPost.content || '')); } async getPostsFiltered(filter: PostFilter): Promise { @@ -647,21 +631,21 @@ export class PostEngine extends EventEmitter { let result: PostData[] = []; for (const dbPost of dbPosts) { - const postData = await this.getPost(dbPost.id); - if (postData) { - // Client-side filtering for tags/categories (JSON array) - if (filter.tags && filter.tags.length > 0) { - const hasAllTags = filter.tags.every(tag => postData.tags.includes(tag)); - if (!hasAllTags) continue; - } - - if (filter.categories && filter.categories.length > 0) { - const hasAnyCategory = filter.categories.some(cat => postData.categories.includes(cat)); - if (!hasAnyCategory) continue; - } - - result.push(postData); + // Use DB data directly instead of reading from filesystem + const postData = this.dbRowToPostData(dbPost, dbPost.content || ''); + + // Client-side filtering for tags/categories (JSON array) + if (filter.tags && filter.tags.length > 0) { + const hasAllTags = filter.tags.every(tag => postData.tags.includes(tag)); + if (!hasAllTags) continue; } + + if (filter.categories && filter.categories.length > 0) { + const hasAnyCategory = filter.categories.some(cat => postData.categories.includes(cat)); + if (!hasAnyCategory) continue; + } + + result.push(postData); } return result; diff --git a/src/main/ipc/chatHandlers.ts b/src/main/ipc/chatHandlers.ts index 5c8573d..aa965ed 100644 --- a/src/main/ipc/chatHandlers.ts +++ b/src/main/ipc/chatHandlers.ts @@ -330,6 +330,19 @@ export function registerChatHandlers(): void { return { success: false, error: (error as Error).message }; } }); + + // ============ Media Analysis ============ + + // Analyze a media image and generate alt text and caption + ipcMain.handle('chat:analyzeMediaImage', async (_, mediaId: string, language?: string) => { + try { + const manager = getOpenCodeManager(); + return await manager.analyzeMediaImage(mediaId, language || 'en'); + } catch (error) { + console.error('[Chat IPC] Error analyzing media image:', error); + return { success: false, error: (error as Error).message }; + } + }); } /** diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.ts index 63fe56c..92c5f78 100644 --- a/src/main/ipc/handlers.ts +++ b/src/main/ipc/handlers.ts @@ -635,7 +635,7 @@ export function registerIpcHandlers(): void { return engine.getProjectMetadata(); }); - safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string }) => { + safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string; mainLanguage?: string }) => { const engine = getMetaEngine(); await engine.updateProjectMetadata(updates); return engine.getProjectMetadata(); diff --git a/src/main/preload.ts b/src/main/preload.ts index bd71f75..0b59593 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -132,7 +132,7 @@ contextBridge.exposeInMainWorld('electronAPI', { syncOnStartup: () => ipcRenderer.invoke('meta:syncOnStartup'), getProjectMetadata: () => ipcRenderer.invoke('meta:getProjectMetadata'), setProjectMetadata: (metadata: { name: string; description?: string }) => ipcRenderer.invoke('meta:setProjectMetadata', metadata), - updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates), + updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; mainLanguage?: string }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates), }, // Tag Management (advanced tag operations) @@ -207,6 +207,9 @@ contextBridge.exposeInMainWorld('electronAPI', { // Taxonomy Analysis analyzeTaxonomy: (categories: Array<{ name: string; slug: string; existsInProject: boolean }>, tags: Array<{ name: string; slug: string; existsInProject: boolean }>, modelId: string) => ipcRenderer.invoke('chat:analyzeTaxonomy', categories, tags, modelId), + // Media Analysis + analyzeMediaImage: (mediaId: string, language?: string) => ipcRenderer.invoke('chat:analyzeMediaImage', mediaId, language), + // Event listeners for streaming/progress onStreamDelta: (callback: (data: { conversationId: string; delta: string }) => void) => { const subscription = (_event: Electron.IpcRendererEvent, data: { conversationId: string; delta: string }) => callback(data); diff --git a/src/renderer/components/Editor/Editor.css b/src/renderer/components/Editor/Editor.css index 9633f6b..ee56259 100644 --- a/src/renderer/components/Editor/Editor.css +++ b/src/renderer/components/Editor/Editor.css @@ -950,3 +950,80 @@ grid-template-columns: repeat(3, 1fr); } } + +/* Quick Actions Dropdown */ +.quick-actions-wrapper { + position: relative; + display: inline-block; +} + +.quick-actions-btn { + display: flex; + align-items: center; + gap: 4px; + white-space: nowrap; +} + +.quick-actions-btn:disabled { + opacity: 0.7; + cursor: wait; +} + +.quick-actions-menu { + position: absolute; + top: 100%; + right: 0; + margin-top: 4px; + min-width: 280px; + background: var(--vscode-dropdown-background, #3c3c3c); + border: 1px solid var(--vscode-dropdown-border, #454545); + border-radius: 6px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); + z-index: 1000; + overflow: hidden; +} + +.quick-action-item { + display: flex; + align-items: flex-start; + gap: 10px; + width: 100%; + padding: 10px 12px; + background: none; + border: none; + color: var(--vscode-dropdown-foreground, #ccc); + cursor: pointer; + text-align: left; + transition: background 0.1s; +} + +.quick-action-item:hover:not(:disabled) { + background: var(--vscode-list-hoverBackground, #2a2d2e); +} + +.quick-action-item:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.quick-action-icon { + font-size: 16px; + flex-shrink: 0; + margin-top: 2px; +} + +.quick-action-text { + display: flex; + flex-direction: column; + gap: 2px; +} + +.quick-action-text strong { + font-size: 13px; + font-weight: 500; +} + +.quick-action-text small { + font-size: 11px; + opacity: 0.7; +} diff --git a/src/renderer/components/Editor/Editor.tsx b/src/renderer/components/Editor/Editor.tsx index 7128230..434949e 100644 --- a/src/renderer/components/Editor/Editor.tsx +++ b/src/renderer/components/Editor/Editor.tsx @@ -1223,6 +1223,65 @@ const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => { const [linkedPosts, setLinkedPosts] = useState<{ postId: string; sortOrder: number }[]>([]); const [showPostPicker, setShowPostPicker] = useState(false); const [postSearchQuery, setPostSearchQuery] = useState(''); + + // Quick action menu state + const [showQuickActions, setShowQuickActions] = useState(false); + const [isAnalyzing, setIsAnalyzing] = useState(false); + const [projectLanguage, setProjectLanguage] = useState('en'); + const quickActionsRef = useRef(null); + + // Load project language setting + useEffect(() => { + window.electronAPI?.meta.getProjectMetadata().then(metadata => { + if (metadata?.mainLanguage) { + setProjectLanguage(metadata.mainLanguage); + } + }); + }, []); + + // Close quick actions menu when clicking outside + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (quickActionsRef.current && !quickActionsRef.current.contains(event.target as Node)) { + setShowQuickActions(false); + } + }; + if (showQuickActions) { + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + } + }, [showQuickActions]); + + // Handle AI image analysis for alt text and caption + const handleAIAnalysis = async () => { + if (!item || isAnalyzing) return; + + setShowQuickActions(false); + setIsAnalyzing(true); + + try { + const result = await window.electronAPI?.chat.analyzeMediaImage(item.id, projectLanguage); + + if (result?.success) { + if (result.alt) setAlt(result.alt); + if (result.caption) setCaption(result.caption); + showToast.success('AI analysis complete'); + } else { + showErrorModal({ + title: 'AI Analysis Failed', + message: result?.error || 'Failed to analyze image', + }); + } + } catch (error) { + console.error('Failed to analyze image:', error); + showErrorModal({ + title: 'AI Analysis Error', + message: (error as Error).message || 'Failed to analyze image', + }); + } finally { + setIsAnalyzing(false); + } + }; // Load linked posts for this media useEffect(() => { @@ -1381,6 +1440,34 @@ const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
+ {/* Quick Actions Dropdown */} + {item.mimeType.startsWith('image/') && ( +
+ + {showQuickActions && ( +
+ +
+ )} +
+ )}
diff --git a/src/renderer/components/SettingsView/SettingsView.tsx b/src/renderer/components/SettingsView/SettingsView.tsx index b00ffe4..6af2137 100644 --- a/src/renderer/components/SettingsView/SettingsView.tsx +++ b/src/renderer/components/SettingsView/SettingsView.tsx @@ -110,6 +110,7 @@ export const SettingsView: React.FC = () => { const [projectDescription, setProjectDescription] = useState(''); const [projectDataPath, setProjectDataPath] = useState(''); const [defaultProjectPath, setDefaultProjectPath] = useState(''); + const [projectMainLanguage, setProjectMainLanguage] = useState('en'); // Post categories management const [postCategories, setPostCategories] = useState(DEFAULT_POST_CATEGORIES); @@ -143,6 +144,13 @@ export const SettingsView: React.FC = () => { window.electronAPI?.app.getDefaultProjectPath(activeProject.id).then(path => { setDefaultProjectPath(path); }); + + // Load project metadata (includes mainLanguage) + window.electronAPI?.meta.getProjectMetadata().then(metadata => { + if (metadata?.mainLanguage) { + setProjectMainLanguage(metadata.mainLanguage); + } + }); } }, [activeProject]); @@ -299,12 +307,13 @@ export const SettingsView: React.FC = () => { setActiveProject(updated as any); useAppStore.getState().updateProject(activeProject.id, updated as any); - // Also update project.json to keep dataPath in sync + // Also update project.json to keep dataPath and mainLanguage in sync await window.electronAPI?.meta.updateProjectMetadata({ name: projectName.trim() || activeProject.name, description: projectDescription.trim(), dataPath: projectDataPath.trim() || undefined, - } as any); + mainLanguage: projectMainLanguage, + }); } showToast.success('Project settings saved'); } catch (error) { @@ -325,7 +334,7 @@ export const SettingsView: React.FC = () => { }; // Keywords for each section for search filtering - const projectKeywords = ['project', 'name', 'description', 'blog', 'site', 'path', 'folder', 'location', 'data']; + const projectKeywords = ['project', 'name', 'description', 'blog', 'site', 'path', 'folder', 'location', 'data', 'language']; const editorKeywords = ['editor', 'mode', 'wysiwyg', 'markdown', 'preview', 'visual']; const contentKeywords = ['content', 'categories', 'post', 'article', 'picture', 'aside', 'page']; const aiKeywords = ['ai', 'assistant', 'chat', 'model', 'prompt', 'system', 'api', 'key', 'claude', 'gpt', 'opencode']; @@ -392,6 +401,39 @@ export const SettingsView: React.FC = () => { + + + +