hugo/bDS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: AI Quickaction to generate caption and alt text for images

Browse Source
This commit is contained in:
hugo
2026-02-13 22:48:15 +01:00
parent 7240bae3ec
commit 642c6f5294
10 changed files with 388 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/engine/MetaEngine.ts
View File

@@ -13,6 +13,7 @@ export interface ProjectMetadata {
name: string;
description?: string;
dataPath?: string; // Custom path for project data
mainLanguage?: string; // Main language for AI-generated content (ISO code, e.g., 'en', 'de', 'es')
}
/**

129
src/main/engine/OpenCodeManager.ts
View File

@@ -1330,6 +1330,135 @@ Remember: Only suggest mappings from NEW items to EXISTING items. Consider langu
}
}
/**
* Analyze a media image and generate alt text and caption using AI
* This is a one-shot request that looks at the image and suggests metadata
*/
async analyzeMediaImage(mediaId: string, language: string = 'en'): Promise<{
success: boolean;
alt?: string;
caption?: string;
error?: string;
}> {
if (!this.apiKey) {
return { success: false, error: 'API key not configured. Please set your OpenCode API key in Settings.' };
}
// Get media metadata
const mediaItem = await this.mediaEngine.getMedia(mediaId);
if (!mediaItem) {
return { success: false, error: 'Media item not found' };
}
// Verify it's an image
if (!mediaItem.mimeType.startsWith('image/')) {
return { success: false, error: `Cannot analyze this file type: ${mediaItem.mimeType}. Only images are supported.` };
}
// Get the large thumbnail for better quality analysis (or medium as fallback)
let dataUrl = await this.mediaEngine.getThumbnailDataUrl(mediaId, 'large');
if (!dataUrl) {
dataUrl = await this.mediaEngine.getThumbnailDataUrl(mediaId, 'medium');
}
if (!dataUrl) {
return { success: false, error: 'Image thumbnail not available. Try regenerating thumbnails from Settings.' };
}
// Extract base64 data (remove data:image/webp;base64, prefix)
const base64Data = dataUrl.replace(/^data:image\/\w+;base64,/, '');
// Map language code to full name for clearer instructions
const languageNames: Record<string, string> = {
en: 'English', de: 'German', es: 'Spanish', fr: 'French', it: 'Italian',
pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ru: 'Russian', ja: 'Japanese',
zh: 'Chinese', ko: 'Korean', ar: 'Arabic', hi: 'Hindi', tr: 'Turkish',
sv: 'Swedish', da: 'Danish', no: 'Norwegian', fi: 'Finnish', cs: 'Czech',
};
const languageName = languageNames[language] || language;
const systemPrompt = `Generate concise alt text and caption for this image in ${languageName}.
ALT: Brief, objective description for screen readers (5-15 words). No "Image of" prefix.
CAPTION: Short, engaging blog caption (5-20 words).
Respond with JSON only: {"alt": "...", "caption": "..."}`;
try {
// Using Claude Sonnet 4.5 for best image analysis
const modelId = 'claude-sonnet-4-5';
const body = {
model: modelId,
max_tokens: 200,
system: systemPrompt,
messages: [
{
role: 'user',
content: [
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/webp',
data: base64Data,
},
},
{
type: 'text',
text: 'Analyze and respond with JSON.',
},
],
},
],
};
const response = await this.httpRequest(ZEN_ANTHROPIC_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': this.apiKey,
'Authorization': `Bearer ${this.apiKey}`,
'anthropic-version': '2023-06-01',
},
body: JSON.stringify(body),
});
if (response.statusCode !== 200) {
console.error('[OpenCodeManager] Image analysis failed:', response.body);
const errorMsg = this.parseErrorResponse(response);
return { success: false, error: errorMsg };
}
const data = JSON.parse(response.body);
// Extract text from Anthropic response
let responseText = '';
for (const block of data.content || []) {
if (block.type === 'text') {
responseText += block.text;
}
}
// Parse the JSON response
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
if (!jsonMatch) {
console.error('[OpenCodeManager] No JSON found in image analysis response:', responseText);
return { success: false, error: 'Invalid response format from AI' };
}
const result = JSON.parse(jsonMatch[0]);
return {
success: true,
alt: result.alt || undefined,
caption: result.caption || undefined,
};
} catch (error) {
console.error('[OpenCodeManager] Error analyzing media image:', error);
return { success: false, error: (error as Error).message };
}
}
private httpRequest(
urlStr: string,
options: {

68
src/main/engine/PostEngine.ts
View File

@@ -542,14 +542,12 @@ export class PostEngine extends EventEmitter {
.offset(offset)
.all();
const items: PostData[] = [];
for (const dbPost of dbPosts) {
const postData = await this.getPost(dbPost.id);
if (postData) {
items.push(postData);
}
}
// For listing, we don't need to load content from filesystem.
// Use DB content for drafts, empty string for published posts.
// This avoids expensive filesystem reads for each post.
const items: PostData[] = dbPosts.map(dbPost =>
this.dbRowToPostData(dbPost, dbPost.content || '')
);
return {
items,
@@ -571,16 +569,9 @@ export class PostEngine extends EventEmitter {
.orderBy(desc(posts.createdAt))
.all();
const result: PostData[] = [];
for (const dbPost of dbPosts) {
const postData = await this.getPost(dbPost.id);
if (postData) {
result.push(postData);
}
}
return result;
// Use DB content for drafts, empty string for published posts.
// This avoids expensive filesystem reads.
return dbPosts.map(dbPost => this.dbRowToPostData(dbPost, dbPost.content || ''));
}
async getPostsByStatus(status: 'draft' | 'published' | 'archived'): Promise<PostData[]> {
@@ -595,16 +586,9 @@ export class PostEngine extends EventEmitter {
.orderBy(desc(posts.createdAt))
.all();
const result: PostData[] = [];
for (const dbPost of dbPosts) {
const postData = await this.getPost(dbPost.id);
if (postData) {
result.push(postData);
}
}
return result;
// Use DB content for drafts, empty string for published posts.
// This avoids expensive filesystem reads.
return dbPosts.map(dbPost => this.dbRowToPostData(dbPost, dbPost.content || ''));
}
async getPostsFiltered(filter: PostFilter): Promise<PostData[]> {
@@ -647,21 +631,21 @@ export class PostEngine extends EventEmitter {
let result: PostData[] = [];
for (const dbPost of dbPosts) {
const postData = await this.getPost(dbPost.id);
if (postData) {
// Client-side filtering for tags/categories (JSON array)
if (filter.tags && filter.tags.length > 0) {
const hasAllTags = filter.tags.every(tag => postData.tags.includes(tag));
if (!hasAllTags) continue;
}
if (filter.categories && filter.categories.length > 0) {
const hasAnyCategory = filter.categories.some(cat => postData.categories.includes(cat));
if (!hasAnyCategory) continue;
}
result.push(postData);
// Use DB data directly instead of reading from filesystem
const postData = this.dbRowToPostData(dbPost, dbPost.content || '');
// Client-side filtering for tags/categories (JSON array)
if (filter.tags && filter.tags.length > 0) {
const hasAllTags = filter.tags.every(tag => postData.tags.includes(tag));
if (!hasAllTags) continue;
}
if (filter.categories && filter.categories.length > 0) {
const hasAnyCategory = filter.categories.some(cat => postData.categories.includes(cat));
if (!hasAnyCategory) continue;
}
result.push(postData);
}
return result;

13
src/main/ipc/chatHandlers.ts
View File

@@ -330,6 +330,19 @@ export function registerChatHandlers(): void {
return { success: false, error: (error as Error).message };
}
});
// ============ Media Analysis ============
// Analyze a media image and generate alt text and caption
ipcMain.handle('chat:analyzeMediaImage', async (_, mediaId: string, language?: string) => {
try {
const manager = getOpenCodeManager();
return await manager.analyzeMediaImage(mediaId, language || 'en');
} catch (error) {
console.error('[Chat IPC] Error analyzing media image:', error);
return { success: false, error: (error as Error).message };
}
});
}
/**

2
src/main/ipc/handlers.ts
View File

@@ -635,7 +635,7 @@ export function registerIpcHandlers(): void {
return engine.getProjectMetadata();
});
safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string }) => {
safeHandle('meta:updateProjectMetadata', async (_, updates: { name?: string; description?: string; dataPath?: string; mainLanguage?: string }) => {
const engine = getMetaEngine();
await engine.updateProjectMetadata(updates);
return engine.getProjectMetadata();

5
src/main/preload.ts
View File

@@ -132,7 +132,7 @@ contextBridge.exposeInMainWorld('electronAPI', {
syncOnStartup: () => ipcRenderer.invoke('meta:syncOnStartup'),
getProjectMetadata: () => ipcRenderer.invoke('meta:getProjectMetadata'),
setProjectMetadata: (metadata: { name: string; description?: string }) => ipcRenderer.invoke('meta:setProjectMetadata', metadata),
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates),
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; mainLanguage?: string }) => ipcRenderer.invoke('meta:updateProjectMetadata', updates),
},
// Tag Management (advanced tag operations)
@@ -207,6 +207,9 @@ contextBridge.exposeInMainWorld('electronAPI', {
// Taxonomy Analysis
analyzeTaxonomy: (categories: Array<{ name: string; slug: string; existsInProject: boolean }>, tags: Array<{ name: string; slug: string; existsInProject: boolean }>, modelId: string) => ipcRenderer.invoke('chat:analyzeTaxonomy', categories, tags, modelId),
// Media Analysis
analyzeMediaImage: (mediaId: string, language?: string) => ipcRenderer.invoke('chat:analyzeMediaImage', mediaId, language),
// Event listeners for streaming/progress
onStreamDelta: (callback: (data: { conversationId: string; delta: string }) => void) => {
const subscription = (_event: Electron.IpcRendererEvent, data: { conversationId: string; delta: string }) => callback(data);

77
src/renderer/components/Editor/Editor.css
View File

@@ -950,3 +950,80 @@
grid-template-columns: repeat(3, 1fr);
}
}
/* Quick Actions Dropdown */
.quick-actions-wrapper {
position: relative;
display: inline-block;
}
.quick-actions-btn {
display: flex;
align-items: center;
gap: 4px;
white-space: nowrap;
}
.quick-actions-btn:disabled {
opacity: 0.7;
cursor: wait;
}
.quick-actions-menu {
position: absolute;
top: 100%;
right: 0;
margin-top: 4px;
min-width: 280px;
background: var(--vscode-dropdown-background, #3c3c3c);
border: 1px solid var(--vscode-dropdown-border, #454545);
border-radius: 6px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
z-index: 1000;
overflow: hidden;
}
.quick-action-item {
display: flex;
align-items: flex-start;
gap: 10px;
width: 100%;
padding: 10px 12px;
background: none;
border: none;
color: var(--vscode-dropdown-foreground, #ccc);
cursor: pointer;
text-align: left;
transition: background 0.1s;
}
.quick-action-item:hover:not(:disabled) {
background: var(--vscode-list-hoverBackground, #2a2d2e);
}
.quick-action-item:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.quick-action-icon {
font-size: 16px;
flex-shrink: 0;
margin-top: 2px;
}
.quick-action-text {
display: flex;
flex-direction: column;
gap: 2px;
}
.quick-action-text strong {
font-size: 13px;
font-weight: 500;
}
.quick-action-text small {
font-size: 11px;
opacity: 0.7;
}

87
src/renderer/components/Editor/Editor.tsx
View File

@@ -1223,6 +1223,65 @@ const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
const [linkedPosts, setLinkedPosts] = useState<{ postId: string; sortOrder: number }[]>([]);
const [showPostPicker, setShowPostPicker] = useState(false);
const [postSearchQuery, setPostSearchQuery] = useState('');
// Quick action menu state
const [showQuickActions, setShowQuickActions] = useState(false);
const [isAnalyzing, setIsAnalyzing] = useState(false);
const [projectLanguage, setProjectLanguage] = useState('en');
const quickActionsRef = useRef<HTMLDivElement>(null);
// Load project language setting
useEffect(() => {
window.electronAPI?.meta.getProjectMetadata().then(metadata => {
if (metadata?.mainLanguage) {
setProjectLanguage(metadata.mainLanguage);
}
});
}, []);
// Close quick actions menu when clicking outside
useEffect(() => {
const handleClickOutside = (event: MouseEvent) => {
if (quickActionsRef.current && !quickActionsRef.current.contains(event.target as Node)) {
setShowQuickActions(false);
}
};
if (showQuickActions) {
document.addEventListener('mousedown', handleClickOutside);
return () => document.removeEventListener('mousedown', handleClickOutside);
}
}, [showQuickActions]);
// Handle AI image analysis for alt text and caption
const handleAIAnalysis = async () => {
if (!item || isAnalyzing) return;
setShowQuickActions(false);
setIsAnalyzing(true);
try {
const result = await window.electronAPI?.chat.analyzeMediaImage(item.id, projectLanguage);
if (result?.success) {
if (result.alt) setAlt(result.alt);
if (result.caption) setCaption(result.caption);
showToast.success('AI analysis complete');
} else {
showErrorModal({
title: 'AI Analysis Failed',
message: result?.error || 'Failed to analyze image',
});
}
} catch (error) {
console.error('Failed to analyze image:', error);
showErrorModal({
title: 'AI Analysis Error',
message: (error as Error).message || 'Failed to analyze image',
});
} finally {
setIsAnalyzing(false);
}
};
// Load linked posts for this media
useEffect(() => {
@@ -1381,6 +1440,34 @@ const MediaEditor: React.FC<{ mediaId: string }> = ({ mediaId }) => {
</div>
</div>
<div className="editor-actions">
{/* Quick Actions Dropdown */}
{item.mimeType.startsWith('image/') && (
<div className="quick-actions-wrapper" ref={quickActionsRef}>
<button
className="secondary quick-actions-btn"
onClick={() => setShowQuickActions(!showQuickActions)}
disabled={isAnalyzing}
title="Quick Actions"
>
{isAnalyzing ? '⏳ Analyzing...' : '⚡ Quick Actions'}
</button>
{showQuickActions && (
<div className="quick-actions-menu">
<button
className="quick-action-item"
onClick={handleAIAnalysis}
disabled={isAnalyzing}
>
<span className="quick-action-icon">🤖</span>
<span className="quick-action-text">
<strong>AI: Generate Alt & Caption</strong>
<small>Uses Claude Sonnet 4.5 to analyze the image</small>
</span>
</button>
</div>
)}
</div>
)}
<button onClick={handleSave}>Save</button>
<button onClick={handleDelete} className="secondary danger">Delete</button>
</div>

48
src/renderer/components/SettingsView/SettingsView.tsx
View File

@@ -110,6 +110,7 @@ export const SettingsView: React.FC = () => {
const [projectDescription, setProjectDescription] = useState('');
const [projectDataPath, setProjectDataPath] = useState('');
const [defaultProjectPath, setDefaultProjectPath] = useState('');
const [projectMainLanguage, setProjectMainLanguage] = useState('en');
// Post categories management
const [postCategories, setPostCategories] = useState<string[]>(DEFAULT_POST_CATEGORIES);
@@ -143,6 +144,13 @@ export const SettingsView: React.FC = () => {
window.electronAPI?.app.getDefaultProjectPath(activeProject.id).then(path => {
setDefaultProjectPath(path);
});
// Load project metadata (includes mainLanguage)
window.electronAPI?.meta.getProjectMetadata().then(metadata => {
if (metadata?.mainLanguage) {
setProjectMainLanguage(metadata.mainLanguage);
}
});
}
}, [activeProject]);
@@ -299,12 +307,13 @@ export const SettingsView: React.FC = () => {
setActiveProject(updated as any);
useAppStore.getState().updateProject(activeProject.id, updated as any);
// Also update project.json to keep dataPath in sync
// Also update project.json to keep dataPath and mainLanguage in sync
await window.electronAPI?.meta.updateProjectMetadata({
name: projectName.trim() || activeProject.name,
description: projectDescription.trim(),
dataPath: projectDataPath.trim() || undefined,
} as any);
mainLanguage: projectMainLanguage,
});
}
showToast.success('Project settings saved');
} catch (error) {
@@ -325,7 +334,7 @@ export const SettingsView: React.FC = () => {
};
// Keywords for each section for search filtering
const projectKeywords = ['project', 'name', 'description', 'blog', 'site', 'path', 'folder', 'location', 'data'];
const projectKeywords = ['project', 'name', 'description', 'blog', 'site', 'path', 'folder', 'location', 'data', 'language'];
const editorKeywords = ['editor', 'mode', 'wysiwyg', 'markdown', 'preview', 'visual'];
const contentKeywords = ['content', 'categories', 'post', 'article', 'picture', 'aside', 'page'];
const aiKeywords = ['ai', 'assistant', 'chat', 'model', 'prompt', 'system', 'api', 'key', 'claude', 'gpt', 'opencode'];
@@ -392,6 +401,39 @@ export const SettingsView: React.FC = () => {
</div>
</SettingRow>
<SettingRow
id="project-language"
label="Main Language"
description="The primary language for your blog content. AI-generated alt text and captions will use this language."
>
<select
id="project-language"
value={projectMainLanguage}
onChange={(e) => setProjectMainLanguage(e.target.value)}
>
<option value="en">English</option>
<option value="de">German (Deutsch)</option>
<option value="es">Spanish (Español)</option>
<option value="fr">French (Français)</option>
<option value="it">Italian (Italiano)</option>
<option value="pt">Portuguese (Português)</option>
<option value="nl">Dutch (Nederlands)</option>
<option value="pl">Polish (Polski)</option>
<option value="ru">Russian (Русский)</option>
<option value="ja">Japanese (日本語)</option>
<option value="zh">Chinese (中文)</option>
<option value="ko">Korean (한국어)</option>
<option value="ar">Arabic (العربية)</option>
<option value="hi">Hindi (हिन्दी)</option>
<option value="tr">Turkish (Türkçe)</option>
<option value="sv">Swedish (Svenska)</option>
<option value="da">Danish (Dansk)</option>
<option value="no">Norwegian (Norsk)</option>
<option value="fi">Finnish (Suomi)</option>
<option value="cs">Czech (Čeština)</option>
</select>
</SettingRow>
<div className="setting-actions">
<button className="primary" onClick={handleSaveProject}>
Save Project Settings

6
src/renderer/types/electron.d.ts vendored
View File

@@ -15,6 +15,7 @@ export interface ProjectMetadata {
name: string;
description?: string;
dataPath?: string;
mainLanguage?: string;
}
export interface ProjectData {
@@ -377,7 +378,7 @@ export interface ElectronAPI {
syncOnStartup: () => Promise<{ tags: string[]; categories: string[]; projectMetadata: ProjectMetadata | null }>;
getProjectMetadata: () => Promise<ProjectMetadata | null>;
setProjectMetadata: (metadata: { name: string; description?: string }) => Promise<ProjectMetadata | null>;
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string }) => Promise<ProjectMetadata | null>;
updateProjectMetadata: (updates: { name?: string; description?: string; dataPath?: string; mainLanguage?: string }) => Promise<ProjectMetadata | null>;
};
tags: {
getAll: () => Promise<TagData[]>;
@@ -436,6 +437,9 @@ export interface ElectronAPI {
// Taxonomy Analysis
analyzeTaxonomy: (categories: Array<{ name: string; slug: string; existsInProject: boolean }>, tags: Array<{ name: string; slug: string; existsInProject: boolean }>, modelId: string) => Promise<{ success: boolean; categoryMappings?: Record<string, string>; tagMappings?: Record<string, string>; error?: string }>;
// Media Analysis
analyzeMediaImage: (mediaId: string, language?: string) => Promise<{ success: boolean; alt?: string; caption?: string; error?: string }>;
// Event listeners for streaming/progress
onStreamDelta: (callback: (data: ChatStreamDelta) => void) => () => void;
onToolCall: (callback: (data: ChatToolCall) => void) => () => void;