wip: next round of implementation, this time tests

This commit is contained in:
2026-02-25 20:38:34 +01:00
parent 20ea499a6f
commit 7808ce74ac
8 changed files with 633 additions and 1 deletions

View File

@@ -0,0 +1,152 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { OpenCodeManager } from '../../src/main/engine/OpenCodeManager';
import { getProtocolTelemetryService } from '../../src/main/agentic/observability/protocolTelemetry';
interface MockConversation {
id: string;
model?: string;
messages: Array<{ role: 'user' | 'assistant' | 'system' | 'tool'; content: string }>;
}
function createChatEngineMock(conversation: MockConversation) {
const settings = new Map<string, string>();
const addMessage = vi.fn().mockResolvedValue(undefined);
return {
getConversation: vi.fn().mockResolvedValue(conversation),
addMessage,
getDefaultSystemPrompt: vi.fn().mockResolvedValue('system prompt'),
getSetting: vi.fn(async (key: string) => settings.get(key) ?? null),
setSetting: vi.fn(async (key: string, value: string) => {
settings.set(key, value);
}),
};
}
describe('OpenCodeManager protocol integration', () => {
beforeEach(() => {
vi.restoreAllMocks();
});
it('uses protocol envelope flow and persists workflow checkpoint for needs_input turns', async () => {
const conversation: MockConversation = {
id: 'conversation-1',
model: 'gpt-5',
messages: [
{ role: 'system', content: 'system' },
{ role: 'user', content: 'previous user message' },
],
};
const chatEngineMock = createChatEngineMock(conversation);
const manager = new OpenCodeManager(
chatEngineMock as never,
{} as never,
{} as never,
() => null,
);
manager.setApiKey('test-api-key');
const providerSpy = vi.spyOn(manager as never, 'sendOpenAIMessage').mockResolvedValue({
content: JSON.stringify({
protocolVersion: '2.0',
assistantText: 'Please provide a date range.',
intent: 'ask_input',
needsInput: {
required: true,
fields: [{ key: 'dateRange', label: 'Date range', inputType: 'date' }],
},
actions: [],
confidence: 0.82,
traceId: 'trace-needs-input',
}),
toolCalls: [],
});
const telemetryBefore = getProtocolTelemetryService().getSnapshot();
const result = await manager.sendMessage('conversation-1', 'Generate report', {
metadata: { surface: 'tab' },
});
expect(result.success).toBe(true);
expect(result.envelope?.protocolVersion).toBe('2.0');
expect(result.envelope?.needsInput.required).toBe(true);
expect(result.envelope?.needsInput.fields[0]?.key).toBe('dateRange');
expect(providerSpy).toHaveBeenCalledTimes(1);
const providerMessages = providerSpy.mock.calls[0][2] as Array<{ role: string; content?: string }>;
const latestUserMessage = providerMessages[providerMessages.length - 1]?.content ?? '';
expect(latestUserMessage).toContain('[Protocol request envelope]');
expect(latestUserMessage).toContain('"protocolVersion": "2.0"');
expect(latestUserMessage).toContain('"surface": "tab"');
const checkpointKey = 'agui.workflow.conversation-1';
expect(chatEngineMock.setSetting).toHaveBeenCalledWith(
checkpointKey,
expect.any(String),
);
const persistedCheckpoint = await chatEngineMock.getSetting(checkpointKey);
const checkpoint = JSON.parse(persistedCheckpoint as string) as { state: string; pendingFields: string[]; lastTraceId: string };
expect(checkpoint.state).toBe('awaiting_input');
expect(checkpoint.pendingFields).toEqual(['dateRange']);
expect(checkpoint.lastTraceId).toBe('trace-needs-input');
const telemetryAfter = getProtocolTelemetryService().getSnapshot();
expect(telemetryAfter.totalTurns).toBe(telemetryBefore.totalTurns + 1);
expect(telemetryAfter.validEnvelopeTurns).toBe(telemetryBefore.validEnvelopeTurns + 1);
});
it('blocks unsupported actions and records blocked-action telemetry', async () => {
const conversation: MockConversation = {
id: 'conversation-2',
model: 'gpt-5',
messages: [{ role: 'user', content: 'previous user message' }],
};
const chatEngineMock = createChatEngineMock(conversation);
const manager = new OpenCodeManager(
chatEngineMock as never,
{} as never,
{} as never,
() => null,
);
manager.setApiKey('test-api-key');
vi.spyOn(manager as never, 'sendOpenAIMessage').mockResolvedValue({
content: JSON.stringify({
protocolVersion: '2.0',
assistantText: 'Try toggling sidebar.',
intent: 'propose_action',
needsInput: { required: false, fields: [] },
actions: [
{
id: 'a1',
action: 'toggleAssistantSidebar',
label: 'Toggle assistant sidebar',
policy: 'silent',
requiresConfirmation: false,
},
],
confidence: 0.74,
traceId: 'trace-blocked-action',
}),
toolCalls: [],
});
const telemetryBefore = getProtocolTelemetryService().getSnapshot();
const result = await manager.sendMessage('conversation-2', 'Toggle it', {
metadata: { surface: 'tab' },
});
expect(result.success).toBe(true);
expect(result.envelope?.actions).toEqual([]);
expect(result.warnings?.some((warning) => warning.includes('Blocked unsupported action: toggleAssistantSidebar'))).toBe(true);
const telemetryAfter = getProtocolTelemetryService().getSnapshot();
expect(telemetryAfter.blockedActionCount).toBe(telemetryBefore.blockedActionCount + 1);
});
});

View File

@@ -1,7 +1,20 @@
import { describe, expect, it } from 'vitest';
import { ProtocolTelemetryService } from '../../../../src/main/agentic/observability/protocolTelemetry';
import {
ProtocolTelemetryService,
getProtocolTelemetryService,
} from '../../../../src/main/agentic/observability/protocolTelemetry';
describe('ProtocolTelemetryService', () => {
it('returns zero rates before any turns are recorded', () => {
const telemetry = new ProtocolTelemetryService();
const snapshot = telemetry.getSnapshot();
expect(snapshot.totalTurns).toBe(0);
expect(snapshot.parseValidityRate).toBe(0);
expect(snapshot.repairRate).toBe(0);
expect(snapshot.fallbackRate).toBe(0);
});
it('tracks parse validity, repairs, fallback, and blocked actions', () => {
const telemetry = new ProtocolTelemetryService();
@@ -35,4 +48,11 @@ describe('ProtocolTelemetryService', () => {
expect(snapshot.blockedActionCount).toBe(3);
expect(snapshot.parseValidityRate).toBeCloseTo(2 / 3, 5);
});
it('returns the same singleton telemetry service instance', () => {
const first = getProtocolTelemetryService();
const second = getProtocolTelemetryService();
expect(first).toBe(second);
});
});

View File

@@ -114,4 +114,61 @@ describe('ProtocolResponseBuilder', () => {
expect(result.envelope.actions).toHaveLength(0);
expect(result.warnings.some((warning) => warning.includes('openSettings'))).toBe(true);
});
it('extracts declarative actions from UI elements and applies policy defaults', () => {
const builder = new ProtocolResponseBuilder();
const raw = JSON.stringify({
protocolVersion: '2.0',
assistantText: 'Choose an option',
intent: 'propose_action',
needsInput: { required: false, fields: [] },
actions: [],
ui: {
specVersion: '1',
elements: [
{
type: 'card',
title: 'Actions',
body: 'Pick one',
actions: [{ label: 'Delete', action: 'deletePost' }],
},
{
type: 'tabs',
tabs: [
{
id: 'first',
label: 'First',
elements: [{ type: 'action', label: 'Open post', action: 'openPost' }],
},
],
},
],
},
confidence: 0.7,
traceId: 'trace-ui-actions',
});
const result = builder.build({
rawAssistantOutput: raw,
surface: 'tab',
capabilities: {
widgets: ['card', 'tabs', 'action'],
actions: ['deletePost', 'openPost'],
tools: ['search_posts'],
},
});
expect(result.envelope.actions).toHaveLength(2);
expect(result.envelope.actions[0]).toEqual(expect.objectContaining({
action: 'deletePost',
policy: 'danger',
requiresConfirmation: true,
}));
expect(result.envelope.actions[1]).toEqual(expect.objectContaining({
action: 'openPost',
policy: 'silent',
requiresConfirmation: false,
}));
});
});

View File

@@ -0,0 +1,93 @@
import { describe, expect, it } from 'vitest';
import { extractAssistantUiSpec } from '../../../../src/main/agentic/protocol/uiSpecParser';
describe('extractAssistantUiSpec', () => {
it('extracts fenced JSON spec and preserves assistant text around it', () => {
const input = [
'Here is your dashboard.',
'```json',
'{"specVersion":"1","elements":[{"type":"text","text":"Hello"}]}',
'```',
'Anything else?',
].join('\n');
const result = extractAssistantUiSpec(input);
expect(result.ui).not.toBeNull();
expect(result.ui?.specVersion).toBe('1');
expect(result.ui?.elements).toHaveLength(1);
expect(result.assistantText).toContain('Here is your dashboard.');
expect(result.assistantText).toContain('Anything else?');
});
it('normalizes markdown element into canonical text element', () => {
const input = JSON.stringify({
specVersion: '1',
elements: [{ type: 'markdown', content: '## Title' }],
});
const result = extractAssistantUiSpec(input);
const first = result.ui?.elements[0] as { type: string; text?: string };
expect(result.ui).not.toBeNull();
expect(first.type).toBe('text');
expect(first.text).toBe('## Title');
});
it('normalizes chart data datasets into chart series', () => {
const input = JSON.stringify({
specVersion: '1',
elements: [
{
type: 'chart',
chartType: 'bar',
data: {
labels: ['Jan', 'Feb'],
datasets: [{ data: [10, 20] }],
},
},
],
});
const result = extractAssistantUiSpec(input);
const first = result.ui?.elements[0] as { series?: Array<{ label: string; value: number }>; data?: unknown };
expect(result.ui).not.toBeNull();
expect(first.series).toEqual([
{ label: 'Jan', value: 10 },
{ label: 'Feb', value: 20 },
]);
expect(first.data).toBeUndefined();
});
it('normalizes tabs content to nested elements arrays', () => {
const input = JSON.stringify({
type: 'tabs',
tabs: [
{
title: 'Overview',
content: { type: 'text', text: 'Summary' },
},
],
});
const result = extractAssistantUiSpec(input);
const tabs = result.ui?.elements[0] as {
tabs: Array<{ id: string; label: string; elements: Array<{ type: string; text?: string }> }>;
};
expect(result.ui).not.toBeNull();
expect(tabs.tabs).toHaveLength(1);
expect(tabs.tabs[0].id).toBe('tab-1');
expect(tabs.tabs[0].label).toBe('Overview');
expect(tabs.tabs[0].elements[0]).toEqual({ type: 'text', text: 'Summary' });
});
it('returns plain assistant text when malformed JSON is provided', () => {
const input = '{"specVersion":"1","elements":[{"type":"text"}';
const result = extractAssistantUiSpec(input);
expect(result.ui).toBeNull();
expect(result.assistantText).toBe(input);
});
});

View File

@@ -71,4 +71,23 @@ describe('agentic protocol validator', () => {
expect(result.ok).toBe(true);
});
it('rejects invalid request envelope and returns structured protocol error', () => {
const result = validateProtocolRequestEnvelope({
protocolVersion: '2.0',
surface: 'tab',
messages: [{ role: 'invalid-role', content: 'Create a chart' }],
context: { projectId: 'project-1' },
capabilities: {
widgets: ['chart'],
actions: ['openPost'],
tools: ['search_posts'],
},
unknown: true,
});
expect(result.ok).toBe(false);
expect(result.error?.code).toBe('AGUI_PROTOCOL_VALIDATION_ERROR');
expect(result.error?.details?.length).toBeGreaterThan(0);
});
});

View File

@@ -36,4 +36,42 @@ describe('WorkflowCheckpointStore', () => {
expect(loaded?.pendingFields).toEqual(['date']);
expect(loaded?.lastTraceId).toBe('trace-1');
});
it('returns null when no checkpoint exists', async () => {
const adapter = new InMemorySettingsAdapter();
const store = new WorkflowCheckpointStore(adapter);
const loaded = await store.load('missing-conversation');
expect(loaded).toBeNull();
});
it('returns null for malformed checkpoint JSON', async () => {
const adapter = new InMemorySettingsAdapter();
await adapter.setSetting('agui.workflow.conversation-2', '{not-valid-json');
const store = new WorkflowCheckpointStore(adapter);
const loaded = await store.load('conversation-2');
expect(loaded).toBeNull();
});
it('returns null when stored checkpoint conversation id does not match', async () => {
const adapter = new InMemorySettingsAdapter();
await adapter.setSetting(
'agui.workflow.conversation-3',
JSON.stringify({
conversationId: 'other-conversation',
state: 'planning',
pendingFields: [],
lastTraceId: 'trace-mismatch',
updatedAt: new Date('2026-02-25T10:00:00.000Z').toISOString(),
}),
);
const store = new WorkflowCheckpointStore(adapter);
const loaded = await store.load('conversation-3');
expect(loaded).toBeNull();
});
});

View File

@@ -35,4 +35,72 @@ describe('AgentTurnStateMachine', () => {
expect(next).toBe('completed');
});
it('transitions to executing when intent requests action execution', () => {
const stateMachine = new AgentTurnStateMachine();
const next = stateMachine.transition({
previousState: 'planning',
envelope: {
intent: 'execute_action',
needsInput: {
required: false,
fields: [],
},
},
});
expect(next).toBe('executing');
});
it('transitions to observing when proposing actions', () => {
const stateMachine = new AgentTurnStateMachine();
const next = stateMachine.transition({
previousState: 'planning',
envelope: {
intent: 'propose_action',
needsInput: {
required: false,
fields: [],
},
},
});
expect(next).toBe('observing');
});
it('returns to executing after awaiting input when input is no longer required', () => {
const stateMachine = new AgentTurnStateMachine();
const next = stateMachine.transition({
previousState: 'awaiting_input',
envelope: {
intent: 'analyze',
needsInput: {
required: false,
fields: [],
},
},
});
expect(next).toBe('executing');
});
it('stays in planning for non-terminal analyze intent by default', () => {
const stateMachine = new AgentTurnStateMachine();
const next = stateMachine.transition({
previousState: 'planning',
envelope: {
intent: 'analyze',
needsInput: {
required: false,
fields: [],
},
},
});
expect(next).toBe('planning');
});
});

View File

@@ -0,0 +1,185 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const registeredHandlers = new Map<string, (...args: any[]) => Promise<any>>();
const webContentsSend = vi.fn();
const mainWindowMock = {
webContents: {
send: webContentsSend,
},
};
const protocolSnapshot = {
totalTurns: 3,
validEnvelopeTurns: 2,
repairAttempts: 1,
fallbackTurns: 1,
blockedActionCount: 2,
parseValidityRate: 2 / 3,
repairRate: 1 / 3,
fallbackRate: 1 / 3,
};
const telemetryServiceMock = {
getSnapshot: vi.fn(() => protocolSnapshot),
};
const chatEngineInstances: Array<Record<string, any>> = [];
const openCodeManagerInstances: Array<Record<string, any>> = [];
vi.mock('electron', () => ({
BrowserWindow: {
fromWebContents: vi.fn(),
},
ipcMain: {
handle: vi.fn((channel: string, handler: (...args: any[]) => Promise<any>) => {
registeredHandlers.set(channel, handler);
}),
},
}));
vi.mock('../../src/main/database', () => ({
getDatabase: vi.fn(() => ({})),
}));
vi.mock('../../src/main/engine/PostEngine', () => ({
getPostEngine: vi.fn(() => ({})),
}));
vi.mock('../../src/main/engine/MediaEngine', () => ({
getMediaEngine: vi.fn(() => ({})),
}));
vi.mock('../../src/main/agentic/observability/protocolTelemetry', () => ({
getProtocolTelemetryService: vi.fn(() => telemetryServiceMock),
}));
vi.mock('../../src/main/engine/ChatEngine', () => ({
ChatEngine: class {
constructor() {
const instance = {
getSetting: vi.fn(async (key: string) => (key === 'opencode_api_key' ? 'stored-key' : null)),
setSetting: vi.fn(async () => undefined),
getSelectedModel: vi.fn(async () => 'gpt-5'),
getDefaultSystemPrompt: vi.fn(async () => 'system prompt'),
};
chatEngineInstances.push(instance);
return instance;
}
},
}));
vi.mock('../../src/main/engine/OpenCodeManager', () => ({
OpenCodeManager: class {
constructor() {
const instance = {
setApiKey: vi.fn(),
checkReady: vi.fn(async () => ({ ready: true })),
validateApiKey: vi.fn(async () => ({ isValid: true, models: [] })),
getApiKey: vi.fn(() => 'abc12345'),
getAvailableModels: vi.fn(async () => []),
sendMessage: vi.fn(async (_conversationId: string, _message: string, options: any) => {
options?.onDelta?.('stream-delta');
options?.onToolCall?.({ name: 'search_posts', args: { query: 'q' } });
options?.onToolResult?.({ name: 'search_posts', result: { ok: true } });
return {
success: true,
message: 'assistant reply',
envelope: {
protocolVersion: '2.0',
assistantText: 'assistant reply',
intent: 'summarize',
needsInput: { required: false, fields: [] },
actions: [],
confidence: 0.9,
traceId: 'trace-1',
},
protocolVersion: '2.0',
traceId: 'trace-1',
warnings: [],
};
}),
abortMessage: vi.fn(async () => ({ success: true })),
analyzeTaxonomy: vi.fn(async () => ({ success: true })),
analyzeMediaImage: vi.fn(async () => ({ success: true })),
stop: vi.fn(async () => undefined),
};
openCodeManagerInstances.push(instance);
return instance;
}
},
}));
describe('chatHandlers', () => {
beforeEach(() => {
registeredHandlers.clear();
webContentsSend.mockReset();
chatEngineInstances.length = 0;
openCodeManagerInstances.length = 0;
telemetryServiceMock.getSnapshot.mockClear();
vi.resetModules();
});
afterEach(async () => {
const mod = await import('../../src/main/ipc/chatHandlers');
await mod.cleanupChatHandlers();
});
it('returns protocol health snapshot from telemetry service', async () => {
const mod = await import('../../src/main/ipc/chatHandlers');
mod.registerChatHandlers();
const handler = registeredHandlers.get('chat:getProtocolHealth');
expect(handler).toBeDefined();
const result = await handler!();
expect(telemetryServiceMock.getSnapshot).toHaveBeenCalledTimes(1);
expect(result).toEqual(protocolSnapshot);
});
it('streams sendMessage callbacks through main window events', async () => {
const mod = await import('../../src/main/ipc/chatHandlers');
mod.initializeChatHandlers(() => mainWindowMock as never);
mod.registerChatHandlers();
const handler = registeredHandlers.get('chat:sendMessage');
expect(handler).toBeDefined();
const result = await handler!(
undefined,
'conversation-1',
'hello assistant',
{ surface: 'sidebar' },
);
expect(result.success).toBe(true);
expect(result.envelope?.protocolVersion).toBe('2.0');
const manager = openCodeManagerInstances[0];
expect(manager.setApiKey).toHaveBeenCalledWith('stored-key');
expect(manager.sendMessage).toHaveBeenCalledWith(
'conversation-1',
'hello assistant',
expect.objectContaining({
metadata: { surface: 'sidebar' },
onDelta: expect.any(Function),
onToolCall: expect.any(Function),
onToolResult: expect.any(Function),
}),
);
expect(webContentsSend).toHaveBeenCalledWith('chat-stream-delta', {
conversationId: 'conversation-1',
delta: 'stream-delta',
});
expect(webContentsSend).toHaveBeenCalledWith('chat-tool-call', {
conversationId: 'conversation-1',
toolCall: { name: 'search_posts', args: { query: 'q' } },
});
expect(webContentsSend).toHaveBeenCalledWith('chat-tool-result', {
conversationId: 'conversation-1',
result: { name: 'search_posts', result: { ok: true } },
});
});
});