hugo/bDS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: better models.dev support

Browse Source
This commit is contained in:
hugo
2026-03-01 17:03:50 +01:00
parent e2c46e94aa
commit 63674266f5
9 changed files with 2140 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
tests/engine/ModelCatalogEngine.test.ts
View File

@@ -2,7 +2,8 @@
* ModelCatalogEngine Tests
*
* Tests the model catalog engine that fetches and caches
* model metadata from models.dev for the OpenCode provider.
* model metadata from models.dev for ALL providers.
* Three normalised tables: providers → models → modalities.
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
@@ -19,12 +20,37 @@ function createSelectChain(mockData: unknown[] = []) {
return chain;
}
let selectMockData: unknown[] = [];
// Per-table mock data keyed by table name reference
let modelMockData: unknown[] = [];
let modalityMockData: unknown[] = [];
let providerMockData: unknown[] = [];
let metaMockData: unknown[] = [];
const insertedValues: unknown[] = [];
function createDrizzleMock() {
return {
select: vi.fn(() => createSelectChain(selectMockData)),
select: vi.fn(() => {
// Returns a chain whose `.from()` picks the right dataset by table reference
const chain: Record<string, unknown> = {
from: vi.fn().mockImplementation((table: unknown) => {
let data: unknown[];
if (table === modelCatalogModalities) {
data = modalityMockData;
} else if (table === modelCatalogProviders) {
data = providerMockData;
} else if (table === modelCatalogMeta) {
data = metaMockData;
} else {
data = modelMockData;
}
const inner = createSelectChain(data);
return inner;
}),
where: vi.fn().mockImplementation(() => chain),
then: (resolve: (v: unknown) => void) => Promise.resolve(modelMockData).then(resolve),
};
return chain;
}),
insert: vi.fn(() => ({
values: vi.fn((data: unknown) => {
insertedValues.push(data);
@@ -49,13 +75,19 @@ vi.mock('../../src/main/database', () => ({
}));
import { ModelCatalogEngine, DEFAULT_MAX_OUTPUT_TOKENS } from '../../src/main/engine/ModelCatalogEngine';
import { modelCatalog, modelCatalogModalities, modelCatalogProviders, modelCatalogMeta } from '../../src/main/database/schema';
// ── Sample models.dev response ──
// ── Sample models.dev response (multi-provider) ──
function sampleModelsDevResponse() {
return {
opencode: {
id: 'opencode',
name: 'OpenCode Zen',
env: ['OPENCODE_API_KEY'],
npm: '@ai-sdk/openai-compatible',
api: 'https://opencode.ai/zen/v1',
doc: 'https://opencode.ai/docs/zen',
models: {
'claude-sonnet-4-5': {
id: 'claude-sonnet-4-5',
@@ -64,6 +96,7 @@ function sampleModelsDevResponse() {
attachment: true,
reasoning: false,
tool_call: true,
modalities: { input: ['text', 'image', 'pdf'], output: ['text'] },
cost: { input: 3, output: 15, cache_read: 0.3 },
limit: { context: 200000, output: 64000 },
},
@@ -74,6 +107,7 @@ function sampleModelsDevResponse() {
attachment: true,
reasoning: true,
tool_call: true,
modalities: { input: ['text', 'image'], output: ['text'] },
cost: { input: 1.07, output: 8.5, cache_read: 0.107 },
limit: { context: 400000, input: 272000, output: 128000 },
},
@@ -81,10 +115,32 @@ function sampleModelsDevResponse() {
id: 'model-no-cost',
name: 'Free Model',
family: 'free',
modalities: { input: ['text'], output: ['text'] },
limit: { context: 32000, output: 4096 },
},
},
},
mistral: {
id: 'mistral',
name: 'Mistral AI',
env: ['MISTRAL_API_KEY'],
npm: '@mistralai/mistralai',
api: 'https://api.mistral.ai/v1',
doc: 'https://docs.mistral.ai',
models: {
'mistral-large-latest': {
id: 'mistral-large-latest',
name: 'Mistral Large',
family: 'mistral',
attachment: true,
reasoning: false,
tool_call: true,
modalities: { input: ['text', 'image'], output: ['text'] },
cost: { input: 2, output: 6 },
limit: { context: 128000, output: 8192 },
},
},
},
};
}
@@ -93,53 +149,75 @@ describe('ModelCatalogEngine', () => {
beforeEach(() => {
vi.clearAllMocks();
selectMockData = [];
modelMockData = [];
modalityMockData = [];
providerMockData = [];
metaMockData = [];
insertedValues.length = 0;
engine = new ModelCatalogEngine();
});
describe('getAll', () => {
it('returns all cached model catalog entries', async () => {
selectMockData = [
it('returns all cached model catalog entries with modalities', async () => {
modelMockData = [
{
id: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5', family: 'claude-sonnet',
provider: 'opencode', modelId: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5',
family: 'claude-sonnet', attachment: true, reasoning: false, toolCall: true,
structuredOutput: false, temperature: false, knowledge: null,
releaseDate: null, lastUpdatedDate: null, openWeights: false,
contextWindow: 200000, maxInputTokens: null, maxOutputTokens: 64000,
inputPrice: 3, outputPrice: 15, cacheReadPrice: 0.3,
supportsAttachments: true, supportsReasoning: false, supportsToolCall: true,
inputPrice: 3, outputPrice: 15, cacheReadPrice: 0.3, cacheWritePrice: null,
interleaved: null, status: null, providerNpm: null,
},
];
modalityMockData = [
{ provider: 'opencode', modelId: 'claude-sonnet-4-5', direction: 'input', modality: 'text' },
{ provider: 'opencode', modelId: 'claude-sonnet-4-5', direction: 'input', modality: 'image' },
{ provider: 'opencode', modelId: 'claude-sonnet-4-5', direction: 'output', modality: 'text' },
];
const result = await engine.getAll();
expect(result).toHaveLength(1);
expect(result[0].id).toBe('claude-sonnet-4-5');
expect(result[0].provider).toBe('opencode');
expect(result[0].maxOutputTokens).toBe(64000);
expect(result[0].inputPrice).toBe(3);
expect(result[0].inputModalities).toEqual(['text', 'image']);
expect(result[0].outputModalities).toEqual(['text']);
});
it('returns empty array when no catalog entries exist', async () => {
selectMockData = [];
const result = await engine.getAll();
expect(result).toEqual([]);
});
});
describe('getModel', () => {
it('returns a specific model by ID', async () => {
selectMockData = [{
id: 'gpt-5', name: 'GPT 5', family: 'gpt',
it('returns a specific model by ID (cross-provider search)', async () => {
modelMockData = [{
provider: 'opencode', modelId: 'gpt-5', name: 'GPT 5', family: 'gpt',
attachment: true, reasoning: true, toolCall: true,
structuredOutput: false, temperature: false, knowledge: null,
releaseDate: null, lastUpdatedDate: null, openWeights: false,
contextWindow: 400000, maxInputTokens: 272000, maxOutputTokens: 128000,
inputPrice: 1.07, outputPrice: 8.5, cacheReadPrice: 0.107,
supportsAttachments: true, supportsReasoning: true, supportsToolCall: true,
inputPrice: 1.07, outputPrice: 8.5, cacheReadPrice: 0.107, cacheWritePrice: null,
interleaved: null, status: null, providerNpm: null,
}];
modalityMockData = [
{ provider: 'opencode', modelId: 'gpt-5', direction: 'input', modality: 'text' },
{ provider: 'opencode', modelId: 'gpt-5', direction: 'input', modality: 'image' },
];
const result = await engine.getModel('gpt-5');
expect(result).not.toBeNull();
expect(result!.name).toBe('GPT 5');
expect(result!.maxOutputTokens).toBe(128000);
expect(result!.inputModalities).toEqual(['text', 'image']);
});
it('returns null for unknown model', async () => {
selectMockData = [];
modelMockData = [];
modalityMockData = [];
const result = await engine.getModel('nonexistent');
expect(result).toBeNull();
});
@@ -147,38 +225,92 @@ describe('ModelCatalogEngine', () => {
describe('getMaxOutputTokens', () => {
it('returns output tokens from catalog when available', async () => {
selectMockData = [{
id: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5', family: 'claude-sonnet',
modelMockData = [{
provider: 'opencode', modelId: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5',
family: 'claude-sonnet', attachment: true, reasoning: false, toolCall: true,
structuredOutput: false, temperature: false, knowledge: null,
releaseDate: null, lastUpdatedDate: null, openWeights: false,
contextWindow: 200000, maxInputTokens: null, maxOutputTokens: 64000,
inputPrice: 3, outputPrice: 15, cacheReadPrice: 0.3,
supportsAttachments: true, supportsReasoning: false, supportsToolCall: true,
inputPrice: 3, outputPrice: 15, cacheReadPrice: 0.3, cacheWritePrice: null,
interleaved: null, status: null, providerNpm: null,
}];
modalityMockData = [];
const result = await engine.getMaxOutputTokens('claude-sonnet-4-5');
expect(result).toBe(64000);
});
it('returns DEFAULT_MAX_OUTPUT_TOKENS for uncatalogued model', async () => {
selectMockData = [];
modelMockData = [];
modalityMockData = [];
const result = await engine.getMaxOutputTokens('unknown-model');
expect(result).toBe(DEFAULT_MAX_OUTPUT_TOKENS);
});
it('returns DEFAULT_MAX_OUTPUT_TOKENS when model has null maxOutputTokens', async () => {
selectMockData = [{
id: 'weird-model', name: 'Weird', family: null,
modelMockData = [{
provider: 'opencode', modelId: 'weird-model', name: 'Weird', family: null,
attachment: false, reasoning: false, toolCall: false,
structuredOutput: false, temperature: false, knowledge: null,
releaseDate: null, lastUpdatedDate: null, openWeights: false,
contextWindow: null, maxInputTokens: null, maxOutputTokens: null,
inputPrice: null, outputPrice: null, cacheReadPrice: null,
supportsAttachments: false, supportsReasoning: false, supportsToolCall: false,
inputPrice: null, outputPrice: null, cacheReadPrice: null, cacheWritePrice: null,
interleaved: null, status: null, providerNpm: null,
}];
modalityMockData = [];
const result = await engine.getMaxOutputTokens('weird-model');
expect(result).toBe(DEFAULT_MAX_OUTPUT_TOKENS);
});
});
describe('hasInputModality', () => {
it('returns true when model has the modality', async () => {
modelMockData = [{
provider: 'opencode', modelId: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5',
family: 'claude-sonnet', attachment: true, reasoning: false, toolCall: true,
structuredOutput: false, temperature: false, knowledge: null,
releaseDate: null, lastUpdatedDate: null, openWeights: false,
contextWindow: 200000, maxInputTokens: null, maxOutputTokens: 64000,
inputPrice: 3, outputPrice: 15, cacheReadPrice: 0.3, cacheWritePrice: null,
interleaved: null, status: null, providerNpm: null,
}];
modalityMockData = [
{ provider: 'opencode', modelId: 'claude-sonnet-4-5', direction: 'input', modality: 'image' },
];
const result = await engine.hasInputModality('claude-sonnet-4-5', 'image');
expect(result).toBe(true);
});
it('returns false when model lacks the modality', async () => {
modelMockData = [{
provider: 'opencode', modelId: 'text-only', name: 'Text Only',
family: null, attachment: false, reasoning: false, toolCall: false,
structuredOutput: false, temperature: false, knowledge: null,
releaseDate: null, lastUpdatedDate: null, openWeights: false,
contextWindow: 32000, maxInputTokens: null, maxOutputTokens: 4096,
inputPrice: null, outputPrice: null, cacheReadPrice: null, cacheWritePrice: null,
interleaved: null, status: null, providerNpm: null,
}];
modalityMockData = [
{ provider: 'opencode', modelId: 'text-only', direction: 'input', modality: 'text' },
];
const result = await engine.hasInputModality('text-only', 'image');
expect(result).toBe(false);
});
it('returns false for unknown model', async () => {
modelMockData = [];
modalityMockData = [];
const result = await engine.hasInputModality('nonexistent', 'image');
expect(result).toBe(false);
});
});
describe('refresh', () => {
it('parses models.dev response and inserts models into DB', async () => {
it('parses multi-provider models.dev response and inserts all providers and models', async () => {
const mockResponse = sampleModelsDevResponse();
vi.spyOn(engine as any, 'httpGet').mockResolvedValue({
statusCode: 200,
@@ -186,13 +318,16 @@ describe('ModelCatalogEngine', () => {
headers: { etag: '"abc123"' },
});
// getMeta returns null (no existing etag)
selectMockData = [];
metaMockData = [];
const result = await engine.refresh();
expect(result.success).toBe(true);
expect(result.modelsUpdated).toBe(3);
// 3 opencode models + 1 mistral model = 4
expect(result.modelsUpdated).toBe(4);
expect(result.notModified).toBeUndefined();
// Should have inserted provider rows and model rows and modality rows
expect(insertedValues.length).toBeGreaterThan(0);
});
it('sends If-None-Match header when ETag is cached', async () => {
@@ -208,7 +343,9 @@ describe('ModelCatalogEngine', () => {
mockLocalDb.select = vi.fn(() => {
metaCallCount++;
if (metaCallCount === 1) {
return createSelectChain([{ key: 'etag', value: '"old-etag"' }]);
// getMeta('etag') → picks up model_catalog_meta table
const chain = createSelectChain([{ key: 'etag', value: '"old-etag"' }]);
return { from: vi.fn().mockReturnValue({ where: vi.fn().mockReturnValue(chain), ...chain }), ...chain };
}
return createSelectChain([]);
}) as any;
@@ -231,7 +368,7 @@ describe('ModelCatalogEngine', () => {
body: 'Internal Server Error',
headers: {},
});
selectMockData = [];
metaMockData = [];
const result = await engine.refresh();
expect(result.success).toBe(false);
@@ -240,24 +377,24 @@ describe('ModelCatalogEngine', () => {
it('handles network errors gracefully', async () => {
vi.spyOn(engine as any, 'httpGet').mockRejectedValue(new Error('ECONNREFUSED'));
selectMockData = [];
metaMockData = [];
const result = await engine.refresh();
expect(result.success).toBe(false);
expect(result.error).toBe('ECONNREFUSED');
});
it('handles invalid response (missing opencode provider)', async () => {
it('handles invalid response (no providers)', async () => {
vi.spyOn(engine as any, 'httpGet').mockResolvedValue({
statusCode: 200,
body: JSON.stringify({ other_provider: { models: {} } }),
body: JSON.stringify({}),
headers: {},
});
selectMockData = [];
metaMockData = [];
const result = await engine.refresh();
expect(result.success).toBe(false);
expect(result.error).toContain('no opencode models');
expect(result.error).toContain('no providers');
});
it('handles malformed JSON gracefully', async () => {
@@ -266,7 +403,7 @@ describe('ModelCatalogEngine', () => {
body: 'not valid json {{{',
headers: {},
});
selectMockData = [];
metaMockData = [];
const result = await engine.refresh();
expect(result.success).toBe(false);