forked from Blendman974/faceclaw
Add compatible assistant model discovery
This commit is contained in:
1 parent
77e8a86196
commit
cc8dc544be
9 files changed
+880
-6
No files matched your search
@@ -0,0 +1,160 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
function discoveryEnv(responder) {
|
||||
const requests = [];
|
||||
const load = loader({}, {
|
||||
'../util/http': {
|
||||
fetchTextWithUserAgent: async (url, init) => {
|
||||
requests.push({ url, init });
|
||||
return responder(url, init, requests.length);
|
||||
},
|
||||
},
|
||||
'./sse': {},
|
||||
});
|
||||
const api = load('app/native/openai-chat-models.ts');
|
||||
api.clearCompatibleModelDiscoveryCache();
|
||||
return { api, requests };
|
||||
}
|
||||
|
||||
const response = (body, status = 200) => ({ status, text: async () => typeof body === 'string' ? body : JSON.stringify(body) });
|
||||
const veniceModel = (id, name, tools, extra = {}) => ({
|
||||
id,
|
||||
model_spec: {
|
||||
name,
|
||||
capabilities: { supportsFunctionCalling: tools },
|
||||
...extra,
|
||||
},
|
||||
});
|
||||
|
||||
const options = {
|
||||
baseUrl: 'https://api.venice.ai/api/v1',
|
||||
apiKey: 'venice-key',
|
||||
};
|
||||
|
||||
test('Venice discovery uses its exact API root, auth, timeout, no redirects, and capability metadata', async () => {
|
||||
const env = discoveryEnv(async () => response({ data: [
|
||||
veniceModel('chat-only', 'Chat only', false),
|
||||
veniceModel('offline-tools', 'Offline tools', true, { offline: true }),
|
||||
veniceModel('tools-model', 'Tools model', true, { betaModel: true }),
|
||||
{ id: 'generic-model', name: 'Generic model' },
|
||||
veniceModel('old-model', 'Old model', true, { deprecation: { date: '2027-01-01' } }),
|
||||
] }));
|
||||
const models = await env.api.discoverOpenAiCompatibleModels(options);
|
||||
assert.equal(env.requests.length, 1);
|
||||
assert.equal(env.requests[0].url, 'https://api.venice.ai/api/v1/models');
|
||||
assert.equal(env.requests[0].init.method, 'GET');
|
||||
assert.equal(env.requests[0].init.headers.Authorization, 'Bearer venice-key');
|
||||
assert.equal(env.requests[0].init.timeoutMs, 10_000);
|
||||
assert.equal(env.requests[0].init.dontFollowRedirects, true);
|
||||
assert.equal(models.map(model => model.id).join(','), 'old-model,tools-model,generic-model,chat-only,offline-tools');
|
||||
const tools = models.find(model => model.id === 'tools-model');
|
||||
assert.equal(tools.name, 'Tools model');
|
||||
assert.equal(tools.supportsTools, true);
|
||||
assert.equal(tools.beta, true);
|
||||
assert.equal(models.find(model => model.id === 'old-model').deprecated, true);
|
||||
assert.equal(models.find(model => model.id === 'generic-model').supportsTools, null);
|
||||
assert.equal(models.find(model => model.id === 'chat-only').supportsTools, false);
|
||||
assert.equal(models.find(model => model.id === 'offline-tools').offline, true);
|
||||
});
|
||||
|
||||
test('Open WebUI discovery uses /api/models, supports optional auth, and marks tool support unknown', async () => {
|
||||
const env = discoveryEnv(async () => response({ data: [
|
||||
{ id: 'qwen-local', name: 'Qwen local' },
|
||||
{ id: 'second-model' },
|
||||
] }));
|
||||
const models = await env.api.discoverOpenAiCompatibleModels({
|
||||
baseUrl: 'http://100.64.0.1:3000/api', apiKey: '',
|
||||
});
|
||||
assert.equal(env.requests[0].url, 'http://100.64.0.1:3000/api/models');
|
||||
assert.equal('Authorization' in env.requests[0].init.headers, false);
|
||||
assert.equal(JSON.stringify(models.map(model => [model.id, model.supportsTools])), JSON.stringify([
|
||||
['qwen-local', null], ['second-model', null],
|
||||
]));
|
||||
});
|
||||
|
||||
test('successful catalogs are cached briefly per endpoint and credential; refresh bypasses the cache', async () => {
|
||||
let now = 1000;
|
||||
const env = discoveryEnv(async (_url, _init, count) => response({ data: [{ id: `model-${count}` }] }));
|
||||
const deps = { now: () => now };
|
||||
const first = await env.api.discoverOpenAiCompatibleModels(options, deps);
|
||||
const cached = await env.api.discoverOpenAiCompatibleModels(options, deps);
|
||||
assert.equal(first, cached);
|
||||
assert.equal(env.requests.length, 1);
|
||||
|
||||
await env.api.discoverOpenAiCompatibleModels({ ...options, apiKey: 'other-key' }, deps);
|
||||
assert.equal(env.requests.length, 2, 'a different credential cannot reuse the old response');
|
||||
await env.api.discoverOpenAiCompatibleModels({ ...options, baseUrl: 'https://other.example/api' }, deps);
|
||||
assert.equal(env.requests.length, 3, 'a different endpoint has its own catalog');
|
||||
await env.api.discoverOpenAiCompatibleModels({ ...options, forceRefresh: true }, deps);
|
||||
assert.equal(env.requests.length, 4, 'explicit refresh always fetches');
|
||||
|
||||
now += 61_000;
|
||||
await env.api.discoverOpenAiCompatibleModels(options, deps);
|
||||
assert.equal(env.requests.length, 5, 'the short-lived entry expires');
|
||||
});
|
||||
|
||||
test('invalid URLs and unsupported or malformed responses fail without querying alternate endpoints', async () => {
|
||||
const env = discoveryEnv(async (_url, _init, count) => {
|
||||
if (count === 1) return response('<html>login</html>');
|
||||
if (count === 2) return response({ models: [] });
|
||||
return response({ data: [] });
|
||||
});
|
||||
await assert.rejects(env.api.discoverOpenAiCompatibleModels({ baseUrl: 'not-a-url', apiKey: '' }), /valid compatible API URL/);
|
||||
assert.equal(env.requests.length, 0);
|
||||
await assert.rejects(env.api.discoverOpenAiCompatibleModels(options), /invalid model list/);
|
||||
await assert.rejects(env.api.discoverOpenAiCompatibleModels({ ...options, forceRefresh: true }), /did not return an OpenAI-compatible model list/);
|
||||
assert.deepEqual(env.requests.map(request => request.url), [
|
||||
'https://api.venice.ai/api/v1/models', 'https://api.venice.ai/api/v1/models',
|
||||
]);
|
||||
});
|
||||
|
||||
test('HTTP and reachability errors are actionable and never fall back to inference', async () => {
|
||||
const statuses = [401, 403, 404, 429, 503, 302];
|
||||
for (const status of statuses) {
|
||||
const env = discoveryEnv(async () => response({}, status));
|
||||
await assert.rejects(env.api.discoverOpenAiCompatibleModels(options), error => {
|
||||
assert.match(error.message, new RegExp(String(status)));
|
||||
return true;
|
||||
});
|
||||
assert.equal(env.requests.length, 1);
|
||||
assert.equal(env.requests[0].init.method, 'GET');
|
||||
}
|
||||
const unreachable = discoveryEnv(async () => { throw new Error('socket details'); });
|
||||
await assert.rejects(unreachable.api.discoverOpenAiCompatibleModels(options), /Could not reach the configured server/);
|
||||
});
|
||||
|
||||
test('model parsing skips invalid rows, deduplicates IDs, and caps catalog size', () => {
|
||||
const env = discoveryEnv(async () => response({ data: [] }));
|
||||
const rows = [null, {}, { id: '' }, { id: 'same', name: 'First' }, { id: 'same', name: 'Duplicate' }];
|
||||
for (let index = 0; index < 510; index++) rows.push({ id: `model-${String(index).padStart(3, '0')}` });
|
||||
const parsed = env.api.parseDiscoveredCompatibleModels(rows);
|
||||
assert.equal(parsed.length, 500);
|
||||
assert.equal(parsed.filter(model => model.id === 'same').length, 1);
|
||||
assert.equal(parsed.find(model => model.id === 'same').name, 'First');
|
||||
});
|
||||
|
||||
test('native text HTTP forwards discovery timeout and redirect protections with the app User-Agent', async () => {
|
||||
const requests = [];
|
||||
const load = loader({ global: { isAndroid: false, isIOS: true } }, {
|
||||
'../version': { USER_AGENT: 'Faceclaw/test' },
|
||||
'@nativescript/core/http': {
|
||||
request: async init => {
|
||||
requests.push(init);
|
||||
return { statusCode: 200, content: { toString: () => '{"data":[]}' } };
|
||||
},
|
||||
},
|
||||
});
|
||||
const { fetchTextWithUserAgent } = load('app/util/http.ts');
|
||||
const result = await fetchTextWithUserAgent('https://provider.example/api/models', {
|
||||
headers: { Authorization: 'Bearer secret' },
|
||||
timeoutMs: 10_000,
|
||||
dontFollowRedirects: true,
|
||||
});
|
||||
assert.equal(requests[0].timeout, 10_000);
|
||||
assert.equal(requests[0].dontFollowRedirects, true);
|
||||
assert.equal(requests[0].headers.Authorization, 'Bearer secret');
|
||||
assert.equal(requests[0].headers['User-Agent'], 'Faceclaw/test');
|
||||
assert.equal(JSON.stringify(await result.json()), JSON.stringify({ data: [] }));
|
||||
});
|
||||
@@ -0,0 +1,301 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
const flush = () => new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
class FakeMenuLayer {
|
||||
constructor(title, items, layout) {
|
||||
this.title = title;
|
||||
this.layout = layout;
|
||||
this.setItems(items);
|
||||
}
|
||||
setItems(items, selectedIndex = 0) {
|
||||
this.items = items;
|
||||
this.selectedIndex = items.length
|
||||
? Math.max(0, Math.min(selectedIndex ?? 0, items.length - 1))
|
||||
: null;
|
||||
}
|
||||
get selectedItem() {
|
||||
return this.selectedIndex === null ? null : this.items[this.selectedIndex];
|
||||
}
|
||||
}
|
||||
|
||||
class FakeEditTextSettingLayer {
|
||||
constructor(setting) { this.setting = setting; }
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve, reject;
|
||||
const promise = new Promise((res, rej) => { resolve = res; reject = rej; });
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function model(id, extra = {}) {
|
||||
return {
|
||||
id,
|
||||
name: extra.name ?? id,
|
||||
supportsTools: extra.supportsTools ?? null,
|
||||
offline: extra.offline ?? false,
|
||||
deprecated: extra.deprecated ?? false,
|
||||
beta: extra.beta ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
function pickerEnv({ models = [], saved = '', discover } = {}) {
|
||||
const calls = [];
|
||||
const settings = {
|
||||
baseUrl: 'https://provider.example/api/v1',
|
||||
apiKey: 'secret',
|
||||
model: saved,
|
||||
getBaseUrl() { return this.baseUrl; },
|
||||
getApiKey() { return this.apiKey; },
|
||||
getModel() { return this.model; },
|
||||
setModel(value) { this.model = value; },
|
||||
};
|
||||
const modelSetting = { id: 'compatible-model' };
|
||||
const renders = [];
|
||||
const edits = [];
|
||||
const layers = [{ id: 'settings-base' }];
|
||||
const remove = layer => layer.onRemoved?.();
|
||||
const stack = {
|
||||
getBaseSize: () => ({ width: 320, height: 240 }),
|
||||
push(layer) { layers.push(layer); },
|
||||
topMatches(predicate) { return layers.length > 1 && predicate(layers.at(-1)); },
|
||||
pop() { if (layers.length > 1) remove(layers.pop()); },
|
||||
popThrough(layer) {
|
||||
const index = layers.indexOf(layer);
|
||||
if (index <= 0) return false;
|
||||
for (const removed of layers.splice(index).reverse()) remove(removed);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
const ctx = {
|
||||
stack,
|
||||
actions: {
|
||||
requestRender: () => renders.push('render'),
|
||||
startTextSettingEdit: setting => { edits.push(setting); },
|
||||
},
|
||||
};
|
||||
const load = loader({}, {
|
||||
'./menu': { MenuLayer: FakeMenuLayer },
|
||||
'./dashboard-settings': { EditTextSettingLayer: FakeEditTextSettingLayer },
|
||||
'../native/openai-chat-models': {},
|
||||
});
|
||||
const api = load('app/ui/openai-compatible-model-picker.ts');
|
||||
const discoverImpl = discover ?? (async options => {
|
||||
calls.push(options);
|
||||
return models;
|
||||
});
|
||||
const picker = new api.OpenAiCompatibleModelPicker(
|
||||
settings, modelSetting, discoverImpl, stack.getBaseSize(),
|
||||
);
|
||||
return {
|
||||
api, picker, settings, modelSetting, discoverImpl,
|
||||
calls, renders, edits, layers, ctx,
|
||||
};
|
||||
}
|
||||
|
||||
function row(picker, text) {
|
||||
const found = picker.items.find(item => item.label.includes(text));
|
||||
assert.ok(found, `Missing row ${text}; got: ${picker.items.map(item => item.label).join(' | ')}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
test('catalog shows capabilities, hides offline models, and saves only an explicit selection', async () => {
|
||||
const env = pickerEnv({
|
||||
saved: 'generic-model',
|
||||
models: [
|
||||
model('tools-model', { name: 'Tools Model', supportsTools: true }),
|
||||
model('generic-model', { name: 'Generic Model' }),
|
||||
model('offline-model', { supportsTools: true, offline: true }),
|
||||
],
|
||||
});
|
||||
env.picker.open(env.ctx);
|
||||
assert.equal(row(env.picker, 'Discovering').disabled, true);
|
||||
await flush();
|
||||
assert.equal(env.settings.model, 'generic-model');
|
||||
assert.equal(env.calls.length, 1);
|
||||
assert.equal(env.calls[0].baseUrl, 'https://provider.example/api/v1');
|
||||
assert.equal(env.calls[0].apiKey, 'secret');
|
||||
assert.equal(env.calls[0].forceRefresh, false);
|
||||
assert.equal(env.picker.items.some(item => item.label.includes('offline-model')), false);
|
||||
assert.match(row(env.picker, 'Tools Model').label, /\[tools\]/);
|
||||
assert.match(row(env.picker, 'Generic Model').label, /\[tools\?\]/);
|
||||
assert.ok(row(env.picker, 'Generic Model').label.endsWith(' *'));
|
||||
assert.equal(env.picker.selectedItem.label, row(env.picker, 'Generic Model').label);
|
||||
assert.equal(env.picker.layout.width, 280);
|
||||
assert.equal(env.picker.layout.maxHeight, 200);
|
||||
await row(env.picker, 'Tools Model').onSelect(env.ctx, env.picker);
|
||||
assert.equal(env.settings.model, 'tools-model');
|
||||
assert.equal(env.layers.length, 1);
|
||||
});
|
||||
|
||||
test('refresh bypasses cache and preserves an unlisted saved model without auto-selecting', async () => {
|
||||
const env = pickerEnv({
|
||||
saved: 'keep-model',
|
||||
discover: async options => {
|
||||
env.calls.push(options);
|
||||
return [model(env.calls.length === 1 ? 'first-model' : 'second-model')];
|
||||
},
|
||||
});
|
||||
env.picker.open(env.ctx);
|
||||
await flush();
|
||||
assert.match(row(env.picker, 'keep-model').label, /saved only/);
|
||||
assert.equal(env.picker.selectedItem.label, row(env.picker, 'keep-model').label);
|
||||
await row(env.picker, 'Refresh models').onSelect(env.ctx, env.picker);
|
||||
await flush();
|
||||
assert.equal(env.calls.length, 2);
|
||||
assert.equal(env.calls[1].forceRefresh, true);
|
||||
assert.equal(env.settings.model, 'keep-model');
|
||||
assert.ok(row(env.picker, 'second-model'));
|
||||
assert.ok(row(env.picker, 'keep-model'));
|
||||
});
|
||||
|
||||
test('cancel invalidates a pending result and manual entry remains available while loading', async () => {
|
||||
const pending = deferred();
|
||||
const env = pickerEnv({
|
||||
saved: 'original-model',
|
||||
discover: options => { env.calls.push(options); return pending.promise; },
|
||||
});
|
||||
env.picker.open(env.ctx);
|
||||
await row(env.picker, 'Cancel discovery').onSelect(env.ctx, env.picker);
|
||||
assert.equal(env.layers.length, 1);
|
||||
const renderCount = env.renders.length;
|
||||
pending.resolve([model('late-model')]);
|
||||
await flush();
|
||||
assert.equal(env.settings.model, 'original-model');
|
||||
assert.equal(env.renders.length, renderCount);
|
||||
|
||||
const second = deferred();
|
||||
const manual = pickerEnv({
|
||||
saved: 'original-model',
|
||||
discover: () => second.promise,
|
||||
});
|
||||
manual.picker.open(manual.ctx);
|
||||
await row(manual.picker, 'Enter model manually').onSelect(manual.ctx, manual.picker);
|
||||
assert.equal(manual.edits.length, 1);
|
||||
assert.equal(manual.edits[0], manual.modelSetting);
|
||||
assert.equal(manual.layers.length, 2);
|
||||
assert.ok(manual.layers[1] instanceof FakeEditTextSettingLayer);
|
||||
second.resolve([model('late-model')]);
|
||||
await flush();
|
||||
assert.equal(manual.settings.model, 'original-model');
|
||||
assert.ok(manual.layers[1] instanceof FakeEditTextSettingLayer);
|
||||
});
|
||||
|
||||
test('offline models are optional but a saved offline model remains selectable', async () => {
|
||||
const env = pickerEnv({
|
||||
saved: 'saved-offline',
|
||||
models: [
|
||||
model('online-model'),
|
||||
model('saved-offline', { offline: true }),
|
||||
model('other-offline', { offline: true }),
|
||||
],
|
||||
});
|
||||
env.picker.open(env.ctx);
|
||||
await flush();
|
||||
assert.ok(row(env.picker, 'saved-offline'));
|
||||
assert.equal(env.picker.items.some(item => item.label.includes('other-offline')), false);
|
||||
await row(env.picker, 'Show offline (2)').onSelect(env.ctx, env.picker);
|
||||
assert.ok(row(env.picker, 'other-offline'));
|
||||
assert.match(row(env.picker, 'other-offline').label, /offline/);
|
||||
await row(env.picker, 'Hide offline models').onSelect(env.ctx, env.picker);
|
||||
assert.equal(env.picker.items.some(item => item.label.includes('other-offline')), false);
|
||||
assert.equal(env.settings.model, 'saved-offline');
|
||||
});
|
||||
|
||||
test('errors retain the saved model and Try again reloads the current endpoint', async () => {
|
||||
const first = deferred();
|
||||
const env = pickerEnv({
|
||||
saved: 'original-model',
|
||||
discover: options => {
|
||||
env.calls.push(options);
|
||||
if (env.calls.length === 1) return first.promise;
|
||||
return Promise.resolve([model('new-endpoint-model')]);
|
||||
},
|
||||
});
|
||||
env.picker.open(env.ctx);
|
||||
env.settings.baseUrl = 'https://other.example/api';
|
||||
first.resolve([model('old-endpoint-model')]);
|
||||
await flush();
|
||||
assert.match(row(env.picker, 'Failed:').description, /URL or key changed/);
|
||||
assert.equal(env.picker.items.some(item => item.label.includes('old-endpoint-model')), false);
|
||||
assert.equal(env.settings.model, 'original-model');
|
||||
await row(env.picker, 'Try again').onSelect(env.ctx, env.picker);
|
||||
await flush();
|
||||
assert.equal(env.calls[1].baseUrl, 'https://other.example/api');
|
||||
assert.equal(env.calls[1].forceRefresh, true);
|
||||
assert.ok(row(env.picker, 'new-endpoint-model'));
|
||||
assert.equal(env.settings.model, 'original-model');
|
||||
});
|
||||
|
||||
test('a stale catalog cannot be selected after the endpoint changes', async () => {
|
||||
const env = pickerEnv({ models: [model('old-endpoint-model')] });
|
||||
env.picker.open(env.ctx);
|
||||
await flush();
|
||||
const staleRow = row(env.picker, 'old-endpoint-model');
|
||||
env.settings.baseUrl = 'https://other.example/api';
|
||||
await staleRow.onSelect(env.ctx, env.picker);
|
||||
assert.equal(env.settings.model, '');
|
||||
assert.equal(env.layers.length, 2);
|
||||
});
|
||||
|
||||
test('empty catalogs retain recovery actions and settings row requires a URL', async () => {
|
||||
const env = pickerEnv({ saved: 'keep-model', models: [] });
|
||||
env.picker.open(env.ctx);
|
||||
await flush();
|
||||
assert.ok(row(env.picker, 'No models listed'));
|
||||
assert.ok(row(env.picker, 'Refresh models'));
|
||||
assert.ok(row(env.picker, 'Enter model manually'));
|
||||
env.ctx.stack.pop();
|
||||
|
||||
const item = env.api.compatibleModelDiscoveryMenuItem(
|
||||
env.settings, env.modelSetting, async () => [],
|
||||
);
|
||||
env.settings.baseUrl = '';
|
||||
assert.equal(item.disabled(), true);
|
||||
env.settings.baseUrl = 'https://provider.example/api/v1';
|
||||
assert.equal(item.disabled(), false);
|
||||
item.onSelect(env.ctx);
|
||||
assert.equal(env.layers.length, 2);
|
||||
assert.equal(env.layers[1].layout.width, 280);
|
||||
env.layers[1].onRemoved();
|
||||
});
|
||||
|
||||
test('a temporary overlay does not invalidate a pending result while the picker remains open', async () => {
|
||||
const pending = deferred();
|
||||
const env = pickerEnv({ discover: () => pending.promise });
|
||||
env.picker.open(env.ctx);
|
||||
env.layers.push({ id: 'temporary-overlay' });
|
||||
pending.resolve([model('available-model')]);
|
||||
await flush();
|
||||
assert.equal(env.picker.items.some(item => item.label.includes('available-model')), true);
|
||||
assert.equal(env.layers.length, 3);
|
||||
assert.ok(env.renders.length > 0);
|
||||
});
|
||||
|
||||
test('a discovery error is visible in the modal row and retains manual recovery', async () => {
|
||||
const env = pickerEnv({
|
||||
discover: async () => { throw new Error('Model discovery is not supported at this URL (HTTP 404)'); },
|
||||
});
|
||||
env.picker.open(env.ctx);
|
||||
await flush();
|
||||
assert.match(row(env.picker, 'Failed:').label, /404/);
|
||||
assert.match(row(env.picker, 'Failed:').description, /Manual model entry/);
|
||||
assert.ok(row(env.picker, 'Try again'));
|
||||
assert.ok(row(env.picker, 'Enter model manually'));
|
||||
env.ctx.stack.pop();
|
||||
});
|
||||
|
||||
test('long labels stay bounded while descriptions retain the exact model ID', async () => {
|
||||
const longId = `provider/${'x'.repeat(90)}-model`;
|
||||
const longName = `A very long model name ${'y'.repeat(70)}`;
|
||||
const env = pickerEnv({
|
||||
models: [model(longId, { name: longName, supportsTools: true })],
|
||||
});
|
||||
env.picker.open(env.ctx);
|
||||
await flush();
|
||||
const longRow = row(env.picker, '[tools]');
|
||||
assert.ok(longRow.label.length <= 54);
|
||||
assert.ok(longRow.description.includes(longId));
|
||||
});
|
||||
Reference in new issue
Block a user