Port assistant integration (with cloud AIs) to iOS

This commit is contained in:
James Babcock committed 2026-09-19 00:02:15 -07:00
1 parent 9e18745e53
commit 617cdc39eb
25 files changed
+508 -63

No files matched your search

+1 -1
View File
@@ -46,7 +46,7 @@ function conversations(saved = '') {
const env = sessions();
const { AssistantConversations } = load('app/assistant/conversations.ts', {
'./session': { AssistantSession: env.AssistantSession },
'./models': { ASSISTANT_MODEL_VALUES: ['auto', 'terra', 'sonnet'] },
'./models': { supportedAssistantModel: value => value, ASSISTANT_MODEL_VALUES: ['auto', 'terra', 'sonnet'] },
});
let stored = '';
const store = new AssistantConversations((model, effort) => config(model, effort), () => 'terra', (value) => { stored = value; }, saved);
+10 -2
View File
@@ -58,9 +58,9 @@ test('iOS worker frames preserve baked grayscale bytes through the message bound
test('settings-driven repaint runs after font cache invalidation, regardless of subscription order', () => {
const listeners = [], tasks = [], painted = [];
let cachedFont = 'Light';
const api = load('app/ui/dashboard-settings.ts', {
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
: { Layer: class {}, ASSISTANT_MODEL_VALUES: [] },
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
setTimeout: fn => tasks.push(fn),
});
api.onAnySettingChanged(() => painted.push(cachedFont));
@@ -89,6 +89,10 @@ test('background glasses input still composites frames; phone resume preserves t
}
const settings = { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } };
const modules = {
'../assistant/system-tools': { registerSystemTools() {} },
'../assistant/window-tools': { registerWindowTools() {} },
'../assistant/navigate-tools': { registerNavigateTools() {} },
'../assistant/roam-tools': { registerRoamTools() {} },
"../native/ios-navigation-sensors": {},
'../native/compass.ios': { bindCompassSession() {}, receiveCompassEvent() {} },
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) }, knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
@@ -158,6 +162,10 @@ test('iOS bandwidth footer toggles live, polls only in foreground and resets its
on: (key, fn) => appEvents.set(key, fn), off: key => appEvents.delete(key) },
Button: View, Color: class {}, Dialogs: {}, GridLayout: View, Image: View, Label: View, Page: View, StackLayout: View };
const modules = {
'../assistant/system-tools': { registerSystemTools() {} },
'../assistant/window-tools': { registerWindowTools() {} },
'../assistant/navigate-tools': { registerNavigateTools() {} },
'../assistant/roam-tools': { registerRoamTools() {} },
"../native/ios-navigation-sensors": {},
'../native/compass.ios': { bindCompassSession() {}, receiveCompassEvent() {} },
'@nativescript/core': core,
+129
View File
@@ -0,0 +1,129 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
const flush = () => new Promise(resolve => setImmediate(resolve));
function assistant(provider) {
const requests = [], calls = [], deltas = [], errors = [], done = [];
const registry = { listTools: () => [{ name: 'timer.set', description: 'Set a timer', inputSchema: { type: 'object' } }],
callTool: async (name, args) => { calls.push({ name, args }); return { ok: true, content: 'Timer set' }; } };
const load = loader({ global: { isIOS: true, isAndroid: false }, setTimeout, clearTimeout }, {
'./sse': { openSseRequest: (url, body, headers, listener) => {
const request = { url, body: JSON.parse(body), headers, listener, cancelled: false };
requests.push(request); return { cancel: () => { request.cancelled = true; } };
} },
'../native/llama': { streamLocalQwen: () => assert.fail('iOS must not invoke a local LLM') },
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'test', buildAssistantSystemPrompt: () => 'test', describeAssistantContext: () => '' },
});
const { AssistantSession } = load('app/assistant/session.ts');
const session = new AssistantSession({ kind: 'direct', llm: { selection: 'auto', provider, model: 'fixture-model', apiKey: 'fixture-key', effort: 'low' } }, registry);
const send = text => session.sendUtterance(text, {}, { onTextDelta: (_delta, full) => deltas.push(full),
onToolActivity() {}, onError: error => errors.push(error), onTurnDone: result => done.push(result) });
const event = (request, event) => request.listener.onLine('data: ' + JSON.stringify(event));
function text(request, value) {
if (provider === 'anthropic') {
event(request, { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } });
event(request, { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: value } });
} else event(request, { type: 'response.output_text.delta', output_index: 0, delta: value });
}
function finish(request, tool = false) {
if (provider === 'anthropic') {
event(request, { type: 'message_delta', delta: { stop_reason: tool ? 'tool_use' : 'end_turn' } });
event(request, { type: 'message_stop' });
} else event(request, { type: 'response.completed' });
}
function tool(request) {
if (provider === 'anthropic') {
event(request, { type: 'content_block_start', index: 1, content_block: { type: 'tool_use', id: 'tool-1', name: 'timer_set' } });
event(request, { type: 'content_block_delta', index: 1, delta: { type: 'input_json_delta', partial_json: '{"seconds":' } });
event(request, { type: 'content_block_delta', index: 1, delta: { type: 'input_json_delta', partial_json: '60}' } });
} else {
event(request, { type: 'response.output_item.done', output_index: 1, item: { type: 'reasoning', id: 'reason', encrypted_content: 'opaque-fixture' } });
event(request, { type: 'response.output_item.added', output_index: 2, item: { type: 'function_call', call_id: 'tool-1', id: 'item-1', name: 'timer_set', arguments: '' } });
event(request, { type: 'response.function_call_arguments.delta', output_index: 2, delta: '{"seconds":' });
event(request, { type: 'response.function_call_arguments.delta', output_index: 2, delta: '60}' });
}
}
return { load, session, send, event, text, finish, tool, requests, calls, deltas, errors, done };
}
for (const provider of ['openai', 'anthropic']) {
test(`iOS ${provider} streams, calls shared tools, continues with results, and preserves follow-up history`, async () => {
const h = assistant(provider); h.send('Set a one-minute timer');
const first = h.requests[0]; assert.equal(first.body.stream, true);
if (provider === 'openai') { assert.equal(first.headers.Authorization, 'Bearer fixture-key'); assert.equal(first.body.store, false); }
else assert.equal(first.headers['x-api-key'], 'fixture-key');
h.text(first, 'Setting π 👓'); h.tool(first); h.finish(first, true); await flush();
assert.equal(first.cancelled, true); assert.equal(h.calls[0].name, 'timer.set'); assert.equal(h.calls[0].args.seconds, 60);
assert.equal(h.requests.length, 2);
const second = h.requests[1];
assert.match(JSON.stringify(second.body), /Timer set/);
if (provider === 'openai') assert.match(JSON.stringify(second.body), /opaque-fixture/);
h.text(second, 'Your timer is running.'); h.finish(second);
assert.equal(h.session.isTurnActive(), false); assert.equal(h.done.length, 1); assert.equal(h.errors.length, 0);
h.send('What did you set?');
assert.match(JSON.stringify(h.requests[2].body), /Your timer is running/);
h.session.cancel(); assert.equal(h.requests[2].cancelled, true);
});
test(`iOS ${provider} cancellation, HTTP errors and premature EOF never execute incomplete tools`, async () => {
const h = assistant(provider); h.send('test'); const first = h.requests[0]; h.session.cancel();
h.text(first, 'late'); h.tool(first); h.finish(first, true); await flush();
assert.equal(h.calls.length, 0); assert.equal(h.deltas.length, 0);
h.send('retry'); h.requests[1].listener.onHttpError(401, '{"error":{"message":"Invalid API key"}}');
assert.equal(h.session.isTurnActive(), false); assert.match(h.errors[0], /key/i);
h.send('retry'); const third = h.requests[2]; h.text(third, 'Partial'); h.tool(third); third.listener.onComplete(); await flush();
assert.match(h.errors.at(-1), /before.*completed/); assert.equal(h.calls.length, 0); assert.equal(third.cancelled, true);
});
}
test('iOS SSE adapter invokes listeners on its polling thread and stops polling at cancellation/EOF', () => {
const timers = new Set(), instances = [], seen = [];
const api = loader({
setInterval: fn => { timers.add(fn); return fn; }, clearInterval: fn => timers.delete(fn),
FaceclawSseRequest: { alloc: () => ({ initWithURLBodyHeaders(url, body, headers) {
const instance = { events: [], cancelled: false, takeEvents() { const events = this.events; this.events = []; return JSON.stringify(events); }, cancel() { this.cancelled = true; } };
instances.push(instance); assert.equal(JSON.parse(headers)['User-Agent'], 'fixture'); return instance;
} }) },
}, { '../util/http': { withUserAgent: headers => ({ ...headers, 'User-Agent': 'fixture' }) } })('app/native/sse.ios.ts');
const listener = { onLine: line => seen.push(line), onComplete: () => seen.push('complete'), onHttpError() {}, onFailure: error => assert.fail(error) };
const handle = api.openSseRequest('https://fixture', '{}', {}, listener);
instances[0].events.push({ kind: 'line', text: 'first' }); assert.deepEqual(seen, []);
for (const tick of timers) tick(); assert.deepEqual(seen, ['first']);
handle.cancel(); assert.equal(timers.size, 0); assert.equal(instances[0].cancelled, true);
api.openSseRequest('https://fixture', '{}', {}, listener);
instances[1].events.push({ kind: 'line', text: 'last' }, { kind: 'complete' });
for (const tick of [...timers]) tick(); assert.deepEqual(seen, ['first', 'last', 'complete']); assert.equal(timers.size, 0);
});
test('iOS only offers cloud assistant models and preserves old local-model conversations as Auto', () => {
const api = loader({ global: { isIOS: true } }, { '../native/llama': { LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false } })('app/assistant/models.ts');
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), false);
assert.equal(api.supportedAssistantModel('qwen'), 'auto');
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai');
assert.equal(api.resolveAssistantModel('auto', { openai: '', anthropic: 'key' }).provider, 'anthropic');
assert.equal(api.resolveAssistantModel('auto', { openai: '', anthropic: '' }), null);
const { AssistantConversations } = loader({}, { './models': api, './session': {} })('app/assistant/conversations.ts');
const history = { messages: [], transcript: [{ role: 'user', text: 'Keep this conversation' }] };
const saved = JSON.stringify({ selectedId: 'old', conversations: [{ id: 'old', model: 'qwen', reasoning: 'default', history }] });
const conversations = new AssistantConversations(() => null, () => 'auto', () => {}, saved);
assert.equal(conversations.current().model, 'auto'); assert.equal(conversations.title(), 'Keep this conversation');
});
test('cancelling during a tool prevents subsequent tool side effects and a stalled turn times out', async () => {
let stream, completeTool, deadline;
const calls = [], failures = [];
const { DirectAssistantBackend } = loader({ setTimeout: fn => { deadline = fn; return 1; }, clearTimeout() {} }, {
'../native/openai': { streamOpenAiResponse: options => { stream = options; return { cancel() {} }; } },
'../native/anthropic': {}, '../native/llama': {},
})('app/assistant/direct-backend.ts');
const backend = new DirectAssistantBackend();
const options = { provider: 'openai', messages: [], buildTools: () => [], resolveToolName: name => name,
registry: { callTool: name => { calls.push(name); return new Promise(resolve => { completeTool = resolve; }); } },
callbacks: { onTextDelta() {}, onToolActivity() {}, onTurnDone() {}, onError: message => failures.push(message) } };
const handle = backend.runTurn(options);
stream.onDone({ text: '', stopReason: 'tool_use', content: [1, 2].map(id => ({ type: 'tool_use', id: String(id), name: 'tool' + id, input: {} })) });
handle.cancel(); completeTool({ ok: true, content: 'done' }); await flush();
assert.deepEqual(calls, ['tool1']);
backend.runTurn({ ...options, messages: [] }); deadline(); assert.match(failures[0], /too long/);
});
+4
View File
@@ -68,6 +68,10 @@ function fixture() {
paint: () => { boardStats.paints++; return new images.GrayImage(100, 80, 200); } }),
};
const modules = {
'../assistant/system-tools': { registerSystemTools() {} },
'../assistant/window-tools': { registerWindowTools() {} },
'../assistant/navigate-tools': { registerNavigateTools() {} },
'../assistant/roam-tools': { registerRoamTools() {} },
"../native/ios-navigation-sensors": {},
"../native/compass.ios": { bindCompassSession() {}, receiveCompassEvent() {} },
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
+45
View File
@@ -0,0 +1,45 @@
const { execFileSync, spawn } = require('node:child_process');
const { mkdtempSync, rmSync } = require('node:fs');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const http = require('node:http');
const assert = require('node:assert/strict');
const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-sse-'));
execFileSync('xcrun', ['clang', '-fobjc-arc', '-framework', 'Foundation', '-I', 'App_Resources/iOS/src',
'tests/native/ios-sse-test.m', 'App_Resources/iOS/src/FaceclawSseRequest.m', '-o', join(dir, 'sse-test')], { cwd: root, stdio: 'inherit' });
const server = http.createServer(async (request, response) => {
assert.notEqual(request.url, '/redirected', 'Credentials must not follow redirects');
assert.equal(request.headers.authorization, 'Bearer fixture-key');
assert.equal(request.headers['user-agent'], 'Faceclaw-test');
assert.equal(request.method, 'POST');
const chunks = []; for await (const chunk of request) chunks.push(chunk);
assert.deepEqual(JSON.parse(Buffer.concat(chunks)), { test: true });
if (request.url === '/error') { response.writeHead(401); response.end('{"error":{"type":"invalid_api_key"}}'); return; }
if (request.url === '/redirect') { response.writeHead(302, { Location: '/redirected' }); response.end(); return; }
response.writeHead(200, { 'Content-Type': 'text/event-stream', ...(request.url === '/disconnect' ? { 'Content-Length': '1000' } : {}) });
if (request.url === '/overflow') { response.end('x'.repeat(2 * 1024 * 1024 + 1)); return; }
if (request.url === '/disconnect') {
response.write('data: partial'); setTimeout(() => response.destroy(), 50); return;
}
if (request.url === '/cancel') {
response.write('data: first\n'); const timer = setTimeout(() => response.end('data: late\n'), 200);
response.on('close', () => clearTimeout(timer)); return;
}
// Split UTF-8 characters and CRLF boundaries across individual network writes.
const bytes = Buffer.from('\uFEFFdata: π 👓\r\n\r\n: ping\rdata: last');
let index = 0;
const timer = setInterval(() => {
if (index === bytes.length) { clearInterval(timer); response.end(); }
else response.write(bytes.subarray(index, ++index));
}, 2);
response.on('close', () => clearInterval(timer));
});
server.listen(0, '127.0.0.1', async () => {
try {
for (const mode of ['stream', 'error', 'redirect', 'cancel', 'overflow', 'disconnect']) await new Promise((resolve, reject) => {
const child = spawn(join(dir, 'sse-test'), [`http://127.0.0.1:${server.address().port}/${mode}`, mode], { stdio: 'inherit' });
child.on('error', reject); child.on('exit', code => code === 0 ? resolve() : reject(new Error(`${mode}: exit ${code}`)));
});
} catch (error) { console.error(error); process.exitCode = 1; }
finally { server.closeAllConnections(); server.close(); rmSync(dir, { recursive: true, force: true }); }
});
+38
View File
@@ -0,0 +1,38 @@
#import <Foundation/Foundation.h>
#import "FaceclawSseRequest.h"
int main(int argc, const char **argv) {
@autoreleasepool {
NSString *mode = [NSString stringWithUTF8String:argv[2]];
FaceclawSseRequest *request = [[FaceclawSseRequest alloc] initWithURL:[NSString stringWithUTF8String:argv[1]]
body:@"{\"test\":true}" headers:@"{\"Authorization\":\"Bearer fixture-key\",\"User-Agent\":\"Faceclaw-test\"}"];
NSMutableArray *lines = [NSMutableArray new];
NSDictionary *terminal = nil;
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:10];
while (!terminal && deadline.timeIntervalSinceNow > 0) {
NSArray *events = [NSJSONSerialization JSONObjectWithData:[[request takeEvents] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil];
for (NSDictionary *event in events) {
if ([event[@"kind"] isEqual:@"line"]) [lines addObject:event[@"text"]];
else terminal = event;
}
if ([mode isEqual:@"cancel"] && lines.count) {
[request cancel];
[[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.3]];
if (![[request takeEvents] isEqual:@"[]"]) return 1;
puts("PASS native SSE cancellation suppresses later callbacks"); return 0;
}
[[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.01]];
}
[request cancel];
BOOL ok = NO;
if ([mode isEqual:@"stream"]) ok = [terminal[@"kind"] isEqual:@"complete"] &&
[lines isEqual:@[@"data: π 👓", @"", @": ping", @"data: last"]];
if ([mode isEqual:@"error"]) ok = [terminal[@"kind"] isEqual:@"http-error"] &&
[terminal[@"code"] intValue] == 401 && [terminal[@"text"] containsString:@"invalid_api_key"];
if ([mode isEqual:@"redirect"]) ok = [terminal[@"kind"] isEqual:@"http-error"] && [terminal[@"code"] intValue] == 302;
if ([mode isEqual:@"overflow"] || [mode isEqual:@"disconnect"]) ok = [terminal[@"kind"] isEqual:@"failure"];
if (!ok) { NSLog(@"FAIL %@ lines=%@ terminal=%@", mode, lines, terminal); return 1; }
printf("PASS native SSE %s\n", mode.UTF8String);
}
return 0;
}