Port assistant integration (with cloud AIs) to iOS

This commit is contained in:
James Babcock committed 2026-09-19 00:02:15 -07:00
1 parent 9e18745e53
commit 617cdc39eb
25 files changed
+508 -63

No files matched your search

+3 -3
View File
@@ -1,7 +1,7 @@
import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { truncateText, wrapText } from "../../graphics/textwrap";
import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
import { ASSISTANT_MODEL_CHOICES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
import type { AssistantTranscriptEntry } from "../../assistant/session";
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
@@ -102,7 +102,7 @@ class AiChatLayer implements Layer {
}
case "long-press":
if (!this.conversations.current().session?.isTurnActive()) {
if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; }
if (!shell.isAssistantAvailable()) { shell.showAlert(global.isIOS ? "Set an OpenAI or Anthropic API key in Settings." : "Set an API key or download the on-phone model in Settings."); return; }
void this.draft.start();
}
return;
@@ -126,7 +126,7 @@ class AiChatLayer implements Layer {
onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); },
}))) },
{ label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model",
ASSISTANT_MODEL_VALUES.map((model) => ({
ASSISTANT_MODEL_CHOICES.map((model) => ({
label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`,
disabled: () => !this.conversations.available(model, record.reasoning),
onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); },
+2 -2
View File
@@ -1,5 +1,5 @@
import { AssistantSession, type AssistantBackendConfig, type AssistantSessionHistory } from "./session";
import { ASSISTANT_MODEL_VALUES, type AssistantModel } from "./models";
import { ASSISTANT_MODEL_VALUES, supportedAssistantModel, type AssistantModel } from "./models";
export type ReasoningLevel = "default" | "low" | "medium" | "high";
export type Conversation = {
@@ -31,7 +31,7 @@ export class AssistantConversations {
typeof record.id === "string" && ASSISTANT_MODEL_VALUES.includes(record.model) &&
["default", "low", "medium", "high"].includes(record.reasoning) &&
Array.isArray(record.history?.messages) && Array.isArray(record.history?.transcript),
).map((record) => ({ ...record, session: null }));
).map((record) => ({ ...record, model: supportedAssistantModel(record.model), session: null }));
this.selectedId = data.selectedId;
}
} catch { /* First launch or invalid saved state. */ }
+12 -1
View File
@@ -51,10 +51,13 @@ export class DirectAssistantBackend {
let cancelled = false;
let streamHandle: LlmStreamHandle | null = null;
let priorText = "";
let deadline: ReturnType<typeof setTimeout> | null = null;
const clearDeadline = () => { if (deadline !== null) clearTimeout(deadline); deadline = null; };
const finishError = (message: string) => {
if (cancelled) return;
cancelled = true;
clearDeadline(); streamHandle?.cancel(); streamHandle = null;
options.callbacks.onError(message);
};
@@ -102,10 +105,11 @@ export class DirectAssistantBackend {
block.type === "tool_use",
);
if (result.stopReason !== "tool_use" || toolUses.length === 0) {
clearDeadline();
options.callbacks.onTurnDone({ stopReason: result.stopReason });
return;
}
void this.runToolCalls(toolUses, options)
void this.runToolCalls(toolUses, options, () => cancelled)
.then((toolResults) => {
// Always record the results, even if cancelled mid-flight: an
// assistant tool_use with no matching tool_result would make the
@@ -131,11 +135,13 @@ export class DirectAssistantBackend {
: streamAnthropicMessage(streamOptions);
};
deadline = setTimeout(() => finishError("Assistant stopped: turn took too long"), MAX_TURN_MS);
runIteration(0);
return {
cancel: () => {
cancelled = true;
clearDeadline();
streamHandle?.cancel();
streamHandle = null;
},
@@ -145,9 +151,14 @@ export class DirectAssistantBackend {
private async runToolCalls(
toolUses: Array<Extract<LlmContentBlock, { type: "tool_use" }>>,
options: DirectTurnOptions,
isCancelled: () => boolean,
): Promise<LlmContentBlock[]> {
const results: LlmContentBlock[] = [];
for (const toolUse of toolUses) {
if (isCancelled()) {
results.push({ type: "tool_result", tool_use_id: toolUse.id, content: "Cancelled", is_error: true });
continue;
}
const canonicalName = options.resolveToolName(toolUse.name);
options.callbacks.onToolActivity(canonicalName);
const result = await options.registry.callTool(canonicalName, toolUse.input);
+7
View File
@@ -13,6 +13,13 @@ export const ASSISTANT_MODEL_VALUES = [
] as const;
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
/** Keep saved model IDs portable, while offering only supported choices. */
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
return global.isIOS && value === "qwen" ? "auto" : value;
}
export type AssistantProvider = "anthropic" | "openai" | "local";
export type AssistantApiKeys = {
+1 -1
View File
@@ -4,7 +4,7 @@
* the app closed — so these always-available wrappers launch the app first,
* wait for its tools to register, and forward the call.
*/
import { ensureFineLocationPermission } from "../g2/android-permissions";
import { ensureFineLocationPermission } from "../native/location-permissions";
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
import { callAppToolWithLaunch } from "./launch-on-call";
+4 -1
View File
@@ -20,7 +20,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
registered = true;
// Keep a media listener warm so "what's playing" works without opening Music.
void mediaControllerBridge.start();
if (!global.isIOS) void mediaControllerBridge.start();
registry.registerSystemTool(
{
@@ -54,6 +54,9 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
},
);
// Calendar, phone media and notifications have no iOS backend yet.
if (global.isIOS) return;
registry.registerSystemTool(
{
name: "calendar.list_events",
+9
View File
@@ -1,3 +1,7 @@
import { registerSystemTools } from '../assistant/system-tools'
import { registerWindowTools } from '../assistant/window-tools'
import { registerNavigateTools } from '../assistant/navigate-tools'
import { registerRoamTools } from '../assistant/roam-tools'
import { IosNavigationSensors } from '../native/ios-navigation-sensors'
import { Utils } from '@nativescript/core'
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
@@ -124,6 +128,11 @@ export class IosPreviewController {
shell.registerWindow(launcher)
shell.wake('window')
shell.focusWindow(launcher.windowId)
registerSystemTools()
registerWindowTools({ apps: ALL_APPS.filter(app => !iosAppUnavailableReason(app.appId)),
launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() })
registerNavigateTools(id => this.launchApp(id))
registerRoamTools(id => this.launchApp(id))
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
}
+12 -21
View File
@@ -8,13 +8,13 @@ import type {
LlmToolDefinition,
} from "../assistant/llm-protocol";
declare const com: any;
import { openSseRequest, type SseListener } from "./sse";
/**
* Minimal streaming client for the Anthropic Messages API. There is no
* official SDK for the NativeScript runtime, so this speaks raw HTTP/SSE via
* FaceclawSseRequest (okhttp on the Java side), which delivers the response
* body line by line on this isolate's thread.
* the platform transport (OkHttp on Android, URLSession on iOS), which
* delivers response lines on this isolate's thread.
*/
const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages";
@@ -55,22 +55,19 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
const fail = (message: string) => {
if (settled) return;
settled = true;
request?.cancel();
options.onError(message);
};
const done = () => {
if (settled) return;
settled = true;
request?.cancel();
options.onDone({ text, content: finalizeBlocks(blocks), stopReason });
};
if (!global.isAndroid) {
// Deferred so the caller gets its handle back before the callback runs.
setTimeout(() => fail("Anthropic API is only wired up on Android"), 0);
return { cancel: () => {} };
}
if (!apiKey) {
setTimeout(() => fail("No Anthropic API key set"), 0);
return { cancel: () => {} };
return { cancel: () => { settled = true; } };
}
const body: any = {
@@ -83,7 +80,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
if (options.tools?.length) body.tools = options.tools;
if (options.effort) body.output_config = { effort: options.effort };
const listener = new com.faceclaw.app.FaceclawSseListener({
const listener: SseListener = {
onLine: (line: string) => {
if (settled) return;
const event = parseSseDataLine(String(line));
@@ -144,25 +141,19 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
fail(describeHttpError(Number(code), String(errorBody)));
},
onComplete: () => {
// Normally message_stop settles first; a clean EOF without it still
// resolves with whatever streamed.
done();
// A transport EOF alone must not execute an unfinished tool call.
fail("Anthropic stream ended before the response completed");
},
onFailure: (message: string) => {
fail(`Anthropic connection failed: ${String(message)}`);
},
});
};
const headers = Array.create("java.lang.String", 4) as string[];
headers[0] = "x-api-key";
headers[1] = apiKey;
headers[2] = "anthropic-version";
headers[3] = ANTHROPIC_VERSION;
try {
request = new com.faceclaw.app.FaceclawSseRequest(ANTHROPIC_URL, JSON.stringify(body), headers, listener);
request = openSseRequest(ANTHROPIC_URL, JSON.stringify(body), { "x-api-key": apiKey, "anthropic-version": ANTHROPIC_VERSION }, listener);
} catch (error) {
setTimeout(() => fail(`Anthropic request failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
return { cancel: () => { settled = true; } };
}
return {
+11 -20
View File
@@ -5,11 +5,11 @@ import type {
LlmStreamOptions,
} from "../assistant/llm-protocol";
declare const com: any;
import { openSseRequest, type SseListener } from "./sse";
/**
* Minimal streaming client for the OpenAI Responses API. NativeScript cannot
* use the Node SDK, so this shares the okhttp/SSE bridge used by Anthropic.
* use the Node SDK, so this shares the native streaming transport used by Anthropic.
*/
const OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses";
@@ -31,11 +31,13 @@ export function streamOpenAiResponse(options: LlmStreamOptions): LlmStreamHandle
const fail = (message: string) => {
if (settled) return;
settled = true;
request?.cancel();
options.onError(message);
};
const done = () => {
if (settled) return;
settled = true;
request?.cancel();
const content = finalizeOutputs(outputs);
const hasToolUse = content.some((block) => block.type === "tool_use");
options.onDone({
@@ -45,13 +47,9 @@ export function streamOpenAiResponse(options: LlmStreamOptions): LlmStreamHandle
});
};
if (!global.isAndroid) {
setTimeout(() => fail("OpenAI API is only wired up on Android"), 0);
return { cancel: () => {} };
}
if (!apiKey) {
setTimeout(() => fail("No OpenAI API key set"), 0);
return { cancel: () => {} };
return { cancel: () => { settled = true; } };
}
const body: any = {
@@ -73,7 +71,7 @@ export function streamOpenAiResponse(options: LlmStreamOptions): LlmStreamHandle
}
if (options.effort) body.reasoning = { effort: options.effort };
const listener = new com.faceclaw.app.FaceclawSseListener({
const listener: SseListener = {
onLine: (line: string) => {
if (settled) return;
const event = parseSseDataLine(String(line));
@@ -164,26 +162,19 @@ export function streamOpenAiResponse(options: LlmStreamOptions): LlmStreamHandle
fail(describeHttpError(Number(code), String(errorBody)));
},
onComplete: () => {
done();
// A transport EOF alone must not execute an unfinished tool call.
fail("OpenAI stream ended before the response completed");
},
onFailure: (message: string) => {
fail(`OpenAI connection failed: ${String(message)}`);
},
});
};
const headers = Array.create("java.lang.String", 2) as string[];
headers[0] = "Authorization";
headers[1] = `Bearer ${apiKey}`;
try {
request = new com.faceclaw.app.FaceclawSseRequest(
OPENAI_RESPONSES_URL,
JSON.stringify(body),
headers,
listener,
);
request = openSseRequest(OPENAI_RESPONSES_URL, JSON.stringify(body), { Authorization: `Bearer ${apiKey}` }, listener);
} catch (error) {
setTimeout(() => fail(`OpenAI request failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
return { cancel: () => { settled = true; } };
}
return {
+6
View File
@@ -0,0 +1,6 @@
export type SseListener = {
onLine(line: string): void;
onHttpError(code: number, body: string): void;
onComplete(): void;
onFailure(message: string): void;
};
+28
View File
@@ -0,0 +1,28 @@
import { type SseListener } from './sse-types';
import { withUserAgent } from '../util/http';
export { type SseListener } from './sse-types';
declare const FaceclawSseRequest: any;
/** Drain native events on the requesting JS isolate, never on a URLSession thread. */
export function openSseRequest(url: string, body: string, headers: Record<string, string>, listener: SseListener): { cancel(): void } {
const native = FaceclawSseRequest.alloc().initWithURLBodyHeaders(url, body, JSON.stringify(withUserAgent(headers)));
let ended = false;
const cancel = () => { if (ended) return; ended = true; clearInterval(timer); native.cancel(); };
const timer = setInterval(() => {
try {
for (const event of JSON.parse(native.takeEvents())) {
if (ended) break;
if (event.kind === 'line') listener.onLine(event.text);
else {
cancel();
if (event.kind === 'complete') listener.onComplete();
else if (event.kind === 'http-error') listener.onHttpError(event.code, event.text);
else listener.onFailure(event.text);
}
}
} catch (error) {
cancel(); listener.onFailure(String((error as Error)?.message ?? error));
}
}, 25);
return { cancel };
}
+11
View File
@@ -0,0 +1,11 @@
import { type SseListener } from './sse-types';
export { type SseListener } from './sse-types';
declare const com: any;
/** Android delivers callbacks on the requesting isolate's Looper. */
export function openSseRequest(url: string, body: string, headers: Record<string, string>, listener: SseListener): { cancel(): void } {
const entries = Object.entries(headers);
const nativeHeaders = Array.create('java.lang.String', entries.length * 2);
entries.forEach(([name, value], index) => { nativeHeaders[index * 2] = name; nativeHeaders[index * 2 + 1] = value; });
return new com.faceclaw.app.FaceclawSseRequest(url, body, nativeHeaders, new com.faceclaw.app.FaceclawSseListener(listener));
}
+6 -6
View File
@@ -10,7 +10,7 @@ import {
import { getDefaultSmallFont } from "~/graphics/ui-fonts";
import { wrapText } from "~/graphics/textwrap";
import {
ASSISTANT_MODEL_VALUES,
ASSISTANT_MODEL_CHOICES,
assistantModelLabel,
assistantModelProvider,
type AssistantModel,
@@ -595,7 +595,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
export type AssistantBackendKind = "direct" | "external";
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
direct: "On-phone",
direct: global.isIOS ? "Cloud API" : "On-phone",
external: "My own agent (bridge)",
};
@@ -604,10 +604,10 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
label: "Assistant backend",
storageKey: "assistant.backend",
defaultValue: "direct",
values: ["direct", "external"],
values: global.isIOS ? ["direct"] : ["direct", "external"],
formatValue: (value) => assistantBackendLabels[value] ?? value,
description:
"Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
});
export const assistantBridgeHostSetting = new ConfigSettingString({
@@ -700,7 +700,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
label: "Assistant model",
storageKey: "assistant.model",
defaultValue: "auto",
values: ASSISTANT_MODEL_VALUES,
values: ASSISTANT_MODEL_CHOICES,
formatValue: assistantModelLabel,
isDisabled: (value) => {
const provider = assistantModelProvider(value);
@@ -710,7 +710,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
return false;
},
description:
"Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
});
export const mapboxApiKeySetting = new ConfigSettingString({
+3 -1
View File
@@ -130,14 +130,16 @@ function settingsSections(): SettingsSection[] {
// On-phone LLM loop vs the user's own agent via the bridge plugin.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
localModelMenuItem(),
...(!global.isIOS ? [localModelMenuItem()] : []),
// When on, a wakeword utterance goes straight to the assistant with no
// Send/Type menu step.
toggleSettingMenuItem(assistantSkipConfirmationSetting),
...(!global.isIOS ? [
textSettingMenuItem(assistantBridgeHostSetting),
textSettingMenuItem(assistantBridgePortSetting),
textSettingMenuItem(assistantBridgeTokenSetting),
toggleSettingMenuItem(assistantAllowProactiveSetting),
] : []),
],
},
{
+2 -2
View File
@@ -1067,7 +1067,7 @@ class Shell {
*/
private buildVoiceSendTargets(): VoiceSendTarget[] {
const targets: VoiceSendTarget[] = [];
if (!global.isIOS && this.isAssistantAvailable()) {
if (this.isAssistantAvailable()) {
targets.push({
id: "assistant",
label: "Send to Assistant",
@@ -1254,7 +1254,7 @@ class Shell {
this.showAlert(
assistantBackendSetting.get() === "external"
? "Configure the agent bridge host and token in Settings."
: "Set an API key or download the on-phone model in Settings.",
: global.isIOS ? "Set an OpenAI or Anthropic API key in Settings." : "Set an API key or download the on-phone model in Settings.",
);
return;
}
+2 -2
View File
@@ -224,7 +224,7 @@ export class VoiceInputLayer implements Layer {
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0 && !this.stoppingCapture;
const hasLlmKey = !global.isIOS && anthropicApiKeySetting.get().trim().length > 0;
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
rows.push({
@@ -238,7 +238,7 @@ export class VoiceInputLayer implements Layer {
});
}
rows.push({
label: global.isIOS ? "Continue (Not available on iOS)" : hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey || this.stoppingCapture,
onSelect: () => {
if (hasLlmKey && !this.stoppingCapture) this.startContinuation();