Support Apple Intelligence as an on-device AI model for the voice assistant/etc

This commit is contained in:
James Babcock committed 2026-10-31 17:48:45 -07:00
1 parent 3c3aa31542
commit 4973fd2aea
22 files changed
+1551 -40

No files matched your search

+2 -2
View File
@@ -7,7 +7,7 @@
* surfacing a transport error.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalQwen } from "../../native/llama";
import { streamLocalModel } from "../../native/local-llm";
import { streamOpenAiResponse } from "../../native/openai";
import { resolveAssistantModel } from "../../assistant/models";
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
@@ -41,7 +41,7 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
streamOpenAiResponse(options);
break;
case "local":
streamLocalQwen(options);
streamLocalModel(options);
break;
default:
streamAnthropicMessage(options);
+2 -2
View File
@@ -1,6 +1,6 @@
import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
import { parseConversationQuery } from "../../phone-ui/conversation-search";
@@ -245,7 +245,7 @@ export function askConversations(
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
return { cancel: () => {} };
}
return streamLocalQwen({
return streamLocalModel({
apiKey: "",
model: "local",
system:
+2 -2
View File
@@ -1,4 +1,4 @@
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
import type { NameInference } from "./name-guess";
/**
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
};
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
try {
streamLocalQwen({
streamLocalModel({
apiKey: "",
model: "local",
system:
+2 -2
View File
@@ -1,6 +1,6 @@
import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
import { speakerRegistry } from "./speakers";
declare const com: any;
@@ -124,7 +124,7 @@ function generateForSpeaker(
};
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
try {
streamLocalQwen({
streamLocalModel({
apiKey: "",
model: "local",
system:
+2 -2
View File
@@ -4,7 +4,7 @@
* TranslationModel for how stretches are batched and given context.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalQwen } from "../../native/llama";
import { streamLocalModel } from "../../native/local-llm";
import { streamOpenAiResponse } from "../../native/openai";
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
@@ -55,7 +55,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
case "openai":
return streamOpenAiResponse(options);
case "local":
return streamLocalQwen(options);
return streamLocalModel(options);
default:
return streamAnthropicMessage(options);
}
+2 -2
View File
@@ -1,6 +1,6 @@
import { streamAnthropicMessage } from "../native/anthropic";
import { streamHermesChat } from "../native/hermes";
import { streamLocalQwen } from "../native/llama";
import { streamLocalModel } from "../native/local-llm";
import { streamOpenAiResponse } from "../native/openai";
import type {
LlmContentBlock,
@@ -146,7 +146,7 @@ export class DirectAssistantBackend {
: options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: options.provider === "local"
? streamLocalQwen(streamOptions)
? streamLocalModel(streamOptions)
: streamAnthropicMessage(streamOptions);
};
+13 -7
View File
@@ -1,4 +1,4 @@
import { isLocalModelReady, LOCAL_MODEL } from "../native/llama";
import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm";
export const ASSISTANT_MODEL_VALUES = [
"auto",
@@ -13,12 +13,17 @@ export const ASSISTANT_MODEL_VALUES = [
"hermes",
] as const;
/**
* "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on
* Android, the Apple Intelligence system model on iOS (the ID predates iOS
* support and stays for saved settings and conversations).
*/
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
/** Keep saved model IDs portable, while offering only supported choices. */
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen");
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
return global.isIOS && value === "qwen" ? "auto" : value;
return value === "qwen" && !isLocalModelSupported() ? "auto" : value;
}
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
@@ -58,8 +63,9 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
// not an API key. The default for users who haven't set any key.
// Runs on the phone itself (Android: llama.cpp with the downloaded model
// file; iOS: Apple Intelligence), not an API key. The default for users who
// haven't set any key.
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
// Your own Hermes Agent (its API server), with its own tools and memory.
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
@@ -76,8 +82,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider
/**
* Resolve Auto and attach the credential needed by the chosen provider.
* Auto intentionally prefers Terra over Sonnet when both keys are available,
* and falls back to the on-phone model when no key is set but the model file
* has been downloaded.
* and falls back to the on-phone model when no key is set but that model is
* ready (downloaded on Android; Apple Intelligence on and ready on iOS).
*/
export function resolveAssistantModel(
selection: AssistantModel,
+4 -6
View File
@@ -6,12 +6,14 @@ import type {
LlmStreamOptions,
} from "../assistant/llm-protocol";
import { buildQwenPrompt } from "../assistant/qwen-prompt";
import type { LocalModelState } from "./local-llm-types";
declare const com: any;
declare const java: any;
/**
* On-phone LLM backend: model download management plus a streaming client
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
* local-llm.ios.ts instead): model download management plus a streaming client
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
* the direct assistant loop can treat "local" as just another provider.
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
@@ -38,11 +40,7 @@ const TOP_K = 20;
/** Free the ~3GB of model + KV this long after the last generation. */
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
export type LocalModelState = {
status: "absent" | "downloading" | "ready";
bytesDownloaded: number;
totalBytes: number;
};
export type { LocalModelState } from "./local-llm-types";
let runner: any = null;
let downloader: any = null;
+13
View File
@@ -0,0 +1,13 @@
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
export type LocalModelState = {
/**
* Android downloads its model: absent → downloading → ready. iOS uses the
* system's Apple Intelligence model: ready, or unavailable (see detail).
*/
status: "absent" | "downloading" | "ready" | "unavailable";
bytesDownloaded: number;
totalBytes: number;
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
detail?: string;
};
+383
View File
@@ -0,0 +1,383 @@
import type {
LlmContentBlock,
LlmMessage,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
import type { LocalModelState } from "./local-llm-types";
export type { LocalModelState } from "./local-llm-types";
declare const FaceclawSystemLanguageModel: any;
/**
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
* Calculator and the Microphones helpers treat it as the "local" provider.
*
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
* background (where Faceclaw usually is, driving the glasses with the phone
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
* ~3 GB in-process model makes the app the first thing jetsam kills. The
* system model runs out of process on the Neural Engine, needs no download,
* and decodes tool arguments under a schema constraint (like our GBNF grammar
* on Android). Its costs: a 4096-token context (the Swift side fits history
* and tool definitions into it), Apple's guardrails, a background request
* budget, and Apple Intelligence-capable hardware.
*
* Tool calls: the framework runs tools itself, mid-response. The Swift side
* parks each call and reports it; here that ends the loop iteration with
* stopReason "tool_use", the direct backend executes the tools as for any
* provider, and its next request (ending in the matching tool_result blocks)
* resumes the parked generation instead of starting a new one.
*/
export const LOCAL_MODEL = {
id: "apple-foundation-model",
label: "Apple Intelligence",
sizeBytes: 0,
};
const DEFAULT_MAX_TOKENS = 512;
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
/**
* Only used when tool results arrive with no parked generation to resume (it
* timed out, or the app restarted mid-turn): the framework has no way to
* continue after a tool output without a new prompt.
*/
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
type Availability =
| "available"
| "unsupportedOS"
| "deviceNotEligible"
| "appleIntelligenceNotEnabled"
| "modelNotReady"
| "unavailable";
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
unsupportedOS: "needs iOS 26",
deviceNotEligible: "not on this iPhone",
appleIntelligenceNotEnabled: "Apple Intelligence off",
modelNotReady: "downloading",
unavailable: "unavailable",
};
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
appleIntelligenceNotEnabled:
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
};
export function localModelAvailability(): Availability {
try {
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
const value = String(FaceclawSystemLanguageModel.availability());
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
} catch {
return "unsupportedOS";
}
}
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
export function isLocalModelSupported(): boolean {
const availability = localModelAvailability();
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
}
export function isLocalModelReady(): boolean {
return localModelAvailability() === "available";
}
export function localModelState(): LocalModelState {
const availability = localModelAvailability();
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
}
/** The user-facing reason the model can't run, or "" when it can. */
export function localModelUnavailableMessage(): string {
const availability = localModelAvailability();
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
}
// iOS manages the system model itself: there is nothing to download, cancel,
// delete or unload, and no progress to watch.
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
return () => {};
}
export function startLocalModelDownload(): void {}
export function cancelLocalModelDownload(): void {}
export function deleteLocalModel(): void {}
export function unloadLocalModel(): void {}
type HistoryEntry =
| { type: "prompt" | "response"; text: string }
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
| { type: "toolOutput"; id: string; name: string; text: string };
export type SystemModelRequest = {
system: string;
tools: Array<{ name: string; description: string; parameters: object }>;
history: HistoryEntry[];
prompt: string;
maxTokens: number;
permissive: boolean;
};
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
/**
* Translate provider-neutral history into the Swift request: everything but
* the newest user prompt becomes Transcript entries, and that prompt is what
* the model answers.
*/
export function buildSystemModelRequest(
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
): SystemModelRequest {
const messages = options.messages;
const last = messages[messages.length - 1];
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
const history: HistoryEntry[] = [];
const toolNames = new Map<string, string>();
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
appendHistory(history, message, toolNames);
}
const tools = (options.tools ?? []).map((tool) => ({
name: tool.name,
description: tool.description,
parameters: tool.input_schema,
}));
return {
system: options.system?.trim() || "You are a helpful assistant.",
tools,
history,
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
permissive: tools.length === 0,
};
}
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
if (message.role === "user") {
for (const result of toolResultsOf(message)) {
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
}
const text = textOf(message.content);
if (text) history.push({ type: "prompt", text });
return;
}
const text = textOf(message.content);
if (text) history.push({ type: "response", text });
if (typeof message.content === "string") return;
const calls: Array<{ id: string; name: string; arguments: string }> = [];
for (const block of message.content) {
if (block.type !== "tool_use") continue;
toolNames.set(block.id, block.name);
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
}
if (calls.length) history.push({ type: "toolCalls", calls });
}
function textOf(content: string | LlmContentBlock[]): string {
if (typeof content === "string") return content.trim();
return content
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
.map((block) => block.text.trim())
.filter(Boolean)
.join("\n");
}
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
if (!message || message.role !== "user" || typeof message.content === "string") return [];
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
}
function parseArguments(json: string): any {
try {
const value = JSON.parse(json);
return value && typeof value === "object" ? value : {};
} catch {
return {};
}
}
type NativeEvent = {
kind: string;
text?: string;
calls?: Array<{ id: string; name: string; arguments: string }>;
code?: string;
message?: string;
};
/** Generations parked in tool calls, by call id. */
const parked = new Map<string, Generation>();
/** Every unfinished generation, so its native object outlives the caller's handle. */
const live = new Set<Generation>();
/**
* One native generation, possibly spanning several loop iterations: each
* iteration ends at a batch of tool calls (or the end of the reply), and the
* next one resumes it with the results.
*/
class Generation {
private readonly native: any;
private options: LlmStreamOptions | null;
private iteration = 0;
private text = "";
private callIds: string[] = [];
private expiry: ReturnType<typeof setTimeout> | null = null;
private closed = false;
constructor(options: LlmStreamOptions) {
this.options = options;
this.native = FaceclawSystemLanguageModel.new();
this.native.eventHandler = (json: string) => this.onEvent(String(json));
live.add(this);
}
start(request: SystemModelRequest): LlmStreamHandle {
this.native.start(JSON.stringify(request));
return this.handle();
}
/** Whether these results answer this generation's parked calls. */
awaits(results: ToolResultBlock[]): boolean {
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
}
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
this.unpark();
this.options = options;
this.iteration++;
this.text = "";
this.native.resolveToolCalls(
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
);
return this.handle();
}
cancel(): void {
if (this.closed) return;
this.close();
try {
this.native.cancel();
} catch {
// ignore
}
}
private handle(): LlmStreamHandle {
const iteration = this.iteration;
// A stale handle (an earlier iteration's) still cancels a generation parked
// since then: its owner gave up on the turn.
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
}
private close(): void {
this.closed = true;
this.unpark();
this.options = null;
this.native.eventHandler = null;
live.delete(this);
}
private park(ids: string[]): void {
this.callIds = ids;
for (const id of ids) parked.set(id, this);
this.expiry = setTimeout(() => {
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
this.cancel();
}, PARKED_TIMEOUT_MS);
}
private unpark(): void {
for (const id of this.callIds) {
if (parked.get(id) === this) parked.delete(id);
}
this.callIds = [];
if (this.expiry !== null) clearTimeout(this.expiry);
this.expiry = null;
}
private onEvent(json: string): void {
if (this.closed) return;
let event: NativeEvent;
try {
event = JSON.parse(json);
} catch {
return;
}
if (event.kind === "log") {
console.log(`[local-llm] ${event.message ?? ""}`);
return;
}
const options = this.options;
if (!options) return;
switch (event.kind) {
case "delta": {
const delta = event.text ?? "";
this.text += delta;
options.onTextDelta?.(delta, this.text);
return;
}
case "toolCalls": {
const calls = event.calls ?? [];
const text = this.text.trim();
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
for (const call of calls) {
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
}
this.options = null;
this.park(calls.map((call) => call.id));
options.onDone({ text, content, stopReason: "tool_use" });
return;
}
case "done": {
this.close();
const text = this.text.trim();
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
return;
}
case "error": {
this.close();
if (event.code === "refusal") {
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
} else {
options.onError(event.message || "The on-device model failed");
}
return;
}
}
}
}
/**
* Stream a reply from the system model. A request whose history ends in tool
* results for a parked generation resumes it; anything else starts afresh.
*/
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
const availability = localModelAvailability();
if (availability !== "available") {
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
return { cancel: () => {} };
}
const results = toolResultsOf(options.messages[options.messages.length - 1]);
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
if (waiting?.awaits(results)) return waiting.resume(options, results);
let generation: Generation | null = null;
try {
generation = new Generation(options);
return generation.start(buildSystemModelRequest(options));
} catch (error) {
generation?.cancel();
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
}
}
+28
View File
@@ -0,0 +1,28 @@
/**
* The on-phone LLM: one entry point for every feature that can run on the
* phone without an API key (the direct assistant's "local" provider,
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
* contract, plus model status for Settings.
*
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
* to download, available on Apple Intelligence devices running iOS 26+.
*/
export {
LOCAL_MODEL,
isLocalModelReady,
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
cancelLocalModelDownload,
deleteLocalModel,
unloadLocalModel,
streamLocalQwen as streamLocalModel,
} from "./llama";
export type { LocalModelState } from "./local-llm-types";
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
export function isLocalModelSupported(): boolean {
return true;
}
+4 -4
View File
@@ -16,7 +16,7 @@ import {
assistantModelProvider,
type AssistantModel,
} from "~/assistant/models";
import { isLocalModelReady } from "../native/llama";
import { isLocalModelReady } from "../native/local-llm";
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
@@ -930,7 +930,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
export type AssistantBackendKind = "direct" | "external";
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
direct: global.isIOS ? "Cloud API" : "On-phone",
direct: "On-phone",
external: "My own agent (bridge)",
};
@@ -942,7 +942,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
values: global.isIOS ? ["direct"] : ["direct", "external"],
formatValue: (value) => assistantBackendLabels[value] ?? value,
description:
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys), or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
});
export const assistantHermesUrlSetting = new ConfigSettingString({
@@ -1116,7 +1116,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
return false;
},
description:
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
});
export const mapboxApiKeySetting = new ConfigSettingString({
+27 -4
View File
@@ -12,7 +12,7 @@ import {
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
} from "../../native/llama";
} from "../../native/local-llm";
import {
ASR_MODELS,
asrModelState,
@@ -143,14 +143,14 @@ function settingsSections(): SettingsSection[] {
{
label: "Assistant",
items: [
// iOS offers cloud APIs; Android also supports local models and the bridge.
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
textSettingMenuItem(assistantHermesUrlSetting),
textSettingMenuItem(assistantHermesKeySetting),
...(!global.isIOS ? [localModelMenuItem()] : []),
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
// iOS has no local LLM or agent bridge yet.
// iOS has no agent bridge yet.
...(!global.isIOS ? [
textSettingMenuItem(assistantBridgeHostSetting),
textSettingMenuItem(assistantBridgePortSetting),
@@ -399,6 +399,29 @@ function localModelStatusText(): string {
return "not downloaded";
}
/**
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
* installs and manages; the row only reports whether it can be used.
*/
function systemModelMenuItem(): MenuItem {
const statusText = (): string => {
const state = localModelState();
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
};
return {
label: "On-phone model",
description:
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
"Faster and more private than the cloud models, but simpler, with a short memory. " +
"Used automatically when no API key is set.",
onSelect: () => {},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
},
};
}
/** Download/cancel/delete management for the on-phone assistant model. */
function localModelMenuItem(): MenuItem {
return {