forked from Blendman974/faceclaw
Support Apple Intelligence as an on-device AI model for the voice assistant/etc
This commit is contained in:
1 parent
3c3aa31542
commit
4973fd2aea
22 files changed
+1551
-40
No files matched your search
@@ -7,7 +7,7 @@
|
||||
* surfacing a transport error.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
@@ -41,7 +41,7 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
streamOpenAiResponse(options);
|
||||
break;
|
||||
case "local":
|
||||
streamLocalQwen(options);
|
||||
streamLocalModel(options);
|
||||
break;
|
||||
default:
|
||||
streamAnthropicMessage(options);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
||||
|
||||
@@ -245,7 +245,7 @@ export function askConversations(
|
||||
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
return streamLocalQwen({
|
||||
return streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { NameInference } from "./name-guess";
|
||||
|
||||
/**
|
||||
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import { speakerRegistry } from "./speakers";
|
||||
|
||||
declare const com: any;
|
||||
@@ -124,7 +124,7 @@ function generateForSpeaker(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* TranslationModel for how stretches are batched and given context.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
@@ -55,7 +55,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
case "openai":
|
||||
return streamOpenAiResponse(options);
|
||||
case "local":
|
||||
return streamLocalQwen(options);
|
||||
return streamLocalModel(options);
|
||||
default:
|
||||
return streamAnthropicMessage(options);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { streamAnthropicMessage } from "../native/anthropic";
|
||||
import { streamHermesChat } from "../native/hermes";
|
||||
import { streamLocalQwen } from "../native/llama";
|
||||
import { streamLocalModel } from "../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../native/openai";
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
@@ -146,7 +146,7 @@ export class DirectAssistantBackend {
|
||||
: options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: options.provider === "local"
|
||||
? streamLocalQwen(streamOptions)
|
||||
? streamLocalModel(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
};
|
||||
|
||||
|
||||
+13
-7
@@ -1,4 +1,4 @@
|
||||
import { isLocalModelReady, LOCAL_MODEL } from "../native/llama";
|
||||
import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm";
|
||||
|
||||
export const ASSISTANT_MODEL_VALUES = [
|
||||
"auto",
|
||||
@@ -13,12 +13,17 @@ export const ASSISTANT_MODEL_VALUES = [
|
||||
"hermes",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on
|
||||
* Android, the Apple Intelligence system model on iOS (the ID predates iOS
|
||||
* support and stays for saved settings and conversations).
|
||||
*/
|
||||
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
|
||||
/** Keep saved model IDs portable, while offering only supported choices. */
|
||||
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
|
||||
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
|
||||
isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen");
|
||||
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
|
||||
return global.isIOS && value === "qwen" ? "auto" : value;
|
||||
return value === "qwen" && !isLocalModelSupported() ? "auto" : value;
|
||||
}
|
||||
|
||||
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
|
||||
@@ -58,8 +63,9 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
|
||||
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
|
||||
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
|
||||
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
|
||||
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
|
||||
// not an API key. The default for users who haven't set any key.
|
||||
// Runs on the phone itself (Android: llama.cpp with the downloaded model
|
||||
// file; iOS: Apple Intelligence), not an API key. The default for users who
|
||||
// haven't set any key.
|
||||
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
|
||||
// Your own Hermes Agent (its API server), with its own tools and memory.
|
||||
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
|
||||
@@ -76,8 +82,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider
|
||||
/**
|
||||
* Resolve Auto and attach the credential needed by the chosen provider.
|
||||
* Auto intentionally prefers Terra over Sonnet when both keys are available,
|
||||
* and falls back to the on-phone model when no key is set but the model file
|
||||
* has been downloaded.
|
||||
* and falls back to the on-phone model when no key is set but that model is
|
||||
* ready (downloaded on Android; Apple Intelligence on and ready on iOS).
|
||||
*/
|
||||
export function resolveAssistantModel(
|
||||
selection: AssistantModel,
|
||||
|
||||
+4
-6
@@ -6,12 +6,14 @@ import type {
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* On-phone LLM backend: model download management plus a streaming client
|
||||
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
|
||||
* local-llm.ios.ts instead): model download management plus a streaming client
|
||||
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
||||
* the direct assistant loop can treat "local" as just another provider.
|
||||
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
||||
@@ -38,11 +40,7 @@ const TOP_K = 20;
|
||||
/** Free the ~3GB of model + KV this long after the last generation. */
|
||||
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
||||
|
||||
export type LocalModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
let runner: any = null;
|
||||
let downloader: any = null;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
|
||||
|
||||
export type LocalModelState = {
|
||||
/**
|
||||
* Android downloads its model: absent → downloading → ready. iOS uses the
|
||||
* system's Apple Intelligence model: ready, or unavailable (see detail).
|
||||
*/
|
||||
status: "absent" | "downloading" | "ready" | "unavailable";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
|
||||
detail?: string;
|
||||
};
|
||||
@@ -0,0 +1,383 @@
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmMessage,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const FaceclawSystemLanguageModel: any;
|
||||
|
||||
/**
|
||||
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
|
||||
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
|
||||
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
|
||||
* Calculator and the Microphones helpers treat it as the "local" provider.
|
||||
*
|
||||
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
|
||||
* background (where Faceclaw usually is, driving the glasses with the phone
|
||||
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
|
||||
* ~3 GB in-process model makes the app the first thing jetsam kills. The
|
||||
* system model runs out of process on the Neural Engine, needs no download,
|
||||
* and decodes tool arguments under a schema constraint (like our GBNF grammar
|
||||
* on Android). Its costs: a 4096-token context (the Swift side fits history
|
||||
* and tool definitions into it), Apple's guardrails, a background request
|
||||
* budget, and Apple Intelligence-capable hardware.
|
||||
*
|
||||
* Tool calls: the framework runs tools itself, mid-response. The Swift side
|
||||
* parks each call and reports it; here that ends the loop iteration with
|
||||
* stopReason "tool_use", the direct backend executes the tools as for any
|
||||
* provider, and its next request (ending in the matching tool_result blocks)
|
||||
* resumes the parked generation instead of starting a new one.
|
||||
*/
|
||||
|
||||
export const LOCAL_MODEL = {
|
||||
id: "apple-foundation-model",
|
||||
label: "Apple Intelligence",
|
||||
sizeBytes: 0,
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_TOKENS = 512;
|
||||
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
|
||||
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
|
||||
/**
|
||||
* Only used when tool results arrive with no parked generation to resume (it
|
||||
* timed out, or the app restarted mid-turn): the framework has no way to
|
||||
* continue after a tool output without a new prompt.
|
||||
*/
|
||||
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
|
||||
|
||||
type Availability =
|
||||
| "available"
|
||||
| "unsupportedOS"
|
||||
| "deviceNotEligible"
|
||||
| "appleIntelligenceNotEnabled"
|
||||
| "modelNotReady"
|
||||
| "unavailable";
|
||||
|
||||
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "needs iOS 26",
|
||||
deviceNotEligible: "not on this iPhone",
|
||||
appleIntelligenceNotEnabled: "Apple Intelligence off",
|
||||
modelNotReady: "downloading",
|
||||
unavailable: "unavailable",
|
||||
};
|
||||
|
||||
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
|
||||
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
|
||||
appleIntelligenceNotEnabled:
|
||||
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
|
||||
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
|
||||
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
|
||||
};
|
||||
|
||||
export function localModelAvailability(): Availability {
|
||||
try {
|
||||
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
|
||||
const value = String(FaceclawSystemLanguageModel.availability());
|
||||
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
|
||||
} catch {
|
||||
return "unsupportedOS";
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
const availability = localModelAvailability();
|
||||
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
|
||||
}
|
||||
|
||||
export function isLocalModelReady(): boolean {
|
||||
return localModelAvailability() === "available";
|
||||
}
|
||||
|
||||
export function localModelState(): LocalModelState {
|
||||
const availability = localModelAvailability();
|
||||
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
|
||||
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
|
||||
}
|
||||
|
||||
/** The user-facing reason the model can't run, or "" when it can. */
|
||||
export function localModelUnavailableMessage(): string {
|
||||
const availability = localModelAvailability();
|
||||
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
|
||||
}
|
||||
|
||||
// iOS manages the system model itself: there is nothing to download, cancel,
|
||||
// delete or unload, and no progress to watch.
|
||||
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
|
||||
return () => {};
|
||||
}
|
||||
export function startLocalModelDownload(): void {}
|
||||
export function cancelLocalModelDownload(): void {}
|
||||
export function deleteLocalModel(): void {}
|
||||
export function unloadLocalModel(): void {}
|
||||
|
||||
type HistoryEntry =
|
||||
| { type: "prompt" | "response"; text: string }
|
||||
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
|
||||
| { type: "toolOutput"; id: string; name: string; text: string };
|
||||
|
||||
export type SystemModelRequest = {
|
||||
system: string;
|
||||
tools: Array<{ name: string; description: string; parameters: object }>;
|
||||
history: HistoryEntry[];
|
||||
prompt: string;
|
||||
maxTokens: number;
|
||||
permissive: boolean;
|
||||
};
|
||||
|
||||
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
|
||||
|
||||
/**
|
||||
* Translate provider-neutral history into the Swift request: everything but
|
||||
* the newest user prompt becomes Transcript entries, and that prompt is what
|
||||
* the model answers.
|
||||
*/
|
||||
export function buildSystemModelRequest(
|
||||
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
|
||||
): SystemModelRequest {
|
||||
const messages = options.messages;
|
||||
const last = messages[messages.length - 1];
|
||||
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
|
||||
const history: HistoryEntry[] = [];
|
||||
const toolNames = new Map<string, string>();
|
||||
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
|
||||
appendHistory(history, message, toolNames);
|
||||
}
|
||||
const tools = (options.tools ?? []).map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.input_schema,
|
||||
}));
|
||||
return {
|
||||
system: options.system?.trim() || "You are a helpful assistant.",
|
||||
tools,
|
||||
history,
|
||||
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
|
||||
permissive: tools.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
|
||||
if (message.role === "user") {
|
||||
for (const result of toolResultsOf(message)) {
|
||||
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
|
||||
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "prompt", text });
|
||||
return;
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "response", text });
|
||||
if (typeof message.content === "string") return;
|
||||
const calls: Array<{ id: string; name: string; arguments: string }> = [];
|
||||
for (const block of message.content) {
|
||||
if (block.type !== "tool_use") continue;
|
||||
toolNames.set(block.id, block.name);
|
||||
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
|
||||
}
|
||||
if (calls.length) history.push({ type: "toolCalls", calls });
|
||||
}
|
||||
|
||||
function textOf(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content.trim();
|
||||
return content
|
||||
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
|
||||
.map((block) => block.text.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
|
||||
if (!message || message.role !== "user" || typeof message.content === "string") return [];
|
||||
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
|
||||
}
|
||||
|
||||
function parseArguments(json: string): any {
|
||||
try {
|
||||
const value = JSON.parse(json);
|
||||
return value && typeof value === "object" ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
type NativeEvent = {
|
||||
kind: string;
|
||||
text?: string;
|
||||
calls?: Array<{ id: string; name: string; arguments: string }>;
|
||||
code?: string;
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/** Generations parked in tool calls, by call id. */
|
||||
const parked = new Map<string, Generation>();
|
||||
/** Every unfinished generation, so its native object outlives the caller's handle. */
|
||||
const live = new Set<Generation>();
|
||||
|
||||
/**
|
||||
* One native generation, possibly spanning several loop iterations: each
|
||||
* iteration ends at a batch of tool calls (or the end of the reply), and the
|
||||
* next one resumes it with the results.
|
||||
*/
|
||||
class Generation {
|
||||
private readonly native: any;
|
||||
private options: LlmStreamOptions | null;
|
||||
private iteration = 0;
|
||||
private text = "";
|
||||
private callIds: string[] = [];
|
||||
private expiry: ReturnType<typeof setTimeout> | null = null;
|
||||
private closed = false;
|
||||
|
||||
constructor(options: LlmStreamOptions) {
|
||||
this.options = options;
|
||||
this.native = FaceclawSystemLanguageModel.new();
|
||||
this.native.eventHandler = (json: string) => this.onEvent(String(json));
|
||||
live.add(this);
|
||||
}
|
||||
|
||||
start(request: SystemModelRequest): LlmStreamHandle {
|
||||
this.native.start(JSON.stringify(request));
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
/** Whether these results answer this generation's parked calls. */
|
||||
awaits(results: ToolResultBlock[]): boolean {
|
||||
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
|
||||
}
|
||||
|
||||
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
|
||||
this.unpark();
|
||||
this.options = options;
|
||||
this.iteration++;
|
||||
this.text = "";
|
||||
this.native.resolveToolCalls(
|
||||
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
|
||||
);
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
if (this.closed) return;
|
||||
this.close();
|
||||
try {
|
||||
this.native.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
private handle(): LlmStreamHandle {
|
||||
const iteration = this.iteration;
|
||||
// A stale handle (an earlier iteration's) still cancels a generation parked
|
||||
// since then: its owner gave up on the turn.
|
||||
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
|
||||
}
|
||||
|
||||
private close(): void {
|
||||
this.closed = true;
|
||||
this.unpark();
|
||||
this.options = null;
|
||||
this.native.eventHandler = null;
|
||||
live.delete(this);
|
||||
}
|
||||
|
||||
private park(ids: string[]): void {
|
||||
this.callIds = ids;
|
||||
for (const id of ids) parked.set(id, this);
|
||||
this.expiry = setTimeout(() => {
|
||||
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
|
||||
this.cancel();
|
||||
}, PARKED_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private unpark(): void {
|
||||
for (const id of this.callIds) {
|
||||
if (parked.get(id) === this) parked.delete(id);
|
||||
}
|
||||
this.callIds = [];
|
||||
if (this.expiry !== null) clearTimeout(this.expiry);
|
||||
this.expiry = null;
|
||||
}
|
||||
|
||||
private onEvent(json: string): void {
|
||||
if (this.closed) return;
|
||||
let event: NativeEvent;
|
||||
try {
|
||||
event = JSON.parse(json);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.kind === "log") {
|
||||
console.log(`[local-llm] ${event.message ?? ""}`);
|
||||
return;
|
||||
}
|
||||
const options = this.options;
|
||||
if (!options) return;
|
||||
switch (event.kind) {
|
||||
case "delta": {
|
||||
const delta = event.text ?? "";
|
||||
this.text += delta;
|
||||
options.onTextDelta?.(delta, this.text);
|
||||
return;
|
||||
}
|
||||
case "toolCalls": {
|
||||
const calls = event.calls ?? [];
|
||||
const text = this.text.trim();
|
||||
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
|
||||
for (const call of calls) {
|
||||
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
|
||||
}
|
||||
this.options = null;
|
||||
this.park(calls.map((call) => call.id));
|
||||
options.onDone({ text, content, stopReason: "tool_use" });
|
||||
return;
|
||||
}
|
||||
case "done": {
|
||||
this.close();
|
||||
const text = this.text.trim();
|
||||
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
|
||||
return;
|
||||
}
|
||||
case "error": {
|
||||
this.close();
|
||||
if (event.code === "refusal") {
|
||||
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
|
||||
} else {
|
||||
options.onError(event.message || "The on-device model failed");
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a reply from the system model. A request whose history ends in tool
|
||||
* results for a parked generation resumes it; anything else starts afresh.
|
||||
*/
|
||||
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
|
||||
const availability = localModelAvailability();
|
||||
if (availability !== "available") {
|
||||
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
const results = toolResultsOf(options.messages[options.messages.length - 1]);
|
||||
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
|
||||
if (waiting?.awaits(results)) return waiting.resume(options, results);
|
||||
let generation: Generation | null = null;
|
||||
try {
|
||||
generation = new Generation(options);
|
||||
return generation.start(buildSystemModelRequest(options));
|
||||
} catch (error) {
|
||||
generation?.cancel();
|
||||
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* The on-phone LLM: one entry point for every feature that can run on the
|
||||
* phone without an API key (the direct assistant's "local" provider,
|
||||
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
|
||||
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
|
||||
* contract, plus model status for Settings.
|
||||
*
|
||||
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
|
||||
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
|
||||
* to download, available on Apple Intelligence devices running iOS 26+.
|
||||
*/
|
||||
export {
|
||||
LOCAL_MODEL,
|
||||
isLocalModelReady,
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
unloadLocalModel,
|
||||
streamLocalQwen as streamLocalModel,
|
||||
} from "./llama";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
return true;
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
assistantModelProvider,
|
||||
type AssistantModel,
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/llama";
|
||||
import { isLocalModelReady } from "../native/local-llm";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
@@ -930,7 +930,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
export type AssistantBackendKind = "direct" | "external";
|
||||
|
||||
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
||||
direct: global.isIOS ? "Cloud API" : "On-phone",
|
||||
direct: "On-phone",
|
||||
external: "My own agent (bridge)",
|
||||
};
|
||||
|
||||
@@ -942,7 +942,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
|
||||
values: global.isIOS ? ["direct"] : ["direct", "external"],
|
||||
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
||||
description:
|
||||
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys), or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
});
|
||||
|
||||
export const assistantHermesUrlSetting = new ConfigSettingString({
|
||||
@@ -1116,7 +1116,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
|
||||
return false;
|
||||
},
|
||||
description:
|
||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
});
|
||||
|
||||
export const mapboxApiKeySetting = new ConfigSettingString({
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
} from "../../native/local-llm";
|
||||
import {
|
||||
ASR_MODELS,
|
||||
asrModelState,
|
||||
@@ -143,14 +143,14 @@ function settingsSections(): SettingsSection[] {
|
||||
{
|
||||
label: "Assistant",
|
||||
items: [
|
||||
// iOS offers cloud APIs; Android also supports local models and the bridge.
|
||||
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
textSettingMenuItem(assistantHermesUrlSetting),
|
||||
textSettingMenuItem(assistantHermesKeySetting),
|
||||
...(!global.isIOS ? [localModelMenuItem()] : []),
|
||||
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
// iOS has no local LLM or agent bridge yet.
|
||||
// iOS has no agent bridge yet.
|
||||
...(!global.isIOS ? [
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
@@ -399,6 +399,29 @@ function localModelStatusText(): string {
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/**
|
||||
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
|
||||
* installs and manages; the row only reports whether it can be used.
|
||||
*/
|
||||
function systemModelMenuItem(): MenuItem {
|
||||
const statusText = (): string => {
|
||||
const state = localModelState();
|
||||
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
|
||||
};
|
||||
return {
|
||||
label: "On-phone model",
|
||||
description:
|
||||
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
|
||||
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
|
||||
"Faster and more private than the cloud models, but simpler, with a short memory. " +
|
||||
"Used automatically when no API key is set.",
|
||||
onSelect: () => {},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-phone assistant model. */
|
||||
function localModelMenuItem(): MenuItem {
|
||||
return {
|
||||
|
||||
Reference in new issue
Block a user