Add support for a local model (Qwen3)

This commit is contained in:
James Babcock committed 2026-08-09 23:52:21 -07:00
1 parent 6103c00ca3
commit 6f45888627
16 files changed
+1532 -15

No files matched your search

+11 -4
View File
@@ -1,4 +1,5 @@
import { streamAnthropicMessage } from "../native/anthropic";
import { streamLocalQwen } from "../native/llama";
import { streamOpenAiResponse } from "../native/openai";
import type {
LlmContentBlock,
@@ -20,6 +21,8 @@ import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/** Safety caps so a tool-thrashing model can't loop or run forever. */
const MAX_ITERATIONS = 8;
/** The on-phone model is slower per step and unreliable on long tool chains. */
const MAX_ITERATIONS_LOCAL = 4;
const MAX_TURN_MS = 2 * 60 * 1000;
export type DirectTurnOptions = {
@@ -54,9 +57,10 @@ export class DirectAssistantBackend {
options.callbacks.onError(message);
};
const maxIterations = options.provider === "local" ? MAX_ITERATIONS_LOCAL : MAX_ITERATIONS;
const runIteration = (iteration: number): void => {
if (cancelled) return;
if (iteration >= MAX_ITERATIONS) {
if (iteration >= maxIterations) {
finishError("Assistant stopped: too many tool steps");
return;
}
@@ -115,9 +119,12 @@ export class DirectAssistantBackend {
finishError(message);
},
};
streamHandle = options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: streamAnthropicMessage(streamOptions);
streamHandle =
options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: options.provider === "local"
? streamLocalQwen(streamOptions)
: streamAnthropicMessage(streamOptions);
};
runIteration(0);
+145
View File
@@ -0,0 +1,145 @@
import type { LlmToolDefinition } from "./llm-protocol";
/**
* Generate a GBNF grammar (llama.cpp grammar dialect) that constrains a
* Qwen-style tool call to be well-formed: one <tool_call> block whose JSON
* names a real tool and whose arguments match that tool's schema. Used as a
* *lazy* grammar — sampling is unconstrained until the model emits
* <tool_call>, so plain text answers are unaffected, and once the block
* closes the grammar is complete (the model can only end the message, which
* serializes multi-call turns into one call per loop iteration — deliberate:
* small models are far more reliable one call at a time).
*
* Our tool schemas are flat objects of string/number/boolean/enum properties.
* Anything fancier degrades to a generic JSON value rule rather than failing.
*/
type JsonSchema = {
type?: string;
enum?: unknown[];
properties?: Record<string, JsonSchema>;
required?: string[];
description?: string;
};
/** Longest run of optional properties we'll enumerate subsets for. */
const MAX_SUBSET_PROPS = 5;
export function buildToolCallGrammar(tools: LlmToolDefinition[]): string | null {
if (!tools.length) return null;
const used = new Set<string>();
const toolRules: string[] = [];
for (let i = 0; i < tools.length; i++) {
const args = argsRule(tools[i]!.input_schema as JsonSchema, used);
toolRules.push(
`tc${i} ::= "<tool_call>" nl "{" sp ${jsonKey("name")} sp ":" sp "\\"${gbnfEscape(
tools[i]!.name,
)}\\"" sp "," sp ${jsonKey("arguments")} sp ":" sp ${args} sp "}" nl "</tool_call>"`,
);
}
used.add("sp");
used.add("nl");
const lines = [`root ::= ${toolRules.map((_, i) => `tc${i}`).join(" | ")}`, ...toolRules];
for (const [name, rule] of SHARED_RULES) {
if (used.has(name)) lines.push(rule);
}
return lines.join("\n");
}
/** Build the arguments-object expression for one tool schema. */
function argsRule(schema: JsonSchema | undefined, used: Set<string>): string {
const properties = schema?.properties ?? {};
const names = Object.keys(properties);
if (!names.length) return `"{" sp "}"`;
const required = new Set(schema?.required ?? []);
const optionalCount = names.filter((n) => !required.has(n)).length;
if (names.length > MAX_SUBSET_PROPS && optionalCount > 0) {
used.add("jobj");
return "jobj";
}
// Enumerate every subset of properties that includes all required ones,
// keeping declaration order. Small N keeps this tiny (<= 2^5 alternatives).
const members = names.map((name) => {
const value = valueExpr(properties[name]!, used);
return `${jsonKey(name)} sp ":" sp ${value}`;
});
const alternatives: string[] = [];
for (let mask = 0; mask < 1 << names.length; mask++) {
const chosen: string[] = [];
let valid = true;
for (let j = 0; j < names.length; j++) {
const included = (mask & (1 << j)) !== 0;
if (included) chosen.push(members[j]!);
else if (required.has(names[j]!)) valid = false;
}
if (!valid) continue;
if (!chosen.length) {
alternatives.push(`"{" sp "}"`);
} else {
alternatives.push(`"{" sp ${chosen.join(` sp "," sp `)} sp "}"`);
}
}
return alternatives.length === 1 ? alternatives[0]! : `(${alternatives.join(" | ")})`;
}
/** Grammar expression for one property value. */
function valueExpr(schema: JsonSchema, used: Set<string>): string {
if (Array.isArray(schema.enum) && schema.enum.length) {
if (schema.enum.every((v) => typeof v === "string")) {
return `(${schema.enum.map((v) => `"\\"${gbnfEscape(String(v))}\\""`).join(" | ")})`;
}
if (schema.enum.every((v) => typeof v === "number" || typeof v === "boolean")) {
return `(${schema.enum.map((v) => `"${String(v)}"`).join(" | ")})`;
}
}
switch (schema.type) {
case "string":
used.add("jstring");
used.add("jchar");
used.add("jhex");
return "jstring";
case "number":
used.add("jnumber");
return "jnumber";
case "integer":
used.add("jint");
return "jint";
case "boolean":
used.add("jbool");
return "jbool";
default:
for (const name of ["jvalue", "jstring", "jchar", "jhex", "jnumber", "jbool", "jarray", "jobj", "jmember"]) {
used.add(name);
}
return "jvalue";
}
}
/** A JSON object key as a GBNF literal: "name" -> "\"name\"". */
function jsonKey(name: string): string {
return `"\\"${gbnfEscape(name)}\\""`;
}
/** Escape for the inside of a GBNF double-quoted literal. */
function gbnfEscape(text: string): string {
return text.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
}
// Terminal rules, emitted only when referenced. jchar mirrors llama.cpp's
// json.gbnf string character rule.
const SHARED_RULES: Array<[string, string]> = [
["sp", `sp ::= " "?`],
["nl", `nl ::= "\\n"`],
["jstring", `jstring ::= "\\"" jchar* "\\""`],
["jchar", `jchar ::= [^"\\\\\\x7F\\x00-\\x1F] | "\\\\" (["\\\\bfnrt] | "u" jhex jhex jhex jhex)`],
["jhex", `jhex ::= [0-9a-fA-F]`],
["jnumber", `jnumber ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? (("e" | "E") ("-" | "+")? [0-9]+)?`],
["jint", `jint ::= "-"? ("0" | [1-9] [0-9]*)`],
["jbool", `jbool ::= "true" | "false"`],
["jvalue", `jvalue ::= jstring | jnumber | jbool | "null" | jarray | jobj`],
["jarray", `jarray ::= "[" sp (jvalue (sp "," sp jvalue)*)? sp "]"`],
["jobj", `jobj ::= "{" sp (jmember (sp "," sp jmember)*)? sp "}"`],
["jmember", `jmember ::= jstring sp ":" sp jvalue`],
];
+81
View File
@@ -0,0 +1,81 @@
import type { LlmContentBlock, LlmMessage, LlmToolDefinition } from "./llm-protocol";
/**
* Render our provider-neutral message history into a Qwen3 (ChatML) prompt
* string, matching the model's chat template closely enough that its
* tool-calling priors apply: Hermes-style <tools> declarations in the system
* block, <tool_call> JSON blocks in assistant turns, and tool results wrapped
* in <tool_response> inside a user turn. The caller appends nothing — the
* returned string ends with the assistant generation header.
*/
const IM_START = "<|im_start|>";
const IM_END = "<|im_end|>";
export function buildQwenPrompt(
system: string | undefined,
messages: LlmMessage[],
tools: LlmToolDefinition[] | undefined,
): string {
const parts: string[] = [];
parts.push(`${IM_START}system\n${systemBlock(system, tools)}${IM_END}\n`);
for (const message of messages) {
if (message.role === "assistant") {
parts.push(`${IM_START}assistant\n${assistantBlock(message.content)}${IM_END}\n`);
} else {
parts.push(`${IM_START}user\n${userBlock(message.content)}${IM_END}\n`);
}
}
parts.push(`${IM_START}assistant\n`);
return parts.join("");
}
function systemBlock(system: string | undefined, tools: LlmToolDefinition[] | undefined): string {
const base = system?.trim() ?? "You are a helpful assistant.";
if (!tools?.length) return base;
const declarations = tools
.map((tool) =>
JSON.stringify({
type: "function",
function: { name: tool.name, description: tool.description, parameters: tool.input_schema },
}),
)
.join("\n");
return (
`${base}\n\n# Tools\n\n` +
`You may call one or more functions to assist with the user query.\n\n` +
`You are provided with function signatures within <tools></tools> XML tags:\n` +
`<tools>\n${declarations}\n</tools>\n\n` +
`For each function call, return a json object with function name and arguments ` +
`within <tool_call></tool_call> XML tags:\n` +
`<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>`
);
}
function assistantBlock(content: string | LlmContentBlock[]): string {
if (typeof content === "string") return content;
const parts: string[] = [];
for (const block of content) {
if (block.type === "text") {
if (block.text.trim()) parts.push(block.text.trim());
} else if (block.type === "tool_use") {
parts.push(`<tool_call>\n${JSON.stringify({ name: block.name, arguments: block.input ?? {} })}\n</tool_call>`);
}
// provider_item blocks are OpenAI-specific continuation state; skip.
}
return parts.join("\n");
}
function userBlock(content: string | LlmContentBlock[]): string {
if (typeof content === "string") return content;
const parts: string[] = [];
for (const block of content) {
if (block.type === "text") {
if (block.text.trim()) parts.push(block.text.trim());
} else if (block.type === "tool_result") {
const body = block.is_error ? `Error: ${block.content}` : block.content || "(no output)";
parts.push(`<tool_response>\n${body}\n</tool_response>`);
}
}
return parts.join("\n");
}
+16 -4
View File
@@ -1,4 +1,8 @@
import { buildAssistantSystemPrompt } from "../prompts";
import {
ASSISTANT_SYSTEM_PROMPT_BASE,
buildAssistantSystemPrompt,
describeAssistantContext,
} from "../prompts";
import { assistantBridge } from "./bridge-client";
import { DirectAssistantBackend } from "./direct-backend";
import type { LlmMessage, LlmToolDefinition } from "./llm-protocol";
@@ -108,15 +112,23 @@ export class AssistantSession {
return;
}
this.messages.push({ role: "user", content: text });
this.trimHistory();
const llm = this.config.llm;
// Local provider: keep the system prompt (and thus the tool declarations
// that follow it in the rendered prompt) byte-stable so the on-phone
// model's KV prefix cache survives across turns; the volatile context
// (time, foreground app, battery) rides along with the utterance instead.
const isLocal = llm.provider === "local";
this.messages.push({
role: "user",
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
});
this.trimHistory();
this.turnHandle = this.directBackend.runTurn({
provider: llm.provider,
apiKey: llm.apiKey,
model: llm.model,
effort: llm.effort,
system: buildAssistantSystemPrompt(ctx),
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
messages: this.messages,
buildTools: () => this.buildToolDefinitions(),
registry: this.registry,
+321
View File
@@ -0,0 +1,321 @@
import { Utils } from "@nativescript/core";
import { buildToolCallGrammar } from "../assistant/gbnf";
import type {
LlmContentBlock,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
import { buildQwenPrompt } from "../assistant/qwen-prompt";
declare const com: any;
declare const java: any;
/**
* On-phone LLM backend: model download management plus a streaming client
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
* the direct assistant loop can treat "local" as just another provider.
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
*/
export const LOCAL_MODEL = {
id: "qwen3-4b-instruct-2507",
label: "Qwen3 4B",
fileName: "Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
url: "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/Qwen3-4B-Instruct-2507-Q4_K_M.gguf?download=true",
sha256: "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597",
sizeBytes: 2497281120,
};
// Bounded so KV cache stays ~0.5GB (Qwen3-4B KV is ~144KB/token at f16);
// the assistant trims history well before this in practice.
const N_CTX = 4096;
const N_THREADS = 4;
const DEFAULT_MAX_TOKENS = 512;
// Qwen3-4B-Instruct-2507 model-card sampling posture.
const TEMPERATURE = 0.7;
const TOP_P = 0.8;
const TOP_K = 20;
/** Free the ~3GB of model + KV this long after the last generation. */
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
export type LocalModelState = {
status: "absent" | "downloading" | "ready";
bytesDownloaded: number;
totalBytes: number;
};
let runner: any = null;
let downloader: any = null;
let downloadedBytes = 0;
let unloadTimer: ReturnType<typeof setTimeout> | null = null;
let toolUseCounter = 0;
const stateListeners = new Set<(state: LocalModelState) => void>();
function modelPath(): string {
const context = Utils.android.getApplicationContext();
const dir = context.getExternalFilesDir("llm") ?? context.getFilesDir();
return `${dir.getAbsolutePath()}/${LOCAL_MODEL.fileName}`;
}
export function isLocalModelReady(): boolean {
if (!global.isAndroid) return false;
try {
const file = new java.io.File(modelPath());
return file.exists() && file.length() > 0;
} catch {
return false;
}
}
export function localModelState(): LocalModelState {
if (downloader) {
return { status: "downloading", bytesDownloaded: downloadedBytes, totalBytes: LOCAL_MODEL.sizeBytes };
}
return {
status: isLocalModelReady() ? "ready" : "absent",
bytesDownloaded: 0,
totalBytes: LOCAL_MODEL.sizeBytes,
};
}
export function onLocalModelStateChanged(listener: (state: LocalModelState) => void): () => void {
stateListeners.add(listener);
return () => stateListeners.delete(listener);
}
function notifyStateChanged(): void {
const state = localModelState();
for (const listener of stateListeners) listener(state);
}
export function startLocalModelDownload(): void {
if (!global.isAndroid || downloader || isLocalModelReady()) return;
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
onProgress: (bytes: number, _total: number) => {
downloadedBytes = Number(bytes);
notifyStateChanged();
},
onDone: () => {
downloader = null;
notifyStateChanged();
},
onError: (message: string) => {
console.error(`Local model download failed: ${message}`);
downloader = null;
notifyStateChanged();
},
});
downloader = new com.faceclaw.app.FaceclawModelDownloader(
LOCAL_MODEL.url,
modelPath(),
LOCAL_MODEL.sha256,
LOCAL_MODEL.sizeBytes,
listener,
);
downloadedBytes = 0;
downloader.start();
notifyStateChanged();
}
/** Stops the download; already-fetched bytes are kept and resumed next time. */
export function cancelLocalModelDownload(): void {
if (!downloader) return;
downloader.cancel();
downloader = null;
notifyStateChanged();
}
export function deleteLocalModel(): void {
if (!global.isAndroid) return;
cancelLocalModelDownload();
unloadLocalModel();
try {
new java.io.File(modelPath()).delete();
new java.io.File(`${modelPath()}.part`).delete();
} catch (error) {
console.error(`Local model delete failed: ${String(error)}`);
}
notifyStateChanged();
}
export function unloadLocalModel(): void {
if (unloadTimer) {
clearTimeout(unloadTimer);
unloadTimer = null;
}
runner?.unload();
}
function scheduleIdleUnload(): void {
if (unloadTimer) clearTimeout(unloadTimer);
unloadTimer = setTimeout(() => {
unloadTimer = null;
runner?.unload();
}, IDLE_UNLOAD_MS);
}
function ensureRunner(): any {
if (!runner) {
runner = new com.faceclaw.app.FaceclawLlamaRunner();
}
return runner;
}
const TOOL_CALL_OPEN = "<tool_call>";
const TOOL_CALL_CLOSE = "</tool_call>";
/**
* Stream a completion from the on-phone model. Tool calls arrive inline as
* <tool_call> JSON blocks; they are hidden from the text stream and surfaced
* as tool_use content blocks in the final result, mirroring the cloud
* providers' contract (stopReason "tool_use" when the model called a tool).
*/
export function streamLocalQwen(options: LlmStreamOptions): LlmStreamHandle {
let settled = false;
let raw = "";
let emitted = "";
const fail = (message: string) => {
if (settled) return;
settled = true;
scheduleIdleUnload();
options.onError(message);
};
if (!global.isAndroid) {
setTimeout(() => fail("The on-phone model is only available on Android"), 0);
return { cancel: () => {} };
}
if (!isLocalModelReady()) {
setTimeout(() => fail("The on-phone model isn't downloaded yet (see Settings)"), 0);
return { cancel: () => {} };
}
if (unloadTimer) {
clearTimeout(unloadTimer);
unloadTimer = null;
}
const prompt = buildQwenPrompt(options.system, options.messages, options.tools);
const grammar = options.tools?.length ? buildToolCallGrammar(options.tools) : null;
const listener = new com.faceclaw.app.FaceclawLlamaListener({
onToken: (piece: string) => {
if (settled) return;
raw += String(piece);
const visible = visibleText(raw);
if (visible.length > emitted.length) {
const delta = visible.slice(emitted.length);
emitted = visible;
options.onTextDelta?.(delta, emitted);
}
},
onDone: (nativeStopReason: string) => {
if (settled) return;
settled = true;
scheduleIdleUnload();
const { text, content, toolCount } = assembleContent(raw);
const stopReason =
toolCount > 0 ? "tool_use" : String(nativeStopReason) === "length" ? "max_tokens" : "end_turn";
options.onDone({ text, content, stopReason });
},
onError: (message: string) => fail(String(message)),
});
try {
ensureRunner().generate(
modelPath(),
N_CTX,
N_THREADS,
prompt,
grammar ?? "",
options.maxTokens ?? DEFAULT_MAX_TOKENS,
TEMPERATURE,
TOP_P,
TOP_K,
listener,
);
} catch (error) {
setTimeout(() => fail(`On-phone model failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
}
return {
cancel: () => {
settled = true;
scheduleIdleUnload();
try {
runner?.cancel();
} catch {
// ignore
}
},
};
}
/**
* The streamable portion of the raw output: text outside completed
* <tool_call> blocks, holding back any trailing partial marker so a tool
* call never leaks into the visible stream. Monotonic under appends.
*/
function visibleText(raw: string): string {
let out = "";
let i = 0;
for (;;) {
const start = raw.indexOf(TOOL_CALL_OPEN, i);
if (start === -1) {
const tail = raw.slice(i);
out += tail.slice(0, tail.length - partialSuffixLength(tail, TOOL_CALL_OPEN));
return out;
}
out += raw.slice(i, start);
const end = raw.indexOf(TOOL_CALL_CLOSE, start);
if (end === -1) return out; // inside an unfinished tool call
i = end + TOOL_CALL_CLOSE.length;
}
}
/** Length of the longest proper prefix of marker that text ends with. */
function partialSuffixLength(text: string, marker: string): number {
const max = Math.min(text.length, marker.length - 1);
for (let k = max; k > 0; k--) {
if (text.endsWith(marker.slice(0, k))) return k;
}
return 0;
}
/** Split the finished raw output into text + tool_use content blocks. */
function assembleContent(raw: string): { text: string; content: LlmContentBlock[]; toolCount: number } {
const content: LlmContentBlock[] = [];
let text = "";
let toolCount = 0;
let i = 0;
for (;;) {
const start = raw.indexOf(TOOL_CALL_OPEN, i);
if (start === -1) {
text += raw.slice(i);
break;
}
text += raw.slice(i, start);
const end = raw.indexOf(TOOL_CALL_CLOSE, start);
if (end === -1) break; // truncated call (length stop); drop it
const body = raw.slice(start + TOOL_CALL_OPEN.length, end).trim();
try {
const parsed = JSON.parse(body);
if (parsed && typeof parsed.name === "string") {
toolCount++;
content.push({
type: "tool_use",
id: `local_${Date.now()}_${++toolUseCounter}`,
name: parsed.name,
input: parsed.arguments ?? {},
});
}
} catch {
console.error(`Local model emitted unparseable tool call: ${body.slice(0, 200)}`);
}
i = end + TOOL_CALL_CLOSE.length;
}
text = text.trim();
if (text) content.unshift({ type: "text", text });
return { text, content, toolCount };
}
+9 -2
View File
@@ -15,10 +15,17 @@ export const ASSISTANT_SYSTEM_PROMPT_BASE = [
/** Full assistant system prompt: the base plus the current device context. */
export function buildAssistantSystemPrompt(ctx: AssistantContext): string {
return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeAssistantContext(ctx)}`;
}
function describeContext(ctx: AssistantContext): string {
/**
* The volatile device context on its own. The local provider appends this to
* the user message instead of the system prompt: the system prompt + tool
* declarations are the bulk of the prompt, and keeping them byte-stable lets
* the on-phone model reuse its KV-cache prefix across turns instead of
* re-prefilling ~2-3k tokens because the clock changed.
*/
export function describeAssistantContext(ctx: AssistantContext): string {
const parts = [`Current time: ${ctx.localTime}.`];
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
if (ctx.foregroundApp) {
+6 -3
View File
@@ -19,6 +19,7 @@ import {
assistantModelProvider,
type AssistantModel,
} from "~/assistant/models";
import { isLocalModelReady } from "../native/llama";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
import { DashboardInputEvent, Layer, type LayerContext } from "./layers";
import { GrayImage } from "~/graphics/image";
@@ -357,7 +358,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
export type AssistantBackendKind = "direct" | "external";
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
direct: "On-phone (API key)",
direct: "On-phone",
external: "My own agent (bridge)",
};
@@ -369,7 +370,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
values: ["direct", "external"],
formatValue: (value) => assistantBackendLabels[value] ?? value,
description:
"Who answers assistant queries: an LLM called directly from the phone (needs an API key), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
"Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
});
export const assistantBridgeHostSetting = new ConfigSettingString({
@@ -468,9 +469,11 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
const provider = assistantModelProvider(value);
if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0;
if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0;
if (provider === "local") return !isLocalModelReady();
return false;
},
description: "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set.",
description:
"Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
});
export const mapboxApiKeySetting = new ConfigSettingString({
+82 -1
View File
@@ -3,8 +3,17 @@ import { getDefaultSmallFont } from "../../graphics/bdffont";
import type { GrayImage } from "../../graphics/image";
import { getDashboardLogo } from "../../graphics/logo";
import { wrapText } from "../../graphics/textwrap";
import {
cancelLocalModelDownload,
deleteLocalModel,
LOCAL_MODEL,
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
} from "../../native/llama";
import { TextViewerLayer } from "../../apps/files/text-viewer";
import type { MenuItem } from "../menu";
import type { LayerContext } from "../layers";
import { drawRightValueMenuItem, openModalMenu, type MenuItem } from "../menu";
import { shell } from "../shell/shell";
import {
anthropicApiKeySetting,
@@ -88,6 +97,7 @@ function settingsSections(): SettingsSection[] {
// On-phone LLM loop vs the user's own agent via the bridge plugin.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
localModelMenuItem(),
// When on, a wakeword utterance goes straight to the assistant with no
// Send/Type menu step.
toggleSettingMenuItem(assistantSkipConfirmationSetting),
@@ -161,6 +171,77 @@ function settingsSections(): SettingsSection[] {
];
}
const LOCAL_MODEL_GB = `${(LOCAL_MODEL.sizeBytes / 1e9).toFixed(1)}GB`;
// While a download is running, re-render on progress updates so the row's
// percentage stays live; the watch tears itself down when the download ends.
let localModelRenderUnsub: (() => void) | null = null;
function watchLocalModelDownload(ctx: LayerContext): void {
localModelRenderUnsub?.();
localModelRenderUnsub = onLocalModelStateChanged((state) => {
ctx.actions.requestRender();
if (state.status !== "downloading") {
localModelRenderUnsub?.();
localModelRenderUnsub = null;
}
});
}
function localModelStatusText(): string {
const state = localModelState();
if (state.status === "ready") return "downloaded";
if (state.status === "downloading") {
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
return `${pct}% of ${LOCAL_MODEL_GB}`;
}
return "not downloaded";
}
/** Download/cancel/delete management for the on-phone assistant model. */
function localModelMenuItem(): MenuItem {
return {
label: "On-phone model",
description:
`${LOCAL_MODEL.label} (${LOCAL_MODEL_GB} download over Wi-Fi recommended). ` +
"Answers assistant queries on the phone itself, with no API key or cloud service. " +
"Slower and simpler than the cloud models, but free and private. " +
"Used automatically when no API key is set. An interrupted download resumes where it left off.",
onSelect: (ctx) => {
const state = localModelState();
const action: MenuItem =
state.status === "downloading"
? {
label: "Cancel download",
onSelect: (innerCtx) => {
cancelLocalModelDownload();
innerCtx.stack.pop();
},
}
: state.status === "ready"
? {
label: "Delete model",
onSelect: (innerCtx) => {
deleteLocalModel();
innerCtx.stack.pop();
},
}
: {
label: `Download (${LOCAL_MODEL_GB})`,
onSelect: (innerCtx) => {
startLocalModelDownload();
watchLocalModelDownload(innerCtx);
innerCtx.stack.pop();
},
};
openModalMenu(ctx, "On-phone model", [action], 0);
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", localModelStatusText());
},
};
}
/** A row that opens one of the project docs (copied into the bundle under
* about/ by webpack.config.js) in the paged text viewer. */
function bundledDocMenuItem(fileName: string, label: string): MenuItem {
+1 -1
View File
@@ -801,7 +801,7 @@ class Shell {
this.showAlert(
assistantBackendSetting.get() === "external"
? "Configure the agent bridge host and token in Settings."
: "Set an API key for the selected assistant model in Settings.",
: "Set an API key or download the on-phone model in Settings.",
);
return;
}