forked from Blendman974/faceclaw
Support Apple Intelligence as an on-device AI model for the voice assistant/etc
This commit is contained in:
1 parent
3c3aa31542
commit
4973fd2aea
22 files changed
+1551
-40
No files matched your search
@@ -16,7 +16,7 @@ import {
|
||||
assistantModelProvider,
|
||||
type AssistantModel,
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/llama";
|
||||
import { isLocalModelReady } from "../native/local-llm";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
@@ -930,7 +930,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
export type AssistantBackendKind = "direct" | "external";
|
||||
|
||||
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
||||
direct: global.isIOS ? "Cloud API" : "On-phone",
|
||||
direct: "On-phone",
|
||||
external: "My own agent (bridge)",
|
||||
};
|
||||
|
||||
@@ -942,7 +942,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
|
||||
values: global.isIOS ? ["direct"] : ["direct", "external"],
|
||||
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
||||
description:
|
||||
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys), or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
});
|
||||
|
||||
export const assistantHermesUrlSetting = new ConfigSettingString({
|
||||
@@ -1116,7 +1116,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
|
||||
return false;
|
||||
},
|
||||
description:
|
||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
});
|
||||
|
||||
export const mapboxApiKeySetting = new ConfigSettingString({
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
} from "../../native/local-llm";
|
||||
import {
|
||||
ASR_MODELS,
|
||||
asrModelState,
|
||||
@@ -143,14 +143,14 @@ function settingsSections(): SettingsSection[] {
|
||||
{
|
||||
label: "Assistant",
|
||||
items: [
|
||||
// iOS offers cloud APIs; Android also supports local models and the bridge.
|
||||
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
textSettingMenuItem(assistantHermesUrlSetting),
|
||||
textSettingMenuItem(assistantHermesKeySetting),
|
||||
...(!global.isIOS ? [localModelMenuItem()] : []),
|
||||
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
// iOS has no local LLM or agent bridge yet.
|
||||
// iOS has no agent bridge yet.
|
||||
...(!global.isIOS ? [
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
@@ -399,6 +399,29 @@ function localModelStatusText(): string {
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/**
|
||||
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
|
||||
* installs and manages; the row only reports whether it can be used.
|
||||
*/
|
||||
function systemModelMenuItem(): MenuItem {
|
||||
const statusText = (): string => {
|
||||
const state = localModelState();
|
||||
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
|
||||
};
|
||||
return {
|
||||
label: "On-phone model",
|
||||
description:
|
||||
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
|
||||
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
|
||||
"Faster and more private than the cloud models, but simpler, with a short memory. " +
|
||||
"Used automatically when no API key is set.",
|
||||
onSelect: () => {},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-phone assistant model. */
|
||||
function localModelMenuItem(): MenuItem {
|
||||
return {
|
||||
|
||||
Reference in new issue
Block a user