Merge commit 'e2caf28' into hermes-integration

This commit is contained in:
James Babcock committed 2026-10-17 13:43:31 -07:00
commit 33eef6538c
24 files changed
+1063 -43

No files matched your search

+66 -4
View File
@@ -17,6 +17,7 @@ import {
type AssistantModel,
} from "~/assistant/models";
import { isLocalModelReady } from "../native/llama";
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
import { ANIMATION_SPEEDS, animationSpeedLabel, scaleAnimationDuration, type AnimationSpeed } from "./animation-speed";
import { MENU_ANIMATION_KEY } from "./menu-animation-pref";
@@ -822,7 +823,8 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
// runs -- see the same note in native/voice-control.ts. The "whisper" value
// keeps its name (it's a persisted setting on real installs) but its label
// below now says "OpenAI" to tell the two apart in the picker.
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
// "whisper-server" is a self-hosted OpenAI-compatible Whisper (e.g. vLLM).
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
// On iOS "onboard" is Apple's on-device recognizer, and there is no
// "onboard-whisper" (the sherpa-onnx models are Android-only).
@@ -832,6 +834,7 @@ const voiceProviderLabels: Record<VoiceProvider, string> = {
elevenlabs: "ElevenLabs",
whisper: "OpenAI (Whisper)",
soniox: "Soniox",
"whisper-server": "Whisper server (self-hosted)",
};
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
@@ -839,17 +842,20 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
label: "Transcription Provider",
storageKey: "voice.provider",
defaultValue: "onboard",
values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
values: global.isIOS
? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"]
: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"],
formatValue: (value) => voiceProviderLabels[value] ?? value,
isDisabled: (value) => {
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
if (value === "whisper") return openAiApiKeySetting.get().trim().length === 0;
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
if (value === "whisper-server") return !normalizeWhisperServerUrl(whisperServerUrlSetting.get());
return false;
},
description: global.isIOS
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone."
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone."
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.",
});
const wakeWordActionLabels: Record<WakeWordAction, string> = {
@@ -868,6 +874,14 @@ export const wakeWordActionSetting = new ConfigSettingEnum<WakeWordAction>({
description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.",
});
export const saveTranscribeAudioSetting = new ConfigSettingBoolean({
id: "save-transcribe-audio",
label: "Save Transcribe audio",
storageKey: "voice.saveTranscribeAudio",
defaultValue: true,
description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.",
});
export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
id: "save-voice-recordings",
label: "Save voice recordings",
@@ -976,6 +990,54 @@ export const sonioxApiKeySetting = new ConfigSettingString({
description: "Soniox API key, used when Soniox is the transcription provider.",
});
export const whisperServerUrlSetting = new ConfigSettingString({
id: "whisper-server-url",
label: "Whisper server URL",
storageKey: "voice.whisperServerUrl",
defaultValue: "",
editorTitle: "Whisper server URL (e.g. http://100.x.y.z:8004/v1)",
glassesEditTitle: "Edit Whisper server URL",
formatValue: (value) => value || "(not set)",
validate: (value) => (!value.trim() || normalizeWhisperServerUrl(value) ? null : "Enter an http:// or https:// address."),
description:
"API root of an OpenAI-compatible Whisper server (vLLM, speaches, LocalAI...), used when Whisper server is the transcription provider. On iOS a plain-http address must also be allowed by the app's App Transport Security exceptions.",
});
export const whisperServerModelSetting = new ConfigSettingString({
id: "whisper-server-model",
label: "Whisper server model",
storageKey: "voice.whisperServerModel",
defaultValue: "",
editorTitle: "Whisper model id (empty: server's first)",
glassesEditTitle: "Edit Whisper model",
formatValue: (value) => value || "(server default)",
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
});
export const whisperServerLanguageSetting = new ConfigSettingString({
id: "whisper-server-language",
label: "Whisper language",
storageKey: "voice.whisperServerLanguage",
defaultValue: "",
editorTitle: "Language code, e.g. en or sk (empty: auto)",
glassesEditTitle: "Edit Whisper language",
formatValue: (value) => value || "Auto-detect",
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
description:
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
});
export const whisperServerApiKeySetting = new ConfigSettingString({
id: "whisper-server-api-key",
label: "Whisper server key",
storageKey: "voice.whisperServerApiKey",
defaultValue: "",
editorTitle: "Whisper server API key (optional)",
glassesEditTitle: "Edit Whisper server key",
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(none)"),
description: "Bearer token for the Whisper server, if it requires one.",
});
export const anthropicApiKeySetting = new ConfigSettingString({
id: "anthropic-api-key",
label: "Anthropic key",
+20 -1
View File
@@ -61,6 +61,11 @@ import {
phoneRotationSetting,
ringConnectionModeSetting,
sonioxApiKeySetting,
saveTranscribeAudioSetting,
whisperServerApiKeySetting,
whisperServerLanguageSetting,
whisperServerModelSetting,
whisperServerUrlSetting,
screenFadeSetting,
enumSettingMenuItem,
firmwareDebugFlagsSetting,
@@ -126,6 +131,7 @@ function settingsSections(): SettingsSection[] {
items: [
enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting),
...whisperServerMenuItems(),
asrModelMenuItem("moonshine"),
asrModelMenuItem("whisper-base-en"),
],
@@ -240,7 +246,7 @@ function settingsSections(): SettingsSection[] {
toggleSettingMenuItem(showBleBandwidthSetting)] };
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting)] };
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
if (deferred.has(section.label)) return { label: section.label, items: [{
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
description: `${section.label} integration has not been ported to iOS.`,
@@ -454,6 +460,19 @@ function asrModelStatusText(id: AsrModelId): string {
return "not downloaded";
}
/**
* The self-hosted Whisper server's settings. Always listed: the provider
* option stays disabled until the URL is set.
*/
function whisperServerMenuItems(): MenuItem[] {
return [
textSettingMenuItem(whisperServerUrlSetting),
textSettingMenuItem(whisperServerModelSetting),
textSettingMenuItem(whisperServerLanguageSetting),
textSettingMenuItem(whisperServerApiKeySetting),
];
}
/** Download/cancel/delete management for one on-device transcription model. */
function asrModelMenuItem(id: AsrModelId): MenuItem {
const def = ASR_MODELS[id];
+3 -2
View File
@@ -33,9 +33,10 @@ export type LayerActions = {
stopVoiceCapture: () => Promise<void> | void;
/**
* Start continuous capture on behalf of `client` (one id per window); shares
* the mic with push-to-talk, and runs while any client wants it.
* the mic with push-to-talk, and runs while any client wants it. recordAs
* names a WAV of the captured audio in Downloads (iOS, when enabled).
*/
startContinuousVoiceCapture: (client: string) => Promise<void> | void;
startContinuousVoiceCapture: (client: string, options?: { recordAs?: string }) => Promise<void> | void;
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;