forked from Blendman974/faceclaw
Voice: self-hosted Whisper server provider; Transcribe on iOS
- "Whisper server (self-hosted)" transcription provider for OpenAI-compatible Whisper servers (vLLM, speaches, LocalAI) over plain HTTP batch requests, with URL, model, language and key settings (Voice). Pause-ended spans are one request each, finalized in audio order, with provisional text while a span is open. See notes/whisper-server.md. - Transcribe (and other continuous capture) works on iOS with a cloud or Whisper-server provider; push-to-talk shares a running capture. - Transcribe saves each session's audio as transcribe-<time>.wav in Downloads (setting "Save Transcribe audio", on by default), written as it is captured so a crash leaves a playable file; a saved transcript uses the same stamp. - Calculator stops continuous listening when its window closes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ecb6c7f465
commit
e2caf28392
24 files changed
+1065
-43
No files matched your search
@@ -17,6 +17,7 @@ import {
|
||||
type AssistantModel,
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/llama";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
import { ANIMATION_SPEEDS, animationSpeedLabel, scaleAnimationDuration, type AnimationSpeed } from "./animation-speed";
|
||||
import { MENU_ANIMATION_KEY } from "./menu-animation-pref";
|
||||
@@ -814,7 +815,8 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
|
||||
// runs -- see the same note in native/voice-control.ts. The "whisper" value
|
||||
// keeps its name (it's a persisted setting on real installs) but its label
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper-server" is a self-hosted OpenAI-compatible Whisper (e.g. vLLM).
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
|
||||
// On iOS "onboard" is Apple's on-device recognizer, and there is no
|
||||
// "onboard-whisper" (the sherpa-onnx models are Android-only).
|
||||
@@ -824,6 +826,7 @@ const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
soniox: "Soniox",
|
||||
"whisper-server": "Whisper server (self-hosted)",
|
||||
};
|
||||
|
||||
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
@@ -831,17 +834,20 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
label: "Transcription Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
values: global.isIOS
|
||||
? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"]
|
||||
: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
if (value === "whisper") return openAiApiKeySetting.get().trim().length === 0;
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
if (value === "whisper-server") return !normalizeWhisperServerUrl(whisperServerUrlSetting.get());
|
||||
return false;
|
||||
},
|
||||
description: global.isIOS
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
@@ -860,6 +866,14 @@ export const wakeWordActionSetting = new ConfigSettingEnum<WakeWordAction>({
|
||||
description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.",
|
||||
});
|
||||
|
||||
export const saveTranscribeAudioSetting = new ConfigSettingBoolean({
|
||||
id: "save-transcribe-audio",
|
||||
label: "Save Transcribe audio",
|
||||
storageKey: "voice.saveTranscribeAudio",
|
||||
defaultValue: true,
|
||||
description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.",
|
||||
});
|
||||
|
||||
export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
|
||||
id: "save-voice-recordings",
|
||||
label: "Save voice recordings",
|
||||
@@ -968,6 +982,54 @@ export const sonioxApiKeySetting = new ConfigSettingString({
|
||||
description: "Soniox API key, used when Soniox is the transcription provider.",
|
||||
});
|
||||
|
||||
export const whisperServerUrlSetting = new ConfigSettingString({
|
||||
id: "whisper-server-url",
|
||||
label: "Whisper server URL",
|
||||
storageKey: "voice.whisperServerUrl",
|
||||
defaultValue: "",
|
||||
editorTitle: "Whisper server URL (e.g. http://100.x.y.z:8004/v1)",
|
||||
glassesEditTitle: "Edit Whisper server URL",
|
||||
formatValue: (value) => value || "(not set)",
|
||||
validate: (value) => (!value.trim() || normalizeWhisperServerUrl(value) ? null : "Enter an http:// or https:// address."),
|
||||
description:
|
||||
"API root of an OpenAI-compatible Whisper server (vLLM, speaches, LocalAI...), used when Whisper server is the transcription provider. On iOS a plain-http address must also be allowed by the app's App Transport Security exceptions.",
|
||||
});
|
||||
|
||||
export const whisperServerModelSetting = new ConfigSettingString({
|
||||
id: "whisper-server-model",
|
||||
label: "Whisper server model",
|
||||
storageKey: "voice.whisperServerModel",
|
||||
defaultValue: "",
|
||||
editorTitle: "Whisper model id (empty: server's first)",
|
||||
glassesEditTitle: "Edit Whisper model",
|
||||
formatValue: (value) => value || "(server default)",
|
||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||
});
|
||||
|
||||
export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
id: "whisper-server-language",
|
||||
label: "Whisper language",
|
||||
storageKey: "voice.whisperServerLanguage",
|
||||
defaultValue: "",
|
||||
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
||||
glassesEditTitle: "Edit Whisper language",
|
||||
formatValue: (value) => value || "Auto-detect",
|
||||
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
||||
description:
|
||||
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
|
||||
});
|
||||
|
||||
export const whisperServerApiKeySetting = new ConfigSettingString({
|
||||
id: "whisper-server-api-key",
|
||||
label: "Whisper server key",
|
||||
storageKey: "voice.whisperServerApiKey",
|
||||
defaultValue: "",
|
||||
editorTitle: "Whisper server API key (optional)",
|
||||
glassesEditTitle: "Edit Whisper server key",
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(none)"),
|
||||
description: "Bearer token for the Whisper server, if it requires one.",
|
||||
});
|
||||
|
||||
export const anthropicApiKeySetting = new ConfigSettingString({
|
||||
id: "anthropic-api-key",
|
||||
label: "Anthropic key",
|
||||
|
||||
@@ -61,6 +61,11 @@ import {
|
||||
phoneRotationSetting,
|
||||
ringConnectionModeSetting,
|
||||
sonioxApiKeySetting,
|
||||
saveTranscribeAudioSetting,
|
||||
whisperServerApiKeySetting,
|
||||
whisperServerLanguageSetting,
|
||||
whisperServerModelSetting,
|
||||
whisperServerUrlSetting,
|
||||
screenFadeSetting,
|
||||
enumSettingMenuItem,
|
||||
firmwareDebugFlagsSetting,
|
||||
@@ -124,6 +129,7 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
...whisperServerMenuItems(),
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
],
|
||||
@@ -237,7 +243,7 @@ function settingsSections(): SettingsSection[] {
|
||||
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting)] };
|
||||
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
if (deferred.has(section.label)) return { label: section.label, items: [{
|
||||
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
|
||||
description: `${section.label} integration has not been ported to iOS.`,
|
||||
@@ -451,6 +457,19 @@ function asrModelStatusText(id: AsrModelId): string {
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/**
|
||||
* The self-hosted Whisper server's settings. Always listed: the provider
|
||||
* option stays disabled until the URL is set.
|
||||
*/
|
||||
function whisperServerMenuItems(): MenuItem[] {
|
||||
return [
|
||||
textSettingMenuItem(whisperServerUrlSetting),
|
||||
textSettingMenuItem(whisperServerModelSetting),
|
||||
textSettingMenuItem(whisperServerLanguageSetting),
|
||||
textSettingMenuItem(whisperServerApiKeySetting),
|
||||
];
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for one on-device transcription model. */
|
||||
function asrModelMenuItem(id: AsrModelId): MenuItem {
|
||||
const def = ASR_MODELS[id];
|
||||
|
||||
+5
-2
@@ -31,8 +31,11 @@ export type LayerActions = {
|
||||
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
|
||||
/** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
||||
startContinuousVoiceCapture: () => Promise<void> | void;
|
||||
/**
|
||||
* Start continuous capture (Transcribe); shares the mic with push-to-talk.
|
||||
* recordAs names a WAV of the captured audio in Downloads (iOS, when enabled).
|
||||
*/
|
||||
startContinuousVoiceCapture: (options?: { recordAs?: string }) => Promise<void> | void;
|
||||
stopContinuousVoiceCapture: () => Promise<void> | void;
|
||||
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
|
||||
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
|
||||
|
||||
Reference in new issue
Block a user