feat: select languages for cloud and local Whisper transcription

This commit is contained in:
Blendman974 committed 2026-11-06 04:22:48 +01:00
1 parent 6df52021c8
commit 0b9e4cb367
21 files changed
+375 -39

No files matched your search

+4 -3
View File
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
getInstalledEvenHubAppById,
@@ -483,6 +483,7 @@ class DashboardController {
this.applyVerticalPositionIfChanged();
this.applyAppLayoutsIfChanged();
this.syncAssistantBridgeIfChanged();
this.retargetContinuousCapture();
this.syncLockScreenSettingIfChanged();
// A Preview color or scaling change should show on the mirror at once,
// not at the next composited frame (cheap: floor-limited, dedup'd on
@@ -2197,14 +2198,14 @@ class DashboardController {
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
// A window that needs its own provider and language sets them for every
// capture while it is open, so push-to-talk still shares its stream.
// Otherwise only Whisper server is told a language (its own setting).
// Otherwise use the shared transcription-language preference.
const choice = this.transcriptionChoice();
const provider = choice?.provider ?? voiceProviderSetting.get();
return {
communicator: communicator?.getNativeCommunicator() ?? null,
usePhoneMic: communicator === null,
provider,
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(),
sonioxApiKey: sonioxApiKeySetting.get(),
+4 -4
View File
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
@@ -416,6 +416,7 @@ export class IosPreviewController {
const phone = readPhoneBatteryState()
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
this.offSettings = onAnySettingChanged(() => {
this.syncContinuousCapture()
this.syncLockSetting()
this.pushBrightness()
this.relayout()
@@ -1198,13 +1199,12 @@ export class IosPreviewController {
/**
* The transcription provider setting and the keys the cloud ones need. A
* window that needs its own provider and language (Translate) sets them for
* every capture while it is open; otherwise only Whisper server is told a
* language (its own setting).
* every capture while it is open; otherwise use the shared language preference.
*/
private voiceProvider(): VoiceProviderSelection {
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
const provider = choice?.provider ?? voiceProviderSetting.get()
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get() }
+32 -2
View File
@@ -9,7 +9,7 @@ declare const java: any;
* than bundled in the APK; FaceclawVoiceController reads the files from the
* directory each model's `dirName` names.
*
* Two models, same shape:
* Downloadable models, same shape:
* - "moonshine": the original on-device option (three files). The model is
* no longer bundled in the APK; it is fetched into the same filesDir
* location that earlier releases copied the bundled files to, so
@@ -29,12 +29,17 @@ declare const java: any;
* used for Moonshine below. Every hash here was computed locally from a
* freshly downloaded copy of the file, not copied from a doc page.
*
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
* speech and auto-detection; English-only downloads remain untouched.
* SHA-256/size values come from the maintainer's LFS metadata, with the
* vocabulary hash computed locally. URLs pin the source revision.
*
* Mirrors the on-phone assistant model flow in llama.ts, except each model
* here is multiple files rather than one; they download sequentially through
* FaceclawModelDownloader (resume + pinned sha256 per file).
*/
export type AsrModelId = "moonshine" | "whisper-base-en";
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base";
type AsrModelFile = {
name: string;
@@ -105,6 +110,30 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
],
totalBytes: 160626066,
},
"whisper-base": {
label: "Whisper (multilingual, base)",
dirName: "sherpa-onnx-whisper-base-int8",
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
files: [
{
name: "base-encoder.int8.onnx",
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
sizeBytes: 29120534,
},
{
name: "base-decoder.int8.onnx",
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
sizeBytes: 130672026,
},
{
name: "base-tokens.txt",
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
sizeBytes: 816730,
},
],
totalBytes: 160609290,
},
};
export type AsrModelState = {
@@ -129,6 +158,7 @@ function freshRuntime(): ModelRuntime {
const runtimes: Record<AsrModelId, ModelRuntime> = {
moonshine: freshRuntime(),
"whisper-base-en": freshRuntime(),
"whisper-base": freshRuntime(),
};
function modelDirPath(id: AsrModelId): string {
+3 -2
View File
@@ -14,7 +14,7 @@ export type VoiceProviderSelection = {
provider: VoiceProviderKind;
/**
* Spoken language the provider should expect (ISO 639-1); empty lets it
* detect the language. Cloud providers only.
* detect the language. Also used by on-device Whisper.
*/
language?: string;
elevenLabsApiKey: string;
@@ -77,7 +77,8 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
* to expect. A running capture is shared only by holders asking for the same.
*/
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}`;
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
+24
View File
@@ -0,0 +1,24 @@
/** Languages supported by both Whisper backends, in picker order. */
export const TRANSCRIPTION_LANGUAGES = {
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
} as const;
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
export function resolveTranscriptionLanguage(
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
): string {
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
if (provider === "onboard") return "";
if (choice === "default") {
if (provider === "onboard-whisper") return "en";
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
}
return choice === "auto" ? "" : choice;
}
+2 -1
View File
@@ -58,7 +58,7 @@ export type PushToTalkOptions = {
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerApiKey?: string;
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */
language?: string;
saveRecording: boolean;
/**
@@ -396,6 +396,7 @@ export class FaceclawVoiceControlBridge {
// Which on-device model to load; a no-op setter for every provider except
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
this.controller?.setOnboardLanguage(options.language ?? "en");
const captureId = this.nextNativeCaptureId++;
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
this.controller?.start("onboard", captureId);
+20 -2
View File
@@ -18,6 +18,7 @@ import {
} from "~/assistant/models";
import { isLocalModelReady } from "../native/local-llm";
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language";
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
@@ -1136,9 +1137,26 @@ export const whisperServerModelSetting = new ConfigSettingString({
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
});
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
id: "voice-language",
label: "Transcription language",
storageKey: "voice.language",
defaultValue: "default",
values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])],
formatValue: (value) => value === "default" ? "Provider default"
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.",
});
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get());
}
export const whisperServerLanguageSetting = new ConfigSettingString({
id: "whisper-server-language",
label: "Whisper language",
label: "Whisper server language",
storageKey: "voice.whisperServerLanguage",
defaultValue: "",
editorTitle: "Language code, e.g. en or sk (empty: auto)",
@@ -1146,7 +1164,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({
formatValue: (value) => value || "Auto-detect",
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
description:
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
"Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.",
});
export const whisperServerApiKeySetting = new ConfigSettingString({
+4 -1
View File
@@ -87,6 +87,7 @@ import {
toggleSettingMenuItem,
useMicControlSetting,
voiceProviderSetting,
voiceLanguageSetting,
screenTimeoutSetting,
wakeToLastAppSetting,
wakeWordActionSetting,
@@ -143,9 +144,11 @@ function settingsSections(): SettingsSection[] {
items: [
enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting),
enumSettingMenuItem(voiceLanguageSetting),
...whisperServerMenuItems(),
asrModelMenuItem("moonshine"),
asrModelMenuItem("whisper-base-en"),
asrModelMenuItem("whisper-base"),
],
},
{
@@ -284,7 +287,7 @@ function settingsSections(): SettingsSection[] {
toggleSettingMenuItem(showBleBandwidthSetting)] };
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
item.label !== screenTimeoutSetting.label) };
return section;