diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt index 2b149f09..3c698774 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt @@ -25,18 +25,21 @@ internal object AndroidSpeechEngines { private const val FEATURE_DIM = 80 /** Sherpa-onnx configuration for the on-device model files in [modelDir]. */ - fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig { + fun recognizerConfig( + modelDir: File, kind: VoiceModelKind, + whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null), + ): OfflineRecognizerConfig { val modelConfig = OfflineModelConfig.builder() .setNumThreads(1) if (kind == VoiceModelKind.WHISPER) { modelConfig .setWhisper(OfflineWhisperModelConfig.builder() - .setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath) - .setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath) - .setLanguage("en") + .setEncoder(File(modelDir, whisper.encoder).absolutePath) + .setDecoder(File(modelDir, whisper.decoder).absolutePath) + .setLanguage(whisper.language) .setTask("transcribe") .build()) - .setTokens(File(modelDir, "base.en-tokens.txt").absolutePath) + .setTokens(File(modelDir, whisper.tokens).absolutePath) } else { modelConfig .setMoonshine(OfflineMoonshineModelConfig.builder() diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt index ad0531bb..8b18600e 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -32,17 +32,6 @@ class FaceclawVoiceController(context: Context) { "decoder_model_merged.ort", "tokens.txt" ) - // Second on-device model: sherpa-onnx's offline Whisper backend (base.en, - // int8-quantized -- see the model-choice note in asr-model.ts). Directory - // shared with the TS-side download flow, same convention as ASR_MODEL_DIR. - // Whisper re-encodes the whole buffer on every call, so the shared session - // skips live partials for it; see VoiceCaptureSession.processRecognizer. - private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8" - private val ASR_WHISPER_MODEL_FILES = arrayOf( - "base.en-encoder.int8.onnx", - "base.en-decoder.int8.onnx", - "base.en-tokens.txt" - ) // 50 ms chunks match the G2 packet cadence the rest of the pipeline // (endpointing, transcript pacing) is tuned for. private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20 @@ -84,6 +73,7 @@ class FaceclawVoiceController(context: Context) { private val mainHandler = Handler(Looper.getMainLooper()) @Volatile private var communicator: FaceclawBleCommunicator? = null + private var whisperModel = WhisperModelFiles.forLanguage(null) private val host = object : VoiceCaptureHost { override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } } @@ -122,7 +112,7 @@ class FaceclawVoiceController(context: Context) { override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber { val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing") - return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind)) + return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel)) } override fun speakerEmbedder(modelPath: String): SpeakerEmbedder { @@ -168,6 +158,11 @@ class FaceclawVoiceController(context: Context) { session.setOnboardModelKind(kind) } + /** Language for the next on-device Whisper capture; empty enables auto-detection. */ + fun setOnboardLanguage(language: String?) { + whisperModel = WhisperModelFiles.forLanguage(language) + } + /** * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the * speaker stops. Used by hands-free ("Hey Even") capture, which has no @@ -263,8 +258,8 @@ class FaceclawVoiceController(context: Context) { */ private fun findAsrModelDir(kind: VoiceModelKind): File? { val whisper = kind == VoiceModelKind.WHISPER - val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR - val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES + val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR + val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList() val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName) for (fileName in fileNames) { val file = File(modelDir, fileName) diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 5893a214..707e525f 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, getHeadUpPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, getHeadUpPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -487,6 +487,7 @@ class DashboardController { this.applyVerticalPositionIfChanged(); this.applyAppLayoutsIfChanged(); this.syncAssistantBridgeIfChanged(); + this.retargetContinuousCapture(); this.syncLockScreenSettingIfChanged(); // A Preview color or scaling change should show on the mirror at once, // not at the next composited frame (cheap: floor-limited, dedup'd on @@ -2216,14 +2217,14 @@ class DashboardController { private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { // A window that needs its own provider and language sets them for every // capture while it is open, so push-to-talk still shares its stream. - // Otherwise only Whisper server is told a language (its own setting). + // Otherwise use the shared transcription-language preference. const choice = this.transcriptionChoice(); const provider = choice?.provider ?? voiceProviderSetting.get(); return { communicator: communicator?.getNativeCommunicator() ?? null, usePhoneMic: communicator === null, provider, - language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "", + language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 89a3d934..4a8b8670 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, - openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting, + openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings' import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' @@ -422,6 +422,7 @@ export class IosPreviewController { const phone = readPhoneBatteryState() this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.offSettings = onAnySettingChanged(() => { + this.syncContinuousCapture() this.syncLockSetting() this.pushBrightness() this.relayout() @@ -1204,13 +1205,12 @@ export class IosPreviewController { /** * The transcription provider setting and the keys the cloud ones need. A * window that needs its own provider and language (Translate) sets them for - * every capture while it is open; otherwise only Whisper server is told a - * language (its own setting). + * every capture while it is open; otherwise use the shared language preference. */ private voiceProvider(): VoiceProviderSelection { const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription const provider = choice?.provider ?? voiceProviderSetting.get() - return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '', + return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts index 97fc7e3b..c21675fc 100644 --- a/app/native/asr-model.ts +++ b/app/native/asr-model.ts @@ -9,7 +9,7 @@ declare const java: any; * than bundled in the APK; FaceclawVoiceController reads the files from the * directory each model's `dirName` names. * - * Two models, same shape: + * Downloadable models, same shape: * - "moonshine": the original on-device option (three files). The model is * no longer bundled in the APK; it is fetched into the same filesDir * location that earlier releases copied the bundled files to, so @@ -29,12 +29,17 @@ declare const java: any; * used for Moonshine below. Every hash here was computed locally from a * freshly downloaded copy of the file, not copied from a doc page. * + * - "whisper-base": multilingual base, int8-quantized. Used for non-English + * speech and auto-detection; English-only downloads remain untouched. + * SHA-256/size values come from the maintainer's LFS metadata, with the + * vocabulary hash computed locally. URLs pin the source revision. + * * Mirrors the on-phone assistant model flow in llama.ts, except each model * here is multiple files rather than one; they download sequentially through * FaceclawModelDownloader (resume + pinned sha256 per file). */ -export type AsrModelId = "moonshine" | "whisper-base-en"; +export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base"; type AsrModelFile = { name: string; @@ -105,6 +110,30 @@ export const ASR_MODELS: Record = { ], totalBytes: 160626066, }, + + "whisper-base": { + label: "Whisper (multilingual, base)", + dirName: "sherpa-onnx-whisper-base-int8", + baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/", + files: [ + { + name: "base-encoder.int8.onnx", + sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11", + sizeBytes: 29120534, + }, + { + name: "base-decoder.int8.onnx", + sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d", + sizeBytes: 130672026, + }, + { + name: "base-tokens.txt", + sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", + sizeBytes: 816730, + }, + ], + totalBytes: 160609290, + }, }; export type AsrModelState = { @@ -129,6 +158,7 @@ function freshRuntime(): ModelRuntime { const runtimes: Record = { moonshine: freshRuntime(), "whisper-base-en": freshRuntime(), + "whisper-base": freshRuntime(), }; function modelDirPath(id: AsrModelId): string { diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts index 566cb1d7..51bde438 100644 --- a/app/native/cloud-stt-provider.ts +++ b/app/native/cloud-stt-provider.ts @@ -14,7 +14,7 @@ export type VoiceProviderSelection = { provider: VoiceProviderKind; /** * Spoken language the provider should expect (ISO 639-1); empty lets it - * detect the language. Cloud providers only. + * detect the language. Also used by on-device Whisper. */ language?: string; elevenLabsApiKey: string; @@ -77,7 +77,8 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection * to expect. A running capture is shared only by holders asking for the same. */ export function transcriptionKey(selection: Pick): string { - return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`; + const fallback = selection.provider === "onboard-whisper" ? "en" : ""; + return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}`; } /** Whether createCloudSttClient would return a cloud client for this selection. */ diff --git a/app/native/transcription-language.ts b/app/native/transcription-language.ts new file mode 100644 index 00000000..1b8ae0a7 --- /dev/null +++ b/app/native/transcription-language.ts @@ -0,0 +1,24 @@ +/** Languages supported by both Whisper backends, in picker order. */ +export const TRANSCRIPTION_LANGUAGES = { + en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian", + pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech", + da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi", + hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean", + no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak", + sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese", +} as const; + +export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES; + +/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */ +export function resolveTranscriptionLanguage( + choice: TranscriptionLanguage, provider: string, serverLanguage = "", +): string { + // Moonshine is English-only; Apple's on-device recognizer retains its own locale. + if (provider === "onboard") return ""; + if (choice === "default") { + if (provider === "onboard-whisper") return "en"; + return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : ""; + } + return choice === "auto" ? "" : choice; +} diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index d6a7e37b..4c845798 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -58,7 +58,7 @@ export type PushToTalkOptions = { whisperServerUrl?: string; whisperServerModel?: string; whisperServerApiKey?: string; - /** Spoken language for cloud providers (see VoiceProviderSelection.language). */ + /** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */ language?: string; saveRecording: boolean; /** @@ -396,6 +396,7 @@ export class FaceclawVoiceControlBridge { // Which on-device model to load; a no-op setter for every provider except // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); + this.controller?.setOnboardLanguage(options.language ?? "en"); const captureId = this.nextNativeCaptureId++; this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); }); this.controller?.start("onboard", captureId); diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index d29b9c98..59b422bf 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -18,6 +18,7 @@ import { } from "~/assistant/models"; import { isLocalModelReady } from "../native/local-llm"; import { normalizeWhisperServerUrl } from "../native/whisper-server-stt"; +import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language"; import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat"; import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; @@ -1201,9 +1202,26 @@ export const whisperServerModelSetting = new ConfigSettingString({ description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.", }); +export const voiceLanguageSetting = new ConfigSettingEnum({ + id: "voice-language", + label: "Transcription language", + storageKey: "voice.language", + defaultValue: "default", + values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])], + formatValue: (value) => value === "default" ? "Provider default" + : value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value], + description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " + + "Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " + + "On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.", +}); + +export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string { + return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get()); +} + export const whisperServerLanguageSetting = new ConfigSettingString({ id: "whisper-server-language", - label: "Whisper language", + label: "Whisper server language", storageKey: "voice.whisperServerLanguage", defaultValue: "", editorTitle: "Language code, e.g. en or sk (empty: auto)", @@ -1211,7 +1229,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({ formatValue: (value) => value || "Auto-detect", validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."), description: - "Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.", + "Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.", }); export const whisperServerApiKeySetting = new ConfigSettingString({ diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index a7e5e255..f689107a 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -89,6 +89,7 @@ import { toggleSettingMenuItem, useMicControlSetting, voiceProviderSetting, + voiceLanguageSetting, screenTimeoutSetting, wakeToLastAppSetting, wakeWordActionSetting, @@ -146,9 +147,11 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), + enumSettingMenuItem(voiceLanguageSetting), ...whisperServerMenuItems(), asrModelMenuItem("moonshine"), asrModelMenuItem("whisper-base-en"), + asrModelMenuItem("whisper-base"), ], }, { @@ -287,7 +290,7 @@ function settingsSections(): SettingsSection[] { toggleSettingMenuItem(showBleBandwidthSetting)] }; // No downloadable on-device models: iOS's on-device option is Apple's recognizer. if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), - enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; + enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; if (section.label === "Display") return { ...section, items: section.items.filter(item => item.label !== screenTimeoutSetting.label) }; return section; diff --git a/docs/validation-whisper-languages.md b/docs/validation-whisper-languages.md new file mode 100644 index 00000000..00b67ba1 --- /dev/null +++ b/docs/validation-whisper-languages.md @@ -0,0 +1,46 @@ +# Whisper transcription languages + +Settings > Voice > Transcription language selects French, English, 27 other +languages, or auto-detection. It applies to ordinary dictation and continuous +transcription using OpenAI, on-device Whisper, and the other cloud providers. +Translate retains its own spoken-language preference. Moonshine remains +English-only; Apple's on-device speech recognizer retains its own locale. + +The initial **Provider default** preserves existing installations: English for +on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved +Whisper server language for self-hosted Whisper. Selecting another shared +language overrides the server language without deleting it. Returning to Provider +default restores it. Settings persist through the existing settings store. + +For French or auto-detection with **On-device (Whisper)**, download +**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB). +English uses the existing English-only base model; neither model is deleted or +downloaded automatically. A missing model uses the existing “Voice model not +downloaded” status. Language changes restart active continuous transcription; +other captures use the new language when started. + +## Software validation + +- JavaScript regression tests exercise stored settings/reload, compatible + defaults, French and auto-detection, Translate overrides in Android/iOS capture + options, the OpenAI WebSocket configuration, and local bridge restart/sharing. +- Kotlin tests exercise English/French/other/automatic model selection and file + names. Android release compilation checks the actual sherpa-onnx bridge. +- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and + the existing resumable downloader. Encoder/decoder hashes and sizes come from + repository LFS metadata; the vocabulary hash was computed from its contents. + +## Device validation still required + +With both phone microphone preview and glasses microphone, check French dictation +(including accents), auto-detection, English rollback, continuous capture while +changing language, cancellation/resumption of model download, and persistence +after restarting the app. Compare OpenAI and offline Whisper on the same phrases. +No paid API call, microphone recording, or firmware update is needed for the +software checks above; recognition accuracy and latency require device testing. + +## References + +- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription) +- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc) +- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt new file mode 100644 index 00000000..9ae3817f --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt @@ -0,0 +1,18 @@ +package com.faceclaw.app + +/** The English-only files remain usable after upgrading; other languages need multilingual base. */ +data class WhisperModelFiles(val language: String) { + val englishOnly: Boolean get() = language == "en" + val dirName: String get() = if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8" + private val prefix: String get() = if (englishOnly) "base.en" else "base" + val encoder: String get() = "$prefix-encoder.int8.onnx" + val decoder: String get() = "$prefix-decoder.int8.onnx" + val tokens: String get() = "$prefix-tokens.txt" + val files: List get() = listOf(encoder, decoder, tokens) + + companion object { + /** Null preserves the old English default; an empty code enables automatic detection. */ + fun forLanguage(language: String?): WhisperModelFiles = + WhisperModelFiles(language?.trim()?.lowercase() ?: "en") + } +} diff --git a/tests/assistant-settings-visibility.test.cjs b/tests/assistant-settings-visibility.test.cjs index 43e96713..75a98697 100644 --- a/tests/assistant-settings-visibility.test.cjs +++ b/tests/assistant-settings-visibility.test.cjs @@ -55,6 +55,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') { ASR_MODELS: { moonshine: { label: 'Moonshine', sizeBytes: 1 }, 'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 }, + 'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 }, }, asrModelState: () => ({}), cancelAsrModelDownload() {}, deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {}, diff --git a/tests/brightness-curve.test.cjs b/tests/brightness-curve.test.cjs index 0d4c2aa9..efe5e9db 100644 --- a/tests/brightness-curve.test.cjs +++ b/tests/brightness-curve.test.cjs @@ -26,6 +26,7 @@ function settingsFixture(store = new Map()) { }; if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js'); if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js'); + if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js'); return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }; }}; vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), { diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs index 06abfb5f..5ca17214 100644 --- a/tests/cloud-stt.test.cjs +++ b/tests/cloud-stt.test.cjs @@ -207,6 +207,8 @@ test('provider selection falls back to on-device without a key and connects the // A running capture is shared only on the same provider and language. assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja'); assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' })); + assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' })); + assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' })); const statuses = []; const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} }; const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' }; @@ -225,13 +227,13 @@ test('provider selection falls back to on-device without a key and connects the }); test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => { - for (const language of ['ja', '']) { + for (const language of ['fr', 'ja', '']) { const elevenlabs = harness('elevenlabs', true, { language }); assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null); const soniox = harness('soniox', true, { language }); soniox.open(); - assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined); + assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined); const openai = harness('openai', true, { language }); openai.open(); diff --git a/tests/display-layout-setting.test.cjs b/tests/display-layout-setting.test.cjs index 84332242..1260cd8f 100644 --- a/tests/display-layout-setting.test.cjs +++ b/tests/display-layout-setting.test.cjs @@ -12,6 +12,7 @@ function settingsFixture(store = new Map()) { }; if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js'); if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js'); + if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js'); return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }; }}; vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), { diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index 37cf988f..d087c394 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -6,6 +6,7 @@ const ts = require('typescript'); const path = require('node:path'); // The provider settings ios-preview-controller hands the voice bridge. const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' }, + getVoiceTranscriptionLanguage: () => '', openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' }, whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' }, whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } }; @@ -168,6 +169,7 @@ test('settings-driven repaint runs after font cache invalidation, regardless of let cachedFont = 'Light'; const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true }, require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) } + : id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {}) : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }, setTimeout: fn => tasks.push(fn), }); @@ -384,7 +386,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec recordTo: name => { bridge.recording = name; }, setRawSession: session => { bridge.rawSession = session; } }; const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' }, - whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' }, + whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en', saveTranscribeAudioSetting: { get: () => true } }; const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt new file mode 100644 index 00000000..3cc37184 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt @@ -0,0 +1,35 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class WhisperModelFilesTest { + @Test fun existingEnglishFilesRemainUsable() { + for (language in listOf(null, "en", " EN ")) { + val model = WhisperModelFiles.forLanguage(language) + assertTrue(model.englishOnly) + assertEquals("en", model.language) + assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName) + assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files) + } + } + + @Test fun frenchAndOtherLanguagesUseMultilingualFiles() { + for (language in listOf("fr", "de", "ja", " FR ")) { + val model = WhisperModelFiles.forLanguage(language) + assertFalse(model.englishOnly) + assertEquals(language.trim().lowercase(), model.language) + assertEquals("sherpa-onnx-whisper-base-int8", model.dirName) + assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files) + } + } + + @Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() { + val model = WhisperModelFiles.forLanguage("") + assertFalse(model.englishOnly) + assertEquals("", model.language) + assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files) + } +} diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 5c5fd912..2469bb34 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -28,6 +28,7 @@ "../app/ui/menu-core.ts", "../app/phone-ui/phone-gestures.ts", "../app/native/java-direct-buffer.ts", + "../app/native/transcription-language.ts", "../app/native/notification-types.ts", "../app/g2/firmware-compat.ts", "../app/g2/glasses-hardware-identity.ts", diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs index c049d8b2..565151e0 100644 --- a/tests/voice-finalization.test.cjs +++ b/tests/voice-finalization.test.cjs @@ -163,7 +163,8 @@ function androidBridge() { starts: 0, setListener(listener) { this.listener = listener; }, setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {}, - setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {}, + setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, + setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; }, start(_mode, id) { this.captureId = id; this.active = true; this.starts++; }, stop() { this.active = false; }, isCapturing() { return this.active; }, @@ -171,7 +172,8 @@ function androidBridge() { const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', { '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, './speech-pause': { SpeechPauseDetector: class { reset() {} } }, - './cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider }, + './cloud-stt-provider': { createCloudSttClient: () => null, + transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` }, }, { global: { isAndroid: true }, com: { faceclaw: { app: { FaceclawVoiceController: function () { return controller; }, FaceclawVoiceControllerListener: function (listener) { return listener; }, @@ -179,6 +181,26 @@ function androidBridge() { return { bridge: new FaceclawVoiceControlBridge(), controller }; } +test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => { + const { bridge, controller } = androidBridge(); + bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' }); + assert.equal(controller.modelKind, 'whisper'); + assert.equal(controller.language, 'fr'); + bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' }); + assert.equal(controller.starts, 1); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' }); + assert.equal(controller.language, ''); + assert.equal(controller.starts, 2); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper' }); + // An omitted language preserves the native bridge's original English behavior. + assert.equal(controller.language, 'en'); + assert.equal(controller.starts, 3); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' }); + assert.equal(controller.language, 'en'); + assert.equal(controller.starts, 3); + bridge.stop(); +}); + test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => { const { bridge, controller } = androidBridge(); const options = { provider: 'onboard-whisper' }; diff --git a/tests/whisper-language.test.cjs b/tests/whisper-language.test.cjs new file mode 100644 index 00000000..6d2cc567 --- /dev/null +++ b/tests/whisper-language.test.cjs @@ -0,0 +1,131 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function load(file, requireModule = () => ({}), globals = {}) { + const exports = {}; + vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals }); + return exports; +} +const languages = load('app/native/transcription-language.ts'); +function settings(values = new Map()) { + return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages + : name.includes('settings-store') ? { + getStringSetting: (key, fallback) => values.get(key) ?? fallback, + setStringSetting: (key, value) => values.set(key, value), + getBooleanSetting: (_key, fallback) => fallback, + onSettingsStoreChanged() {}, + } : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }); +} + +test('language defaults preserve local English, cloud detection and an existing server preference', () => { + const values = new Map([['voice.whisperServerLanguage', 'sk']]); + const s = settings(values); + assert.equal(s.voiceLanguageSetting.get(), 'default'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en'); + assert.equal(s.getVoiceTranscriptionLanguage('whisper'), ''); + assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk'); + assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]); +}); + +test('French and auto-detection persist across restart without overwriting server exceptions', () => { + const values = new Map([['voice.whisperServerLanguage', 'sk']]); + let s = settings(values); + s.voiceLanguageSetting.set('fr'); + s = settings(values); + assert.equal(s.voiceLanguageSetting.displayValue(), 'French'); + for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) { + assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr'); + } + assert.equal(s.getVoiceTranscriptionLanguage('onboard'), ''); + s.voiceLanguageSetting.set('auto'); + s = settings(values); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), ''); + assert.equal(s.getVoiceTranscriptionLanguage('whisper'), ''); + s.voiceLanguageSetting.set('default'); + assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en'); +}); + +test('corrupt language preferences revert to the compatible provider defaults', () => { + const s = settings(new Map([['voice.language', 'invalid']])); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en'); + assert.equal(s.getVoiceTranscriptionLanguage('whisper'), ''); +}); + +test('Android and iOS capture options use the shared language and preserve Translate overrides', () => { + const s = settings(); + s.voiceLanguageSetting.set('fr'); + s.voiceProviderSetting.set('whisper'); + for (const [file, name, method] of [ + ['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'], + ['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'], + ]) { + const host = { continuousClients: new Map() }; + host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription; + // Run each real capture-options method with only its native dependencies stubbed. + const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); + const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name); + const member = declaration.members.find(n => n.name?.getText(tree) === method); + const capture = loadBody(member.body.getText(tree), s, host); + assert.equal(capture().language, 'fr'); + host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } }); + assert.equal(capture().provider, 'soniox'); + assert.equal(capture().language, 'ja'); + host.continuousClients.clear(); + assert.equal(capture().language, 'fr'); + } +}); + +function loadBody(body, s, host) { + const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } }; + return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null); +} + +test('multilingual downloads and deletion leave an existing English model intact', () => { + const files = new Map(), downloads = []; + const asr = load('app/native/asr-model.ts', () => ({ + Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } }, + }), { + java: { io: { File: class { + constructor(name) { this.name = name; } + exists() { return files.has(this.name); } + length() { return files.get(this.name) ?? 0; } + delete() { return files.delete(this.name); } + } } }, + com: { faceclaw: { app: { + FaceclawModelDownloaderListener: function(listener) { return listener; }, + FaceclawModelDownloader: class { + constructor(url, destination, hash, size, listener) { + Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this); + } + start() {} + cancel() {} + }, + } } }, + }); + const english = asr.ASR_MODELS['whisper-base-en']; + for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes); + assert.equal(asr.isAsrModelReady('whisper-base-en'), true); + assert.equal(asr.isAsrModelReady('whisper-base'), false); + asr.startAsrModelDownload('whisper-base'); + assert.equal(asr.asrModelState('whisper-base').status, 'downloading'); + for (let index = 0; index < 3; index++) { + const download = downloads[index]; + assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/); + assert.match(download.hash, /^[a-f0-9]{64}$/); + files.set(download.destination, download.size); + download.listener.onDone(); + } + assert.equal(downloads.length, 3); + assert.equal(asr.isAsrModelReady('whisper-base'), true); + asr.deleteAsrModel('whisper-base'); + assert.equal(asr.isAsrModelReady('whisper-base'), false); + assert.equal(asr.isAsrModelReady('whisper-base-en'), true); +});