diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt index 2b149f09..3c698774 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt @@ -25,18 +25,21 @@ internal object AndroidSpeechEngines { private const val FEATURE_DIM = 80 /** Sherpa-onnx configuration for the on-device model files in [modelDir]. */ - fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig { + fun recognizerConfig( + modelDir: File, kind: VoiceModelKind, + whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null), + ): OfflineRecognizerConfig { val modelConfig = OfflineModelConfig.builder() .setNumThreads(1) if (kind == VoiceModelKind.WHISPER) { modelConfig .setWhisper(OfflineWhisperModelConfig.builder() - .setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath) - .setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath) - .setLanguage("en") + .setEncoder(File(modelDir, whisper.encoder).absolutePath) + .setDecoder(File(modelDir, whisper.decoder).absolutePath) + .setLanguage(whisper.language) .setTask("transcribe") .build()) - .setTokens(File(modelDir, "base.en-tokens.txt").absolutePath) + .setTokens(File(modelDir, whisper.tokens).absolutePath) } else { modelConfig .setMoonshine(OfflineMoonshineModelConfig.builder() diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt index ad0531bb..a35c8daa 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -32,17 +32,6 @@ class FaceclawVoiceController(context: Context) { "decoder_model_merged.ort", "tokens.txt" ) - // Second on-device model: sherpa-onnx's offline Whisper backend (base.en, - // int8-quantized -- see the model-choice note in asr-model.ts). Directory - // shared with the TS-side download flow, same convention as ASR_MODEL_DIR. - // Whisper re-encodes the whole buffer on every call, so the shared session - // skips live partials for it; see VoiceCaptureSession.processRecognizer. - private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8" - private val ASR_WHISPER_MODEL_FILES = arrayOf( - "base.en-encoder.int8.onnx", - "base.en-decoder.int8.onnx", - "base.en-tokens.txt" - ) // 50 ms chunks match the G2 packet cadence the rest of the pipeline // (endpointing, transcript pacing) is tuned for. private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20 @@ -84,6 +73,8 @@ class FaceclawVoiceController(context: Context) { private val mainHandler = Handler(Looper.getMainLooper()) @Volatile private var communicator: FaceclawBleCommunicator? = null + private var whisperModel = WhisperModelFiles.forLanguage(null) + private var frenchWhisper = false private val host = object : VoiceCaptureHost { override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } } @@ -122,7 +113,7 @@ class FaceclawVoiceController(context: Context) { override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber { val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing") - return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind)) + return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel)) } override fun speakerEmbedder(modelPath: String): SpeakerEmbedder { @@ -168,6 +159,17 @@ class FaceclawVoiceController(context: Context) { session.setOnboardModelKind(kind) } + /** Select the dedicated French fine-tune before setting the capture language. */ + fun setFrenchWhisper(enabled: Boolean) { + frenchWhisper = enabled + whisperModel = WhisperModelFiles.forSelection(whisperModel.language, enabled) + } + + /** Language for the next on-device Whisper capture; empty enables auto-detection. */ + fun setOnboardLanguage(language: String?) { + whisperModel = WhisperModelFiles.forSelection(language, frenchWhisper) + } + /** * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the * speaker stops. Used by hands-free ("Hey Even") capture, which has no @@ -263,8 +265,8 @@ class FaceclawVoiceController(context: Context) { */ private fun findAsrModelDir(kind: VoiceModelKind): File? { val whisper = kind == VoiceModelKind.WHISPER - val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR - val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES + val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR + val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList() val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName) for (fileName in fileNames) { val file = File(modelDir, fileName) diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 3666f299..7fb8bf7a 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -483,6 +483,7 @@ class DashboardController { this.applyVerticalPositionIfChanged(); this.applyAppLayoutsIfChanged(); this.syncAssistantBridgeIfChanged(); + this.retargetContinuousCapture(); this.syncLockScreenSettingIfChanged(); // A Preview color or scaling change should show on the mirror at once, // not at the next composited frame (cheap: floor-limited, dedup'd on @@ -2197,16 +2198,16 @@ class DashboardController { private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { // A window that needs its own provider and language sets them for every // capture while it is open, so push-to-talk still shares its stream. - // Otherwise only Whisper server is told a language (its own setting). + // Otherwise use the shared transcription-language preference. const choice = this.transcriptionChoice(); const provider = choice?.provider ?? voiceProviderSetting.get(); return { communicator: communicator?.getNativeCommunicator() ?? null, usePhoneMic: communicator === null, provider, - language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "", + language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), elevenLabsApiKey: elevenLabsApiKeySetting.get(), - openAiApiKey: openAiApiKeySetting.get(), + openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 247d8f1d..8cb44b26 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, - openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting, + openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings' import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' @@ -416,6 +416,7 @@ export class IosPreviewController { const phone = readPhoneBatteryState() this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.offSettings = onAnySettingChanged(() => { + this.syncContinuousCapture() this.syncLockSetting() this.pushBrightness() this.relayout() @@ -1198,14 +1199,13 @@ export class IosPreviewController { /** * The transcription provider setting and the keys the cloud ones need. A * window that needs its own provider and language (Translate) sets them for - * every capture while it is open; otherwise only Whisper server is told a - * language (its own setting). + * every capture while it is open; otherwise use the shared language preference. */ private voiceProvider(): VoiceProviderSelection { const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription const provider = choice?.provider ?? voiceProviderSetting.get() - return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '', - elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), + elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } } diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts index 97fc7e3b..9599d0a1 100644 --- a/app/native/asr-model.ts +++ b/app/native/asr-model.ts @@ -9,7 +9,7 @@ declare const java: any; * than bundled in the APK; FaceclawVoiceController reads the files from the * directory each model's `dirName` names. * - * Two models, same shape: + * Downloadable models, same shape: * - "moonshine": the original on-device option (three files). The model is * no longer bundled in the APK; it is fetched into the same filesDir * location that earlier releases copied the bundled files to, so @@ -29,12 +29,20 @@ declare const java: any; * used for Moonshine below. Every hash here was computed locally from a * freshly downloaded copy of the file, not copied from a doc page. * + * - "whisper-base": multilingual base, int8-quantized. Used for non-English + * speech and auto-detection; English-only downloads remain untouched. + * SHA-256/size values come from the maintainer's LFS metadata, with the + * vocabulary hash computed locally. URLs pin the source revision. + * + * - "whisper-small-fr": a dedicated French fine-tune, exported and quantized + * with sherpa-onnx 1.13.0. Versioned release assets include its Apache license. + * * Mirrors the on-phone assistant model flow in llama.ts, except each model * here is multiple files rather than one; they download sequentially through * FaceclawModelDownloader (resume + pinned sha256 per file). */ -export type AsrModelId = "moonshine" | "whisper-base-en"; +export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base" | "whisper-small-fr"; type AsrModelFile = { name: string; @@ -105,6 +113,43 @@ export const ASR_MODELS: Record = { ], totalBytes: 160626066, }, + + "whisper-base": { + label: "Whisper (multilingual, base)", + dirName: "sherpa-onnx-whisper-base-int8", + baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/", + files: [ + { + name: "base-encoder.int8.onnx", + sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11", + sizeBytes: 29120534, + }, + { + name: "base-decoder.int8.onnx", + sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d", + sizeBytes: 130672026, + }, + { + name: "base-tokens.txt", + sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", + sizeBytes: 816730, + }, + ], + totalBytes: 160609290, + }, + "whisper-small-fr": { + label: "Whisper (French, small)", + dirName: "sherpa-onnx-whisper-small-fr-int8-v1", + // Pinned export of bofenghuang/whisper-small-cv11-french; see docs/models/whisper-small-fr. + baseUrl: "https://git.undefined-dev.fr/api/packages/Blendman974/generic/faceclaw-asr/whisper-small-fr-v1/", + files: [ + {"name": "small-fr-encoder.int8.onnx", "sha256": "ec34fc42182f37727ad982d254b18a6001f5dcccebc5a36060031995f1782d5d", "sizeBytes": 112413414}, + {"name": "small-fr-decoder.int8.onnx", "sha256": "b554e35dc8c593b550a82cd01ebb00c1347e8c2b089b780b4c4fe16450cd7dc0", "sizeBytes": 262200764}, + {"name": "small-fr-tokens.txt", "sha256": "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", "sizeBytes": 816730}, + {"name": "LICENSE.txt", "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", "sizeBytes": 11358}, + ], + totalBytes: 375442266, + }, }; export type AsrModelState = { @@ -129,6 +174,8 @@ function freshRuntime(): ModelRuntime { const runtimes: Record = { moonshine: freshRuntime(), "whisper-base-en": freshRuntime(), + "whisper-base": freshRuntime(), + "whisper-small-fr": freshRuntime(), }; function modelDirPath(id: AsrModelId): string { diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts index 566cb1d7..5309a2e2 100644 --- a/app/native/cloud-stt-provider.ts +++ b/app/native/cloud-stt-provider.ts @@ -1,6 +1,6 @@ import type { CloudSttClient, CloudSttOptions } from "./cloud-stt"; import { ElevenLabsSttClient } from "./elevenlabs-stt"; -import { OpenAiRealtimeSttClient } from "./openai-stt"; +import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt"; import { ReconnectingSttClient } from "./reconnecting-stt"; import { SonioxSttClient } from "./soniox-stt"; import type { VoiceProviderKind } from "./voice-control"; @@ -14,11 +14,12 @@ export type VoiceProviderSelection = { provider: VoiceProviderKind; /** * Spoken language the provider should expect (ISO 639-1); empty lets it - * detect the language. Cloud providers only. + * detect the language. Also used by on-device Whisper. */ language?: string; elevenLabsApiKey: string; openAiApiKey: string; + openAiTranscriptionModel?: OpenAiSttModel; sonioxApiKey: string; /** Self-hosted Whisper ("whisper-server"); the key is optional there. */ whisperServerUrl?: string; @@ -52,7 +53,7 @@ const CLOUD_PROVIDERS: Partial> = { whisper: { name: "OpenAI", apiKey: (selection) => selection.openAiApiKey, - create: (options) => new OpenAiRealtimeSttClient(options), + create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }), }, "whisper-server": { name: "Whisper server", @@ -76,8 +77,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection * What a capture transcribes with: the provider and the language it is told * to expect. A running capture is shared only by holders asking for the same. */ -export function transcriptionKey(selection: Pick): string { - return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`; +export function transcriptionKey(selection: Pick): string { + const fallback = selection.provider === "onboard-whisper-fr" ? "fr" : selection.provider === "onboard-whisper" ? "en" : ""; + const language = selection.provider === "onboard-whisper-fr" ? "fr" : (selection.language ?? fallback).trim().toLowerCase(); + const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : ""; + return `${selection.provider}:${language}${model}`; } /** Whether createCloudSttClient would return a cloud client for this selection. */ diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts index 24f109a1..d909cfc5 100644 --- a/app/native/openai-stt.ts +++ b/app/native/openai-stt.ts @@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket"; /** * OpenAI realtime speech-to-text over WebSocket (a transcription-only session - * on the Realtime API, with the streaming gpt-realtime-whisper model). + * on the Realtime API, with a configurable transcription model). * https://developers.openai.com/api/docs/guides/realtime-transcription * * Like the ElevenLabs client, push-to-talk defines the utterance boundary: @@ -16,10 +16,11 @@ import { openSocket, type SocketConnection } from "./socket"; */ const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; -const MODEL_ID = "gpt-realtime-whisper"; +export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper"; +export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe"; const TARGET_SAMPLE_RATE = 24000; -export type OpenAiSttOptions = CloudSttOptions; +export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel }; export class OpenAiRealtimeSttClient implements CloudSttClient { private ws: SocketConnection | null = null; @@ -40,7 +41,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { onOpen: () => { if (this.closed) return; this.open = true; - this.trySend(JSON.stringify(sessionConfig(this.options.language ?? ""))); + this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model))); for (const chunk of this.pendingChunks.splice(0)) { this.sendChunk(chunk); } @@ -165,7 +166,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`); return; } - this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`); + this.options.onError(message.error?.code === "model_not_found" + ? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.` + : `OpenAI: ${String(message.error?.message ?? "unknown error")}`); return; default: return; @@ -201,7 +204,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { } -function sessionConfig(language: string): object { +function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object { return { type: "session.update", session: { @@ -209,7 +212,7 @@ function sessionConfig(language: string): object { audio: { input: { format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE }, - transcription: { model: MODEL_ID, ...(language ? { language } : {}) }, + transcription: { model, ...(language ? { language } : {}) }, // Push-to-talk owns the utterance boundary; no server VAD. turn_detection: null, }, diff --git a/app/native/transcription-language.ts b/app/native/transcription-language.ts new file mode 100644 index 00000000..3aa1e6c5 --- /dev/null +++ b/app/native/transcription-language.ts @@ -0,0 +1,25 @@ +/** Languages supported by both Whisper backends, in picker order. */ +export const TRANSCRIPTION_LANGUAGES = { + en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian", + pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech", + da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi", + hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean", + no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak", + sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese", +} as const; + +export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES; + +/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */ +export function resolveTranscriptionLanguage( + choice: TranscriptionLanguage, provider: string, serverLanguage = "", +): string { + if (provider === "onboard-whisper-fr") return "fr"; + // Moonshine is English-only; Apple's on-device recognizer retains its own locale. + if (provider === "onboard") return ""; + if (choice === "default") { + if (provider === "onboard-whisper") return "en"; + return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : ""; + } + return choice === "auto" ? "" : choice; +} diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index d6a7e37b..22864a78 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,3 +1,4 @@ +import type { OpenAiSttModel } from "./openai-stt"; import { Utils } from "@nativescript/core"; import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; @@ -24,14 +25,14 @@ export type VoiceControlState = { detail: string; }; -// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper, +// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model, // see openai-stt.ts) -- an unfortunate pre-existing name collision with the model // family, not with "onboard-whisper" below, which is the on-device sherpa-onnx // Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than // renamed, since it's a persisted settings value on real installs already. // "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP // (whisper-server-stt.ts). -export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server"; +export type VoiceProviderKind = "onboard" | "onboard-whisper" | "onboard-whisper-fr" | "elevenlabs" | "whisper" | "soniox" | "whisper-server"; export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** @@ -54,11 +55,12 @@ export type PushToTalkOptions = { provider: VoiceProviderKind; elevenLabsApiKey: string; openAiApiKey: string; + openAiTranscriptionModel?: OpenAiSttModel; sonioxApiKey: string; whisperServerUrl?: string; whisperServerModel?: string; whisperServerApiKey?: string; - /** Spoken language for cloud providers (see VoiceProviderSelection.language). */ + /** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */ language?: string; saveRecording: boolean; /** @@ -395,7 +397,10 @@ export class FaceclawVoiceControlBridge { this.started = true; // Which on-device model to load; a no-op setter for every provider except // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). - this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); + const frenchWhisper = options.provider === "onboard-whisper-fr"; + this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" || frenchWhisper ? "whisper" : "moonshine"); + this.controller?.setFrenchWhisper(frenchWhisper); + this.controller?.setOnboardLanguage(frenchWhisper ? "fr" : options.language ?? "en"); const captureId = this.nextNativeCaptureId++; this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); }); this.controller?.start("onboard", captureId); diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 6bed4672..8743142c 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -1,3 +1,4 @@ +import type { OpenAiSttModel } from "../native/openai-stt"; import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts"; import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures"; import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve"; @@ -18,6 +19,7 @@ import { } from "~/assistant/models"; import { isLocalModelReady } from "../native/local-llm"; import { normalizeWhisperServerUrl } from "../native/whisper-server-stt"; +import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language"; import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat"; import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; @@ -882,13 +884,14 @@ export const ringConnectionModeSetting = new ConfigSettingEnum = { onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)", "onboard-whisper": "On-device (Whisper)", + "onboard-whisper-fr": "On-device (Whisper French)", elevenlabs: "ElevenLabs", whisper: "OpenAI (Whisper)", soniox: "Soniox", @@ -902,7 +905,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ defaultValue: "onboard", values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"] - : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"], + : ["onboard", "onboard-whisper", "onboard-whisper-fr", "elevenlabs", "whisper", "soniox", "whisper-server"], formatValue: (value) => voiceProviderLabels[value] ?? value, isDisabled: (value) => { if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0; @@ -913,7 +916,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ }, description: global.isIOS ? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone." - : "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.", + : "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The On-device options need their voice model downloaded (below) and never leave the phone. Whisper French uses the French-only small model and always transcribes in French.", }); const wakeWordActionLabels: Record = { @@ -1136,9 +1139,35 @@ export const whisperServerModelSetting = new ConfigSettingString({ description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.", }); +export const openAiTranscriptionModelSetting = new ConfigSettingEnum({ + id: "openai-transcription-model", + label: "OpenAI transcription model", + storageKey: "voice.openAiTranscriptionModel", + defaultValue: "gpt-realtime-whisper", + values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"], + description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.", +}); + +export const voiceLanguageSetting = new ConfigSettingEnum({ + id: "voice-language", + label: "Transcription language", + storageKey: "voice.language", + defaultValue: "default", + values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])], + formatValue: (value) => value === "default" ? "Provider default" + : value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value], + description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " + + "Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " + + "On-device Whisper needs the multilingual model for auto-detection or languages other than English. The dedicated Whisper French provider always uses French, without changing this preference. Moonshine remains English-only.", +}); + +export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string { + return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get()); +} + export const whisperServerLanguageSetting = new ConfigSettingString({ id: "whisper-server-language", - label: "Whisper language", + label: "Whisper server language", storageKey: "voice.whisperServerLanguage", defaultValue: "", editorTitle: "Language code, e.g. en or sk (empty: auto)", @@ -1146,7 +1175,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({ formatValue: (value) => value || "Auto-detect", validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."), description: - "Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.", + "Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.", }); export const whisperServerApiKeySetting = new ConfigSettingString({ diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index f974bb8f..c889a4b4 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -63,6 +63,7 @@ import { mapboxApiKeySetting, mirrorTouchSetting, openAiApiKeySetting, + openAiTranscriptionModelSetting, previewColorSetting, previewScalingSetting, phoneRotationSetting, @@ -87,6 +88,7 @@ import { toggleSettingMenuItem, useMicControlSetting, voiceProviderSetting, + voiceLanguageSetting, screenTimeoutSetting, wakeToLastAppSetting, wakeWordActionSetting, @@ -143,9 +145,12 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), + enumSettingMenuItem(voiceLanguageSetting), ...whisperServerMenuItems(), asrModelMenuItem("moonshine"), asrModelMenuItem("whisper-base-en"), + asrModelMenuItem("whisper-base"), + asrModelMenuItem("whisper-small-fr"), ], }, { @@ -284,7 +289,7 @@ function settingsSections(): SettingsSection[] { toggleSettingMenuItem(showBleBandwidthSetting)] }; // No downloadable on-device models: iOS's on-device option is Apple's recognizer. if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), - enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; + enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; if (section.label === "Display") return { ...section, items: section.items.filter(item => item.label !== screenTimeoutSetting.label) }; return section; @@ -527,6 +532,7 @@ function asrModelStatusText(id: AsrModelId): string { */ function whisperServerMenuItems(): MenuItem[] { return [ + enumSettingMenuItem(openAiTranscriptionModelSetting), textSettingMenuItem(whisperServerUrlSetting), textSettingMenuItem(whisperServerModelSetting), textSettingMenuItem(whisperServerLanguageSetting), diff --git a/docs/models/whisper-small-fr/LICENSE.txt b/docs/models/whisper-small-fr/LICENSE.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/docs/models/whisper-small-fr/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/models/whisper-small-fr/README.md b/docs/models/whisper-small-fr/README.md new file mode 100644 index 00000000..40f31064 --- /dev/null +++ b/docs/models/whisper-small-fr/README.md @@ -0,0 +1,71 @@ +# On-device Whisper French + +Android Settings → Voice now offers `On-device (Whisper French)` and a separate +`On-device model: Whisper (French, small)` download (~375 MB decimal). Download +first, then choose the provider. Recognition stays on the phone; no API key is +required. This is speech-to-text in French, not translation. + +This provider always requests `language=fr`, `task=transcribe`, without writing +the shared language preference. Switching back to existing Moonshine, English +Whisper or multilingual Whisper restores their previous behavior. Downloads, +cancellation/resume and deletion reuse the existing model manager and leave +other model directories untouched. Missing French files do not silently select +multilingual base. iOS and the Microphones app's separate Moonshine caption +engine are outside this addition. + +## Provenance and distribution + +- Source: https://huggingface.co/bofenghuang/whisper-small-cv11-french +- Pinned revision: `83d7df2928fb2218721c543dcc0be29ef8db2f96`. +- The model author's declared license is Apache-2.0; `LICENSE.txt` is distributed + with the downloadable model. The original Whisper implementation is MIT; + sherpa-onnx's exporter is Apache-2.0. +- Exporter: sherpa-onnx v1.13.0 `scripts/whisper/export-onnx.py`. +- Export: strict tensor-name conversion from HF Whisper to OpenAI Whisper, + then upstream sherpa tensor-cache export, opset 17, dynamic int8 MatMul. +- Assets: https://git.undefined-dev.fr/Blendman974/faceclaw/releases/tag/models-whisper-small-fr-v1 +- Model files use the public Gitea generic package registry (`faceclaw-asr`, + version `whisper-small-fr-v1`), because the encoder and decoder exceed the + instance’s 100 MB attachment limit. No infrastructure limit is changed. +- The JSON manifest records file lengths and SHA-256 hashes. Both the source + checkpoint and exporter are checksum-verified before execution. The existing + Android downloader checks each downloaded asset against its pinned hash. +- Only encoder, decoder, vocabulary and license are downloaded to the phone; + the original checkpoint and unquantized intermediate models are not shipped. + +## Reproducing the export + +Use an isolated Python environment with the versions used for this release: +Python 3.14, torch 2.9.0+cpu (CPU wheel index), openai-whisper 20250625, +onnx 1.23.2, onnxruntime 1.31.1 and sherpa-onnx 1.13.0. These are development +requirements only; no Python or additional runtime is added to the app. +On NixOS expose the host C++ runtime and zlib through LD_LIBRARY_PATH. + +```sh +python tools/export-whisper-small-fr.py --output /tmp/whisper-small-fr +python tools/test-whisper-small-fr.py \ + --model-dir /tmp/whisper-small-fr --wav /path/to/french-test.wav +``` + +The exporter downloads the pinned source (~484 MB), writes unquantized and +quantized intermediate models, and produces `manifest.json`. Artifact changes +require a new versioned release and explicit catalog hash changes; never replace +v1 assets in place. These scripts do not upload or publish files automatically. + +## Validation + +The exported int8 files were actually decoded on CPU with sherpa-onnx 1.13.0, +the same version used by Android. A public 4.97-second French sample from +`csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3/test_wavs/fr.wav` produced: + +> Ne vous demandez pas ce que votre pays peut faire pour vous, demandez-vous plutôt ce que vous pouvez faire pour lui. + +This is a smoke test, not a recognition-quality benchmark or a measurement of +Pixel latency. Application tests cover provider persistence, French selection +without rewriting other languages, switching back, capture sharing, interrupted +download/resume, file hashes and independent deletion. Kotlin tests cover the +native file selection and preservation of existing filenames/defaults. + +Hardware checks remain pending: real French dictation through phone and glasses +microphones, latency and RAM on the Pixel, airplane-mode use after downloading, +reconnection and long utterances. No firmware changes are required. diff --git a/docs/models/whisper-small-fr/manifest.json b/docs/models/whisper-small-fr/manifest.json new file mode 100644 index 00000000..8f3cf6ee --- /dev/null +++ b/docs/models/whisper-small-fr/manifest.json @@ -0,0 +1,29 @@ +{ + "source": "bofenghuang/whisper-small-cv11-french", + "revision": "83d7df2928fb2218721c543dcc0be29ef8db2f96", + "sourceSha256": "855a879fa3a71e75f29ed59fcfce287cf21bcdaab23530cd6f8c4ad192677afb", + "exporterSha256": "5e0ddabfac8330ee9aecaba4c1b4286d1cd9f88df4cd30e54706b011bf976e45", + "torchVersion": "2.9.0+cpu", + "files": [ + { + "name": "small-fr-encoder.int8.onnx", + "sha256": "ec34fc42182f37727ad982d254b18a6001f5dcccebc5a36060031995f1782d5d", + "sizeBytes": 112413414 + }, + { + "name": "small-fr-decoder.int8.onnx", + "sha256": "b554e35dc8c593b550a82cd01ebb00c1347e8c2b089b780b4c4fe16450cd7dc0", + "sizeBytes": 262200764 + }, + { + "name": "small-fr-tokens.txt", + "sha256": "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", + "sizeBytes": 816730 + }, + { + "name": "LICENSE.txt", + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", + "sizeBytes": 11358 + } + ] +} diff --git a/docs/validation-whisper-languages.md b/docs/validation-whisper-languages.md new file mode 100644 index 00000000..601a2d0a --- /dev/null +++ b/docs/validation-whisper-languages.md @@ -0,0 +1,68 @@ +# Whisper transcription languages + +Settings > Voice > Transcription language selects French, English, 27 other +languages, or auto-detection. It applies to ordinary dictation and continuous +transcription using OpenAI, on-device Whisper, and the other cloud providers. +Translate retains its own spoken-language preference. Moonshine remains +English-only; Apple's on-device speech recognizer retains its own locale. + +The initial **Provider default** preserves existing installations: English for +on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved +Whisper server language for self-hosted Whisper. Selecting another shared +language overrides the server language without deleting it. Returning to Provider +default restores it. Settings persist through the existing settings store. + +For French or auto-detection with **On-device (Whisper)**, download +**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB). +English uses the existing English-only base model; neither model is deleted or +downloaded automatically. A missing model uses the existing “Voice model not +downloaded” status. Language changes restart active continuous transcription; +other captures use the new language when started. + +## Software validation + +- JavaScript regression tests exercise stored settings/reload, compatible + defaults, French and auto-detection, Translate overrides in Android/iOS capture + options, the OpenAI WebSocket configuration, and local bridge restart/sharing. +- Kotlin tests exercise English/French/other/automatic model selection and file + names. Android release compilation checks the actual sherpa-onnx bridge. +- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and + the existing resumable downloader. Encoder/decoder hashes and sizes come from + repository LFS metadata; the vocabulary hash was computed from its contents. + +## Device validation still required + +With both phone microphone preview and glasses microphone, check French dictation +(including accents), auto-detection, English rollback, continuous capture while +changing language, cancellation/resumption of model download, and persistence +after restarting the app. Compare OpenAI and offline Whisper on the same phrases. +No paid API call, microphone recording, or firmware update is needed for the +software checks above; recognition accuracy and latency require device testing. + +## References + +- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription) +- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc) +- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc) + +## OpenAI transcription model selection + +Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper, +gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks +gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter. +The unset default retains gpt-realtime-whisper for existing installations; no +provider, language or Assistant model preference is overwritten. This setting +applies to Translate's speech recognition as well as Transcribe and dictation. +The two transcribe alternatives produce transcript deltas after a committed +audio segment; existing pause segmentation and push-to-talk release supply the +commits. Choosing a model retargets active continuous capture. + +Model access errors name the requested model before the long project error and +point to the setting. Factory/session-wire tests cover model, language, manual +commit and final transcript behavior; settings tests cover persistence and the +Android/iOS option plumbing. No real API inference was performed; actual project +access, recognition and latency remain to be checked on a device. The model list +provided by the operator establishes account visibility, not a successful audio +request. + +Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create). diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt new file mode 100644 index 00000000..99afe8b4 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt @@ -0,0 +1,22 @@ +package com.faceclaw.app + +/** Existing English/multilingual files stay usable; the dedicated French provider uses its own download. */ +data class WhisperModelFiles(val language: String, val frenchOnly: Boolean = false) { + val englishOnly: Boolean get() = !frenchOnly && language == "en" + val dirName: String get() = if (frenchOnly) "sherpa-onnx-whisper-small-fr-int8-v1" else if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8" + private val prefix: String get() = if (frenchOnly) "small-fr" else if (englishOnly) "base.en" else "base" + val encoder: String get() = "$prefix-encoder.int8.onnx" + val decoder: String get() = "$prefix-decoder.int8.onnx" + val tokens: String get() = "$prefix-tokens.txt" + val files: List get() = listOf(encoder, decoder, tokens) + + companion object { + /** A dedicated French fine-tune never silently substitutes the generic base model. */ + fun forSelection(language: String?, frenchOnly: Boolean): WhisperModelFiles = + if (frenchOnly) WhisperModelFiles("fr", true) else forLanguage(language) + + /** Null preserves the old English default; an empty code enables automatic detection. */ + fun forLanguage(language: String?): WhisperModelFiles = + WhisperModelFiles(language?.trim()?.lowercase() ?: "en") + } +} diff --git a/tests/assistant-settings-visibility.test.cjs b/tests/assistant-settings-visibility.test.cjs index 43e96713..4173342f 100644 --- a/tests/assistant-settings-visibility.test.cjs +++ b/tests/assistant-settings-visibility.test.cjs @@ -54,7 +54,9 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') { '../../native/asr-model': { ASR_MODELS: { moonshine: { label: 'Moonshine', sizeBytes: 1 }, + 'whisper-small-fr': { label: 'Whisper French Small', sizeBytes: 1 }, 'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 }, + 'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 }, }, asrModelState: () => ({}), cancelAsrModelDownload() {}, deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {}, diff --git a/tests/brightness-curve.test.cjs b/tests/brightness-curve.test.cjs index 0d4c2aa9..efe5e9db 100644 --- a/tests/brightness-curve.test.cjs +++ b/tests/brightness-curve.test.cjs @@ -26,6 +26,7 @@ function settingsFixture(store = new Map()) { }; if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js'); if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js'); + if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js'); return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }; }}; vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), { diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs index 06abfb5f..dbaa71f5 100644 --- a/tests/cloud-stt.test.cjs +++ b/tests/cloud-stt.test.cjs @@ -207,6 +207,10 @@ test('provider selection falls back to on-device without a key and connects the // A running capture is shared only on the same provider and language. assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja'); assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' })); + assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' })); + assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' })); + assert.equal(transcriptionKey({ provider: 'onboard-whisper-fr', language: 'en' }), 'onboard-whisper-fr:fr'); + assert.notEqual(transcriptionKey({ provider: 'onboard-whisper-fr' }), transcriptionKey({ provider: 'onboard-whisper', language: 'fr' })); const statuses = []; const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} }; const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' }; @@ -225,13 +229,13 @@ test('provider selection falls back to on-device without a key and connects the }); test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => { - for (const language of ['ja', '']) { + for (const language of ['fr', 'ja', '']) { const elevenlabs = harness('elevenlabs', true, { language }); assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null); const soniox = harness('soniox', true, { language }); soniox.open(); - assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined); + assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined); const openai = harness('openai', true, { language }); openai.open(); @@ -240,3 +244,43 @@ test('a spoken-language hint reaches each provider on the wire, and is left out assert.equal(transcription.language, language || undefined); } }); + +test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => { + for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) { + const h = harness('openai', true, { model, language: 'fr' }); + h.open(); + const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input; + assert.equal(input.transcription.model, model); + assert.equal(input.transcription.language, 'fr'); + assert.equal(input.turn_detection, null); + h.client.acceptPcm(pcm(1200, 150)); + h.client.acceptPcm(pcm(0, 1500)); + assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit')); + h.message({ type: 'input_audio_buffer.committed', item_id: 'one' }); + h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' }); + h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' }); + assert.equal(h.events.at(-1).text, 'Bonjour à tous.'); + assert.equal(h.events.at(-1).isFinal, true); + h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } }); + assert.ok(h.errors[0].includes(model)); + assert.ok(h.errors[0].includes('OpenAI transcription model')); + } +}); + +test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => { + const created = []; + const load = loader({ setTimeout, clearTimeout }, { + './openai-stt': { + DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper', + OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} }, + }, + }); + const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts'); + const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' }; + createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start(); + assert.equal(created[0].model, selection.openAiTranscriptionModel); + assert.equal(created[0].language, 'fr'); + assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' })); + assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' })); + assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' })); +}); diff --git a/tests/display-layout-setting.test.cjs b/tests/display-layout-setting.test.cjs index 55e44f71..74bb2d3f 100644 --- a/tests/display-layout-setting.test.cjs +++ b/tests/display-layout-setting.test.cjs @@ -12,6 +12,7 @@ function settingsFixture(store = new Map()) { }; if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js'); if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js'); + if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js'); return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }; }}; vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), { diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index 37cf988f..d02d2063 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -6,7 +6,8 @@ const ts = require('typescript'); const path = require('node:path'); // The provider settings ios-preview-controller hands the voice bridge. const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' }, - openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' }, + getVoiceTranscriptionLanguage: () => '', + openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' }, whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' }, whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } }; function load(file, context) { @@ -168,6 +169,7 @@ test('settings-driven repaint runs after font cache invalidation, regardless of let cachedFont = 'Light'; const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true }, require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) } + : id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {}) : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }, setTimeout: fn => tasks.push(fn), }); @@ -384,7 +386,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec recordTo: name => { bridge.recording = name; }, setRawSession: session => { bridge.rawSession = session; } }; const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' }, - whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' }, + whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en', saveTranscribeAudioSetting: { get: () => true } }; const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt new file mode 100644 index 00000000..06a81ad6 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt @@ -0,0 +1,46 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class WhisperModelFilesTest { + @Test fun existingEnglishFilesRemainUsable() { + for (language in listOf(null, "en", " EN ")) { + val model = WhisperModelFiles.forLanguage(language) + assertTrue(model.englishOnly) + assertEquals("en", model.language) + assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName) + assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files) + } + } + + @Test fun frenchAndOtherLanguagesUseMultilingualFiles() { + for (language in listOf("fr", "de", "ja", " FR ")) { + val model = WhisperModelFiles.forLanguage(language) + assertFalse(model.englishOnly) + assertEquals(language.trim().lowercase(), model.language) + assertEquals("sherpa-onnx-whisper-base-int8", model.dirName) + assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files) + } + } + + @Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() { + val model = WhisperModelFiles.forLanguage("") + assertFalse(model.englishOnly) + assertEquals("", model.language) + assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files) + } + @Test fun frenchFineTuneUsesDistinctFilesRegardlessOfTheGenericLanguage() { + for (language in listOf(null, "en", "de", "", "fr")) { + val model = WhisperModelFiles.forSelection(language, true) + assertTrue(model.frenchOnly) + assertFalse(model.englishOnly) + assertEquals("fr", model.language) + assertEquals("sherpa-onnx-whisper-small-fr-int8-v1", model.dirName) + assertEquals(listOf("small-fr-encoder.int8.onnx", "small-fr-decoder.int8.onnx", "small-fr-tokens.txt"), model.files) + assertEquals(WhisperModelFiles.forLanguage(language), WhisperModelFiles.forSelection(language, false)) + } + } +} diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 5c5fd912..2469bb34 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -28,6 +28,7 @@ "../app/ui/menu-core.ts", "../app/phone-ui/phone-gestures.ts", "../app/native/java-direct-buffer.ts", + "../app/native/transcription-language.ts", "../app/native/notification-types.ts", "../app/g2/firmware-compat.ts", "../app/g2/glasses-hardware-identity.ts", diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs index c049d8b2..1cafe7b3 100644 --- a/tests/voice-finalization.test.cjs +++ b/tests/voice-finalization.test.cjs @@ -163,7 +163,9 @@ function androidBridge() { starts: 0, setListener(listener) { this.listener = listener; }, setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {}, - setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {}, + setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, + setFrenchWhisper(enabled) { this.frenchWhisper = enabled; }, + setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; }, start(_mode, id) { this.captureId = id; this.active = true; this.starts++; }, stop() { this.active = false; }, isCapturing() { return this.active; }, @@ -171,7 +173,8 @@ function androidBridge() { const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', { '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, './speech-pause': { SpeechPauseDetector: class { reset() {} } }, - './cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider }, + './cloud-stt-provider': { createCloudSttClient: () => null, + transcriptionKey: options => `${options.provider}:${options.provider === 'onboard-whisper-fr' ? 'fr' : options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` }, }, { global: { isAndroid: true }, com: { faceclaw: { app: { FaceclawVoiceController: function () { return controller; }, FaceclawVoiceControllerListener: function (listener) { return listener; }, @@ -179,6 +182,26 @@ function androidBridge() { return { bridge: new FaceclawVoiceControlBridge(), controller }; } +test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => { + const { bridge, controller } = androidBridge(); + bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' }); + assert.equal(controller.modelKind, 'whisper'); + assert.equal(controller.language, 'fr'); + bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' }); + assert.equal(controller.starts, 1); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' }); + assert.equal(controller.language, ''); + assert.equal(controller.starts, 2); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper' }); + // An omitted language preserves the native bridge's original English behavior. + assert.equal(controller.language, 'en'); + assert.equal(controller.starts, 3); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' }); + assert.equal(controller.language, 'en'); + assert.equal(controller.starts, 3); + bridge.stop(); +}); + test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => { const { bridge, controller } = androidBridge(); const options = { provider: 'onboard-whisper' }; @@ -215,3 +238,16 @@ test('a shared capture restarts for a holder that needs another provider, and is assert.equal(controller.active, true); assert.equal(bridge.holdsCapture('ptt'), true); }); + +test('dedicated French Whisper selects its own files and switches back without retaining the override', () => { + const { bridge, controller } = androidBridge(); + bridge.startContinuousCapture({ provider: 'onboard-whisper-fr', language: 'en' }); + assert.equal(controller.modelKind, 'whisper'); + assert.equal(controller.frenchWhisper, true); + assert.equal(controller.language, 'fr'); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper' }); + assert.equal(controller.frenchWhisper, false); + assert.equal(controller.language, 'en'); + assert.equal(controller.starts, 2); + bridge.stop(); +}); diff --git a/tests/whisper-language.test.cjs b/tests/whisper-language.test.cjs new file mode 100644 index 00000000..77689e56 --- /dev/null +++ b/tests/whisper-language.test.cjs @@ -0,0 +1,210 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function load(file, requireModule = () => ({}), globals = {}) { + const exports = {}; + vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals }); + return exports; +} +const languages = load('app/native/transcription-language.ts'); +function settings(values = new Map()) { + return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages + : name.includes('settings-store') ? { + getStringSetting: (key, fallback) => values.get(key) ?? fallback, + setStringSetting: (key, value) => values.set(key, value), + getBooleanSetting: (_key, fallback) => fallback, + onSettingsStoreChanged() {}, + } : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }); +} + +test('language defaults preserve local English, cloud detection and an existing server preference', () => { + const values = new Map([['voice.whisperServerLanguage', 'sk']]); + const s = settings(values); + assert.equal(s.voiceLanguageSetting.get(), 'default'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en'); + assert.equal(s.getVoiceTranscriptionLanguage('whisper'), ''); + assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk'); + assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]); +}); + +test('French and auto-detection persist across restart without overwriting server exceptions', () => { + const values = new Map([['voice.whisperServerLanguage', 'sk']]); + let s = settings(values); + s.voiceLanguageSetting.set('fr'); + s = settings(values); + assert.equal(s.voiceLanguageSetting.displayValue(), 'French'); + for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) { + assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr'); + } + assert.equal(s.getVoiceTranscriptionLanguage('onboard'), ''); + s.voiceLanguageSetting.set('auto'); + s = settings(values); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), ''); + assert.equal(s.getVoiceTranscriptionLanguage('whisper'), ''); + s.voiceLanguageSetting.set('default'); + assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en'); +}); + +test('corrupt language preferences revert to the compatible provider defaults', () => { + const s = settings(new Map([['voice.language', 'invalid']])); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en'); + assert.equal(s.getVoiceTranscriptionLanguage('whisper'), ''); +}); + +test('Android and iOS capture options use the shared language and preserve Translate overrides', () => { + const s = settings(); + s.voiceLanguageSetting.set('fr'); + s.voiceProviderSetting.set('whisper'); + s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe'); + for (const [file, name, method] of [ + ['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'], + ['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'], + ]) { + const host = { continuousClients: new Map() }; + host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription; + // Run each real capture-options method with only its native dependencies stubbed. + const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); + const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true); + const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name); + const member = declaration.members.find(n => n.name?.getText(tree) === method); + const capture = loadBody(member.body.getText(tree), s, host); + assert.equal(capture().language, 'fr'); + assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe'); + host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } }); + assert.equal(capture().provider, 'soniox'); + assert.equal(capture().language, 'ja'); + host.continuousClients.clear(); + assert.equal(capture().language, 'fr'); + assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe'); + } +}); + +function loadBody(body, s, host) { + const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } }; + return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null); +} + +test('multilingual downloads and deletion leave an existing English model intact', () => { + const files = new Map(), downloads = []; + const asr = load('app/native/asr-model.ts', () => ({ + Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } }, + }), { + java: { io: { File: class { + constructor(name) { this.name = name; } + exists() { return files.has(this.name); } + length() { return files.get(this.name) ?? 0; } + delete() { return files.delete(this.name); } + } } }, + com: { faceclaw: { app: { + FaceclawModelDownloaderListener: function(listener) { return listener; }, + FaceclawModelDownloader: class { + constructor(url, destination, hash, size, listener) { + Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this); + } + start() {} + cancel() {} + }, + } } }, + }); + const english = asr.ASR_MODELS['whisper-base-en']; + for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes); + assert.equal(asr.isAsrModelReady('whisper-base-en'), true); + assert.equal(asr.isAsrModelReady('whisper-base'), false); + asr.startAsrModelDownload('whisper-base'); + assert.equal(asr.asrModelState('whisper-base').status, 'downloading'); + for (let index = 0; index < 3; index++) { + const download = downloads[index]; + assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/); + assert.match(download.hash, /^[a-f0-9]{64}$/); + files.set(download.destination, download.size); + download.listener.onDone(); + } + assert.equal(downloads.length, 3); + assert.equal(asr.isAsrModelReady('whisper-base'), true); + asr.deleteAsrModel('whisper-base'); + assert.equal(asr.isAsrModelReady('whisper-base'), false); + assert.equal(asr.isAsrModelReady('whisper-base-en'), true); +}); + +test('OpenAI transcription model preserves legacy default and persists independently of language', () => { + const values = new Map(); + let s = settings(values); + assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper'); + s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe'); + s.voiceLanguageSetting.set('fr'); + s = settings(values); + assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe'); + assert.equal(s.voiceLanguageSetting.get(), 'fr'); + assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper'); +}); + +test('French provider persists independently and never overwrites shared language preferences', () => { + const values = new Map([['voice.language', 'ja'], ['voice.provider', 'onboard-whisper']]); + let s = settings(values); + s.voiceProviderSetting.set('onboard-whisper-fr'); + s = settings(values); + assert.equal(s.voiceProviderSetting.get(), 'onboard-whisper-fr'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper-fr'), 'fr'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'ja'); + assert.equal(s.voiceLanguageSetting.get(), 'ja'); + s.voiceProviderSetting.set('onboard-whisper'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'ja'); +}); + +test('French model download resumes, verifies every file including its license, and preserves other models', () => { + const files = new Map(), downloads = []; + const asr = load('app/native/asr-model.ts', () => ({ + Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } }, + }), { + java: { io: { File: class { + constructor(name) { this.name = name; } + exists() { return files.has(this.name); } + length() { return files.get(this.name) ?? 0; } + delete() { return files.delete(this.name); } + } } }, + com: { faceclaw: { app: { + FaceclawModelDownloaderListener: function(listener) { return listener; }, + FaceclawModelDownloader: class { + constructor(url, destination, hash, size, listener) { + Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this); + } + start() {} + cancel() {} + }, + } } }, + }); + for (const id of ['moonshine', 'whisper-base-en', 'whisper-base']) { + const model = asr.ASR_MODELS[id]; + for (const file of model.files) files.set(`/files/faceclaw-voice-asr/${model.dirName}/${file.name}`, file.sizeBytes); + } + const model = asr.ASR_MODELS['whisper-small-fr']; + const manifest = JSON.parse(fs.readFileSync(path.join(__dirname, '../docs/models/whisper-small-fr/manifest.json'))); + assert.deepEqual(JSON.parse(JSON.stringify(model.files)), manifest.files); + assert.equal(model.totalBytes, model.files.reduce((total, file) => total + file.sizeBytes, 0)); + assert.equal(asr.isAsrModelReady('whisper-small-fr'), false); + asr.startAsrModelDownload('whisper-small-fr'); + files.set(downloads[0].destination, downloads[0].size); + downloads[0].listener.onDone(); + downloads[1].listener.onError('network unavailable'); + assert.equal(asr.asrModelState('whisper-small-fr').status, 'absent'); + asr.startAsrModelDownload('whisper-small-fr'); + assert.equal(downloads[2].destination, downloads[1].destination, 'resume the missing decoder'); + for (let index = 2; index < 5; index++) { + const download = downloads[index]; + assert.match(download.url, /api\/packages\/Blendman974\/generic\/faceclaw-asr\/whisper-small-fr-v1\//); + assert.match(download.hash, /^[a-f0-9]{64}$/); + files.set(download.destination, download.size); + download.listener.onDone(); + } + assert.equal(asr.isAsrModelReady('whisper-small-fr'), true); + asr.deleteAsrModel('whisper-small-fr'); + assert.equal(asr.isAsrModelReady('whisper-small-fr'), false); + for (const id of ['moonshine', 'whisper-base-en', 'whisper-base']) assert.equal(asr.isAsrModelReady(id), true); +}); diff --git a/tools/export-whisper-small-fr.py b/tools/export-whisper-small-fr.py new file mode 100644 index 00000000..f7f56a8d --- /dev/null +++ b/tools/export-whisper-small-fr.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Export the pinned French fine-tune to sherpa-onnx 1.13.0, using CPU only. + +See docs/models/whisper-small-fr/README.md for the isolated export environment. +This script never uploads assets or touches installed/downloaded phone models. +""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +from types import SimpleNamespace +import urllib.request + +SOURCE_REVISION = '83d7df2928fb2218721c543dcc0be29ef8db2f96' +SOURCE_BASE = f'https://huggingface.co/bofenghuang/whisper-small-cv11-french/resolve/{SOURCE_REVISION}/' +WEIGHTS_SHA256 = '855a879fa3a71e75f29ed59fcfce287cf21bcdaab23530cd6f8c4ad192677afb' +EXPORTER_URL = 'https://raw.githubusercontent.com/k2-fsa/sherpa-onnx/v1.13.0/scripts/whisper/export-onnx.py' +EXPORTER_SHA256 = '5e0ddabfac8330ee9aecaba4c1b4286d1cd9f88df4cd30e54706b011bf976e45' +MODEL_NAME = 'small-fr' + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def download(url, path, expected=None): + if not path.exists(): + temp = path.with_suffix(path.suffix + '.part') + urllib.request.urlretrieve(url, temp) + temp.replace(path) + if expected and digest(path) != expected: + raise ValueError(f'Checksum mismatch: {path.name}') + + +def openai_key(key): + """HF Whisper and OpenAI Whisper have identical tensors, differently named.""" + if not key.startswith('model.'): + raise ValueError(f'Unexpected weight: {key}') + key = key.removeprefix('model.') + replacements = [ + ('embed_positions.weight', 'positional_embedding'), + ('embed_tokens', 'token_embedding'), ('layers.', 'blocks.'), + ('self_attn_layer_norm', 'attn_ln'), ('encoder_attn_layer_norm', 'cross_attn_ln'), + ('final_layer_norm', 'mlp_ln'), ('self_attn', 'attn'), ('encoder_attn', 'cross_attn'), + ('k_proj', 'key'), ('q_proj', 'query'), ('v_proj', 'value'), ('out_proj', 'out'), + ('fc1', 'mlp.0'), ('fc2', 'mlp.2'), + ('encoder.layer_norm', 'encoder.ln_post'), ('decoder.layer_norm', 'decoder.ln'), + ] + for old, new in replacements: + key = key.replace(old, new) + return key + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + download(SOURCE_BASE + 'pytorch_model.bin', output / 'pytorch_model.bin', WEIGHTS_SHA256) + download(SOURCE_BASE + 'config.json', output / 'config.json') + download(EXPORTER_URL, output / 'sherpa-export.py', EXPORTER_SHA256) + + import torch + import whisper + from whisper.model import ModelDimensions, disable_sdpa + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + torch.manual_seed(0) + config = json.loads((output / 'config.json').read_text()) + dimensions = ModelDimensions( + n_mels=config['num_mel_bins'], n_audio_ctx=config['max_source_positions'], + n_audio_state=config['d_model'], n_audio_head=config['encoder_attention_heads'], + n_audio_layer=config['encoder_layers'], n_vocab=config['vocab_size'], + n_text_ctx=config['max_target_positions'], n_text_state=config['d_model'], + n_text_head=config['decoder_attention_heads'], n_text_layer=config['decoder_layers'], + ) + model = whisper.Whisper(dimensions) + weights = torch.load(output / 'pytorch_model.bin', map_location='cpu', weights_only=True) + converted = {openai_key(key): value for key, value in weights.items()} + model.load_state_dict(converted, strict=True) + del weights, converted + model.eval() + + spec = importlib.util.spec_from_file_location('sherpa_export', output / 'sherpa-export.py') + exporter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(exporter) + exporter.get_args = lambda: SimpleNamespace(model=MODEL_NAME) + exporter.load_model = lambda _name: model + original_export = torch.onnx.export + + def legacy_export(*values, **options): + options['dynamo'] = False # sherpa 1.13.0 uses TorchScript's exporter. + return original_export(*values, **options) + + torch.onnx.export = legacy_export + old_cwd = Path.cwd() + try: + os.chdir(output) + with disable_sdpa(): + exporter.main() + finally: + torch.onnx.export = original_export + os.chdir(old_cwd) + + license_path = Path(__file__).resolve().parents[1] / 'docs/models/whisper-small-fr/LICENSE.txt' + shutil.copyfile(license_path, output / 'LICENSE.txt') + names = [f'{MODEL_NAME}-encoder.int8.onnx', f'{MODEL_NAME}-decoder.int8.onnx', f'{MODEL_NAME}-tokens.txt', 'LICENSE.txt'] + manifest = { + 'source': 'bofenghuang/whisper-small-cv11-french', 'revision': SOURCE_REVISION, + 'sourceSha256': WEIGHTS_SHA256, 'exporterSha256': EXPORTER_SHA256, + 'torchVersion': torch.__version__, + 'files': [{'name': name, 'sha256': digest(output / name), 'sizeBytes': (output / name).stat().st_size} for name in names], + } + (output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') + print(json.dumps(manifest, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/tools/test-whisper-small-fr.py b/tools/test-whisper-small-fr.py new file mode 100644 index 00000000..b1705570 --- /dev/null +++ b/tools/test-whisper-small-fr.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Verify released hashes and decode a public/test WAV with sherpa-onnx CPU.""" +import argparse +import hashlib +import json +from pathlib import Path +import time +import wave + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--wav', type=Path, required=True) + parser.add_argument('--expected', help='Optional expected transcription substring') + args = parser.parse_args() + model_dir = args.model_dir + manifest = json.loads((model_dir / 'manifest.json').read_text()) + for entry in manifest['files']: + path = model_dir / entry['name'] + with path.open('rb') as stream: + actual = hashlib.file_digest(stream, 'sha256').hexdigest() + if actual != entry['sha256'] or path.stat().st_size != entry['sizeBytes']: + raise ValueError(f'Model file integrity check failed: {path.name}') + + import numpy as np + import sherpa_onnx + with wave.open(str(args.wav)) as audio: + if audio.getnchannels() != 1 or audio.getsampwidth() != 2: + raise ValueError('Provide a mono PCM16 WAV') + sample_rate = audio.getframerate() + samples = np.frombuffer(audio.readframes(audio.getnframes()), dtype='