Merge multilingual Whisper transcription

# Conflicts:
#	app/g2/dashboard-controller.ts
This commit is contained in:
Blendman974 committed 2026-11-06 04:31:25 +01:00
commit cb4635d959
21 files changed
+375 -39

No files matched your search

+32 -2
View File
@@ -9,7 +9,7 @@ declare const java: any;
* than bundled in the APK; FaceclawVoiceController reads the files from the
* directory each model's `dirName` names.
*
* Two models, same shape:
* Downloadable models, same shape:
* - "moonshine": the original on-device option (three files). The model is
* no longer bundled in the APK; it is fetched into the same filesDir
* location that earlier releases copied the bundled files to, so
@@ -29,12 +29,17 @@ declare const java: any;
* used for Moonshine below. Every hash here was computed locally from a
* freshly downloaded copy of the file, not copied from a doc page.
*
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
* speech and auto-detection; English-only downloads remain untouched.
* SHA-256/size values come from the maintainer's LFS metadata, with the
* vocabulary hash computed locally. URLs pin the source revision.
*
* Mirrors the on-phone assistant model flow in llama.ts, except each model
* here is multiple files rather than one; they download sequentially through
* FaceclawModelDownloader (resume + pinned sha256 per file).
*/
export type AsrModelId = "moonshine" | "whisper-base-en";
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base";
type AsrModelFile = {
name: string;
@@ -105,6 +110,30 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
],
totalBytes: 160626066,
},
"whisper-base": {
label: "Whisper (multilingual, base)",
dirName: "sherpa-onnx-whisper-base-int8",
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
files: [
{
name: "base-encoder.int8.onnx",
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
sizeBytes: 29120534,
},
{
name: "base-decoder.int8.onnx",
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
sizeBytes: 130672026,
},
{
name: "base-tokens.txt",
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
sizeBytes: 816730,
},
],
totalBytes: 160609290,
},
};
export type AsrModelState = {
@@ -129,6 +158,7 @@ function freshRuntime(): ModelRuntime {
const runtimes: Record<AsrModelId, ModelRuntime> = {
moonshine: freshRuntime(),
"whisper-base-en": freshRuntime(),
"whisper-base": freshRuntime(),
};
function modelDirPath(id: AsrModelId): string {
+3 -2
View File
@@ -14,7 +14,7 @@ export type VoiceProviderSelection = {
provider: VoiceProviderKind;
/**
* Spoken language the provider should expect (ISO 639-1); empty lets it
* detect the language. Cloud providers only.
* detect the language. Also used by on-device Whisper.
*/
language?: string;
elevenLabsApiKey: string;
@@ -77,7 +77,8 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
* to expect. A running capture is shared only by holders asking for the same.
*/
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}`;
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
+24
View File
@@ -0,0 +1,24 @@
/** Languages supported by both Whisper backends, in picker order. */
export const TRANSCRIPTION_LANGUAGES = {
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
} as const;
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
export function resolveTranscriptionLanguage(
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
): string {
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
if (provider === "onboard") return "";
if (choice === "default") {
if (provider === "onboard-whisper") return "en";
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
}
return choice === "auto" ? "" : choice;
}
+2 -1
View File
@@ -58,7 +58,7 @@ export type PushToTalkOptions = {
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerApiKey?: string;
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */
language?: string;
saveRecording: boolean;
/**
@@ -396,6 +396,7 @@ export class FaceclawVoiceControlBridge {
// Which on-device model to load; a no-op setter for every provider except
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
this.controller?.setOnboardLanguage(options.language ?? "en");
const captureId = this.nextNativeCaptureId++;
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
this.controller?.start("onboard", captureId);