forked from Blendman974/faceclaw
Merge dedicated offline French Whisper model
This commit is contained in:
18 files changed
+621
-17
No files matched your search
+18
-1
@@ -34,12 +34,15 @@ declare const java: any;
|
||||
* SHA-256/size values come from the maintainer's LFS metadata, with the
|
||||
* vocabulary hash computed locally. URLs pin the source revision.
|
||||
*
|
||||
* - "whisper-small-fr": a dedicated French fine-tune, exported and quantized
|
||||
* with sherpa-onnx 1.13.0. Versioned release assets include its Apache license.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||
* here is multiple files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base";
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base" | "whisper-small-fr";
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
@@ -134,6 +137,19 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
||||
],
|
||||
totalBytes: 160609290,
|
||||
},
|
||||
"whisper-small-fr": {
|
||||
label: "Whisper (French, small)",
|
||||
dirName: "sherpa-onnx-whisper-small-fr-int8-v1",
|
||||
// Pinned export of bofenghuang/whisper-small-cv11-french; see docs/models/whisper-small-fr.
|
||||
baseUrl: "https://git.undefined-dev.fr/api/packages/Blendman974/generic/faceclaw-asr/whisper-small-fr-v1/",
|
||||
files: [
|
||||
{"name": "small-fr-encoder.int8.onnx", "sha256": "ec34fc42182f37727ad982d254b18a6001f5dcccebc5a36060031995f1782d5d", "sizeBytes": 112413414},
|
||||
{"name": "small-fr-decoder.int8.onnx", "sha256": "b554e35dc8c593b550a82cd01ebb00c1347e8c2b089b780b4c4fe16450cd7dc0", "sizeBytes": 262200764},
|
||||
{"name": "small-fr-tokens.txt", "sha256": "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", "sizeBytes": 816730},
|
||||
{"name": "LICENSE.txt", "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", "sizeBytes": 11358},
|
||||
],
|
||||
totalBytes: 375442266,
|
||||
},
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
@@ -159,6 +175,7 @@ const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||
moonshine: freshRuntime(),
|
||||
"whisper-base-en": freshRuntime(),
|
||||
"whisper-base": freshRuntime(),
|
||||
"whisper-small-fr": freshRuntime(),
|
||||
};
|
||||
|
||||
function modelDirPath(id: AsrModelId): string {
|
||||
|
||||
@@ -78,9 +78,10 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
|
||||
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
|
||||
const fallback = selection.provider === "onboard-whisper-fr" ? "fr" : selection.provider === "onboard-whisper" ? "en" : "";
|
||||
const language = selection.provider === "onboard-whisper-fr" ? "fr" : (selection.language ?? fallback).trim().toLowerCase();
|
||||
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
|
||||
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
|
||||
return `${selection.provider}:${language}${model}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
|
||||
@@ -14,6 +14,7 @@ export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPT
|
||||
export function resolveTranscriptionLanguage(
|
||||
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
|
||||
): string {
|
||||
if (provider === "onboard-whisper-fr") return "fr";
|
||||
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
|
||||
if (provider === "onboard") return "";
|
||||
if (choice === "default") {
|
||||
|
||||
@@ -32,7 +32,7 @@ export type VoiceControlState = {
|
||||
// renamed, since it's a persisted settings value on real installs already.
|
||||
// "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP
|
||||
// (whisper-server-stt.ts).
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "onboard-whisper-fr" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
@@ -397,8 +397,10 @@ export class FaceclawVoiceControlBridge {
|
||||
this.started = true;
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
this.controller?.setOnboardLanguage(options.language ?? "en");
|
||||
const frenchWhisper = options.provider === "onboard-whisper-fr";
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" || frenchWhisper ? "whisper" : "moonshine");
|
||||
this.controller?.setFrenchWhisper(frenchWhisper);
|
||||
this.controller?.setOnboardLanguage(frenchWhisper ? "fr" : options.language ?? "en");
|
||||
const captureId = this.nextNativeCaptureId++;
|
||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||
this.controller?.start("onboard", captureId);
|
||||
|
||||
@@ -1078,13 +1078,14 @@ export function effectiveRingConnectionMode(): RingConnectionMode {
|
||||
// keeps its name (it's a persisted setting on real installs) but its label
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
// "whisper-server" is a self-hosted OpenAI-compatible Whisper (e.g. vLLM).
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "onboard-whisper-fr" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
|
||||
// On iOS "onboard" is Apple's on-device recognizer, and there is no
|
||||
// "onboard-whisper" (the sherpa-onnx models are Android-only).
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)",
|
||||
"onboard-whisper": "On-device (Whisper)",
|
||||
"onboard-whisper-fr": "On-device (Whisper French)",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
soniox: "Soniox",
|
||||
@@ -1098,7 +1099,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
defaultValue: "onboard",
|
||||
values: global.isIOS
|
||||
? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"]
|
||||
: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"],
|
||||
: ["onboard", "onboard-whisper", "onboard-whisper-fr", "elevenlabs", "whisper", "soniox", "whisper-server"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
@@ -1109,7 +1110,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
},
|
||||
description: global.isIOS
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The On-device options need their voice model downloaded (below) and never leave the phone. Whisper French uses the French-only small model and always transcribes in French.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
@@ -1351,7 +1352,7 @@ export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>
|
||||
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
|
||||
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
|
||||
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
|
||||
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.",
|
||||
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. The dedicated Whisper French provider always uses French, without changing this preference. Moonshine remains English-only.",
|
||||
});
|
||||
|
||||
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
|
||||
|
||||
@@ -173,6 +173,7 @@ function settingsSections(): SettingsSection[] {
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
asrModelMenuItem("whisper-base"),
|
||||
asrModelMenuItem("whisper-small-fr"),
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user