From 2d717a9cdb5eeffcacdc00fc6b56f3a99b5ace2f Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 1 Sep 2026 22:27:48 -0400 Subject: [PATCH] Add on-device Whisper as a second offline transcription option sherpa-onnx's native library already reads an offline Whisper config (verified against sherpa-onnx/jni/offline-recognizer.cc at v1.13.0); only the Java wrapper was a package-private stub. This adds a public OfflineWhisperModelConfig builder and wires base.en (int8) in as a second on-device model next to Moonshine. - FaceclawVoiceController: setOnboardModelKind() selects the model before start("onboard"); defaults to Moonshine. Whisper skips the 700ms live-partial redecode (each call re-encodes the whole utterance plus ~10s of tail padding) and gates near-silent segments before decode. - asr-model.ts: ASR_MODELS registry with per-model download state. - Settings > Voice: "On-device (Moonshine)", "On-device (Whisper)", and "OpenAI (Whisper)" for the existing cloud provider (stored value unchanged); one model row per on-device model. - Microphones live captions pass "moonshine" explicitly (unchanged). Co-Authored-By: Claude Sonnet 5 Co-Authored-By: Claude Opus 5 --- .../faceclaw/app/FaceclawVoiceController.java | 124 ++++++-- .../k2fsa/sherpa/onnx/OfflineModelConfig.java | 17 +- .../onnx/OfflineWhisperModelConfig.java | 114 ++++++++ app/apps/microphones/mic-session.ts | 5 +- app/apps/microphones/microphones-app.ts | 9 +- app/native/asr-model.ts | 267 +++++++++++------- app/native/voice-control.ts | 12 +- app/ui/dashboard-settings.ts | 17 +- app/ui/dashboard/settings-menus.ts | 56 ++-- 9 files changed, 459 insertions(+), 162 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 12680324..d6b8171d 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -13,6 +13,7 @@ import com.k2fsa.sherpa.onnx.OfflineRecognizer; import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; import com.k2fsa.sherpa.onnx.OfflineStream; +import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig; import java.io.File; import java.nio.charset.StandardCharsets; @@ -61,12 +62,43 @@ public class FaceclawVoiceController { "decoder_model_merged.ort", "tokens.txt" }; + // Second on-device model: sherpa-onnx's offline Whisper backend (base.en, + // int8-quantized -- see the model-choice note in asr-model.ts). Directory + // shared with the TS-side download flow, same convention as ASR_MODEL_DIR. + // + // On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0) + // runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of + // zero tail padding, capped at 30s. Re-decoding on Moonshine's + // TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that + // encode, growing with the utterance, roughly 1.4x/second while the user is + // still speaking, so onboardModelKind gates that loop off for Whisper -- + // see processRecognizer(). Whisper is also known to hallucinate text on + // near-silent input; recognizeTranscriptSegment() gates that too. + private static final String ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"; + private static final String[] ASR_WHISPER_MODEL_FILES = { + "base.en-encoder.int8.onnx", + "base.en-decoder.int8.onnx", + "base.en-tokens.txt" + }; + // Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f + // buffer) a segment is treated as silence and never reaches the Whisper + // recognizer at all, rather than risking a hallucinated non-answer. Picked + // conservatively low (well under typical mic noise floor already seen in + // this pipeline's normalization target) -- UNTESTED on real hardware, tune + // against real glasses captures rather than trusting this number. + private static final float WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f; private enum VoiceInputMode { - ONBOARD, // on-phone Moonshine transcription + ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind) CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side } + /** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */ + private enum OnboardModelKind { + MOONSHINE, + WHISPER + } + private final Context appContext; private final Handler mainHandler = new Handler(Looper.getMainLooper()); private final Object lock = new Object(); @@ -86,6 +118,7 @@ public class FaceclawVoiceController { private volatile boolean usePhoneMic; private boolean activePhoneMic; private VoiceInputMode mode = VoiceInputMode.CLOUD; + private volatile OnboardModelKind onboardModelKind = OnboardModelKind.MOONSHINE; private OfflineRecognizer recognizer; private FaceclawLc3Decoder lc3Decoder; private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES]; @@ -150,6 +183,17 @@ public class FaceclawVoiceController { this.saveRecordings = saveRecordings; } + /** + * Which on-device model {@link #start}("onboard") should load: "whisper" + * selects the second on-device model (sherpa-onnx offline Whisper); any + * other value (including null/absent) keeps the existing Moonshine model, + * so callers that never call this see unchanged behavior. Must be set + * before {@link #start}; has no effect in CLOUD mode. + */ + public void setOnboardModelKind(String kind) { + this.onboardModelKind = "whisper".equals(kind) ? OnboardModelKind.WHISPER : OnboardModelKind.MOONSHINE; + } + /** * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the * speaker stops. Used by hands-free ("Hey Even") capture, which has no @@ -310,14 +354,15 @@ public class FaceclawVoiceController { try { deleteLegacyKwsFiles(); VoiceInputMode currentMode = mode; + OnboardModelKind currentOnboardKind = onboardModelKind; if (currentMode == VoiceInputMode.ONBOARD) { - File modelDir = findAsrModelDir(); + File modelDir = findAsrModelDir(currentOnboardKind); if (modelDir == null) { emitStatus("Voice model not downloaded (see Settings > Voice)."); return; } emitStatus("Loading transcription model..."); - recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir)); + recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind)); resetTranscriptState(); lastTranscript = ""; } @@ -453,32 +498,48 @@ public class FaceclawVoiceController { return b.array(); } - private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) { + private OfflineRecognizerConfig buildRecognizerConfig(File modelDir, OnboardModelKind kind) { + OfflineModelConfig.Builder modelConfig = OfflineModelConfig.builder() + .setNumThreads(1); + if (kind == OnboardModelKind.WHISPER) { + modelConfig + .setWhisper(OfflineWhisperModelConfig.builder() + .setEncoder(new File(modelDir, "base.en-encoder.int8.onnx").getAbsolutePath()) + .setDecoder(new File(modelDir, "base.en-decoder.int8.onnx").getAbsolutePath()) + .setLanguage("en") + .setTask("transcribe") + .build()) + .setTokens(new File(modelDir, "base.en-tokens.txt").getAbsolutePath()); + } else { + modelConfig + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) + .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) + .build()) + .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()); + } return OfflineRecognizerConfig.builder() .setFeatureConfig(FeatureConfig.builder() .setSampleRate(SAMPLE_RATE) .setFeatureDim(FEATURE_DIM) .build()) - .setModelConfig(OfflineModelConfig.builder() - .setMoonshine(OfflineMoonshineModelConfig.builder() - .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) - .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) - .build()) - .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) - .setNumThreads(1) - .build()) + .setModelConfig(modelConfig.build()) .build(); } /** - * The Moonshine model directory, populated by the download flow in - * asr-model.ts (releases before 0.5.0 copied the same files out of the - * APK, so upgraded installs are already complete). Null when any file is + * The on-device model directory for the given kind, populated by the + * download flow in asr-model.ts (releases before 0.5.0 copied the + * Moonshine files out of the APK directly, so upgraded installs are + * already complete for that model). Null when any expected file is * missing, i.e. the model still needs to be downloaded. */ - private File findAsrModelDir() { - File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR); - for (String fileName : ASR_MODEL_FILES) { + private File findAsrModelDir(OnboardModelKind kind) { + boolean whisper = kind == OnboardModelKind.WHISPER; + String dirName = whisper ? ASR_WHISPER_MODEL_DIR : ASR_MODEL_DIR; + String[] fileNames = whisper ? ASR_WHISPER_MODEL_FILES : ASR_MODEL_FILES; + File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + dirName); + for (String fileName : fileNames) { File file = new File(modelDir, fileName); if (!file.exists() || file.length() == 0) { return null; @@ -658,6 +719,16 @@ public class FaceclawVoiceController { private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); + // Each Whisper call re-encodes the whole buffer plus ~10s of tail + // padding (see the ASR_WHISPER_* comment above), + // so it skips the live-partial redecode Moonshine does on this + // interval and only decodes when a segment commits (8s buffer fill) + // or the utterance ends (decodeTranscript(true) in runLoop()). This + // means no live preview text while speaking in Whisper mode -- status + // stays "Listening..." until release. This is a deliberate tradeoff. + if (onboardModelKind == OnboardModelKind.WHISPER) { + return; + } long now = SystemClock.elapsedRealtime(); if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { @@ -767,6 +838,14 @@ public class FaceclawVoiceController { return ""; } float[] segment = Arrays.copyOf(transcriptSamples, sampleCount); + // Whisper hallucinates text on near-silent input (a known quirk of the + // model, not this pipeline); gate it on the PRE-normalization peak, since + // normalizePeak() below would otherwise amplify true silence right up to + // the target level and hide the very thing being checked for. Moonshine + // does not share this failure mode in practice, so it is left unchanged. + if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) { + return ""; + } normalizePeak(segment); OfflineStream offlineStream = currentRecognizer.createStream(); try { @@ -780,7 +859,7 @@ public class FaceclawVoiceController { } } - private static void normalizePeak(float[] samples) { + private static float peakAmplitude(float[] samples) { float peak = 0f; for (float s : samples) { float a = Math.abs(s); @@ -788,6 +867,11 @@ public class FaceclawVoiceController { peak = a; } } + return peak; + } + + private static void normalizePeak(float[] samples) { + float peak = peakAmplitude(samples); if (peak <= 0f) { return; } @@ -805,7 +889,7 @@ public class FaceclawVoiceController { (committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE; String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS ? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."; - Log.i(TAG, "Moonshine decode final=" + isFinal + Log.i(TAG, (onboardModelKind == OnboardModelKind.WHISPER ? "Whisper" : "Moonshine") + " decode final=" + isFinal + " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec) + " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE) + " textLen=" + text.length() + " text=\"" + preview + "\""); diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java index 0d916bf8..ee130387 100644 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java @@ -162,7 +162,7 @@ public class OfflineModelConfig { public static class Builder { private OfflineTransducerModelConfig transducer = new OfflineTransducerModelConfig(); private OfflineParaformerModelConfig paraformer = new OfflineParaformerModelConfig(); - private OfflineWhisperModelConfig whisper = new OfflineWhisperModelConfig(); + private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build(); private OfflineFireRedAsrModelConfig fireRedAsr = new OfflineFireRedAsrModelConfig(); private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build(); private OfflineNemoEncDecCtcModelConfig nemo = new OfflineNemoEncDecCtcModelConfig(); @@ -191,6 +191,11 @@ public class OfflineModelConfig { return this; } + public Builder setWhisper(OfflineWhisperModelConfig whisper) { + this.whisper = whisper; + return this; + } + public Builder setTokens(String tokens) { this.tokens = tokens; return this; @@ -228,16 +233,6 @@ class OfflineParaformerModelConfig { private final QnnConfig qnnConfig = new QnnConfig(); } -class OfflineWhisperModelConfig { - private final String encoder = ""; - private final String decoder = ""; - private final String language = "en"; - private final String task = "transcribe"; - private final int tailPaddings = 1000; - private final boolean enableTokenTimestamps = false; - private final boolean enableSegmentTimestamps = false; -} - class OfflineFireRedAsrModelConfig { private final String encoder = ""; private final String decoder = ""; diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java new file mode 100644 index 00000000..51245a2b --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java @@ -0,0 +1,114 @@ +package com.k2fsa.sherpa.onnx; + +/** + * Config for sherpa-onnx's offline (non-streaming) Whisper backend. + * + * Field names and types here are load-bearing: the native JNI glue reads + * them by name via reflection (GetFieldID), not through the getters below, + * so they must match sherpa-onnx's own OfflineWhisperModelConfig exactly + * (verified against sherpa-onnx/jni/offline-recognizer.cc at the pinned + * release tag: it unconditionally reads encoder, decoder, language, task, + * tailPaddings, enableTokenTimestamps and enableSegmentTimestamps off the + * OfflineModelConfig's "whisper" field). This mirrors the + * OfflineMoonshineModelConfig.java pattern already used in this package. + */ +public class OfflineWhisperModelConfig { + private final String encoder; + private final String decoder; + private final String language; + private final String task; + private final int tailPaddings; + private final boolean enableTokenTimestamps; + private final boolean enableSegmentTimestamps; + + private OfflineWhisperModelConfig(Builder builder) { + this.encoder = builder.encoder; + this.decoder = builder.decoder; + this.language = builder.language; + this.task = builder.task; + this.tailPaddings = builder.tailPaddings; + this.enableTokenTimestamps = builder.enableTokenTimestamps; + this.enableSegmentTimestamps = builder.enableSegmentTimestamps; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEncoder() { + return encoder; + } + + public String getDecoder() { + return decoder; + } + + public String getLanguage() { + return language; + } + + public String getTask() { + return task; + } + + public int getTailPaddings() { + return tailPaddings; + } + + public boolean getEnableTokenTimestamps() { + return enableTokenTimestamps; + } + + public boolean getEnableSegmentTimestamps() { + return enableSegmentTimestamps; + } + + public static class Builder { + private String encoder = ""; + private String decoder = ""; + private String language = "en"; + private String task = "transcribe"; + private int tailPaddings = 1000; + private boolean enableTokenTimestamps = false; + private boolean enableSegmentTimestamps = false; + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setDecoder(String decoder) { + this.decoder = decoder; + return this; + } + + public Builder setLanguage(String language) { + this.language = language; + return this; + } + + public Builder setTask(String task) { + this.task = task; + return this; + } + + public Builder setTailPaddings(int tailPaddings) { + this.tailPaddings = tailPaddings; + return this; + } + + public Builder setEnableTokenTimestamps(boolean enableTokenTimestamps) { + this.enableTokenTimestamps = enableTokenTimestamps; + return this; + } + + public Builder setEnableSegmentTimestamps(boolean enableSegmentTimestamps) { + this.enableSegmentTimestamps = enableSegmentTimestamps; + return this; + } + + public OfflineWhisperModelConfig build() { + return new OfflineWhisperModelConfig(this); + } + } +} diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index e2d9cf5b..5f019ab8 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -730,7 +730,10 @@ class MicSession { }, }); engine.setListener(this.captionListenerProxy); - engine.setAsrModelDir(isAsrModelReady() ? moonshineModelDir() : null); + // Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine) + // that only ever uses Moonshine; unaffected by the two-model on-device + // Whisper addition in native/voice-control.ts's push-to-talk path. + engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null); engine.setSpeakerModelPath(micModelPath("speaker-embedding")); engine.start(); this.captionEngine = engine; diff --git a/app/apps/microphones/microphones-app.ts b/app/apps/microphones/microphones-app.ts index d4a0b0ee..ef0d491e 100644 --- a/app/apps/microphones/microphones-app.ts +++ b/app/apps/microphones/microphones-app.ts @@ -328,17 +328,20 @@ function voiceSpeakersMenu(): MenuLayer { () => startMicModelDownload("speaker-embedding"), ), modelItem( + // Live captions always use Moonshine (see the matching note in + // mic-session.ts); the second on-device model (Whisper) is a + // push-to-talk dictation option only, managed in Settings > Voice. "Caption ASR model", - isAsrModelReady, + () => isAsrModelReady("moonshine"), () => { - const state = asrModelState(); + const state = asrModelState("moonshine"); if (state.status === "ready") return "ready"; if (state.status === "downloading") { return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`; } return "download (141 MB)"; }, - startAsrModelDownload, + () => startAsrModelDownload("moonshine"), ), modelItem( "Re-diarization model", diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts index 3c71e31a..591bdc73 100644 --- a/app/native/asr-model.ts +++ b/app/native/asr-model.ts @@ -4,50 +4,107 @@ declare const com: any; declare const java: any; /** - * Download management for the on-device transcription model (Moonshine, via - * sherpa-onnx). The model is no longer bundled in the APK; it is fetched on - * demand into the same filesDir location that earlier releases copied the - * bundled files to, so upgraded installs that already used on-device - * transcription need no download. FaceclawVoiceController reads the files - * from this directory. + * Download management for the on-device transcription models (sherpa-onnx + * offline recognizers). Models are fetched on demand into filesDir rather + * than bundled in the APK; FaceclawVoiceController reads the files from the + * directory each model's `dirName` names. * - * Mirrors the on-phone assistant model flow in llama.ts, except the model is - * three files rather than one; they download sequentially through + * Two models, same shape: + * - "moonshine": the original on-device option (three files). The model is + * no longer bundled in the APK; it is fetched into the same filesDir + * location that earlier releases copied the bundled files to, so + * upgraded installs that already used on-device transcription need no + * download. + * - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en, + * int8-quantized. Picked over tiny.en for materially better accuracy + * (Whisper's own tiny/base WER gap is real and well documented) on the + * reasoning that FaceclawVoiceController.java no longer re-decodes a + * Whisper segment on every live-partial tick — see its ASR_WHISPER_* + * comment — so base.en's extra compute is a one-time cost per utterance, + * not a cost paid ~1.4x/second while the user is speaking. tiny.en is an + * easy swap (this file's numbers only) if base.en proves too slow on a + * real phone; nobody has measured that yet. Files are csukuangfj's own + * (the sherpa-onnx maintainer's) HF mirror of the upstream sherpa-onnx + * release asset, matching the mirror-not-the-GH-tarball approach already + * used for Moonshine below. Every hash here was computed locally from a + * freshly downloaded copy of the file, not copied from a doc page. + * + * Mirrors the on-phone assistant model flow in llama.ts, except each model + * here is multiple files rather than one; they download sequentially through * FaceclawModelDownloader (resume + pinned sha256 per file). */ +export type AsrModelId = "moonshine" | "whisper-base-en"; + type AsrModelFile = { name: string; sha256: string; sizeBytes: number; }; -export const ASR_MODEL = { - label: "Moonshine (English)", - dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27", - // Hugging Face mirror of the sherpa-onnx release asset of the same name. - // The GitHub release only offers a tar.bz2, which the phone can't unpack; - // this mirror serves the same files (sha256-verified) individually. - baseUrl: - "https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/", - files: [ - { - name: "decoder_model_merged.ort", - sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d", - sizeBytes: 109424400, - }, - { - name: "encoder_model.ort", - sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf", - sizeBytes: 31326816, - }, - { - name: "tokens.txt", - sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049", - sizeBytes: 549350, - }, - ] as AsrModelFile[], - totalBytes: 141300566, +type AsrModelDef = { + label: string; + dirName: string; + baseUrl: string; + files: AsrModelFile[]; + totalBytes: number; +}; + +export const ASR_MODELS: Record = { + moonshine: { + label: "Moonshine (English)", + dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27", + // Hugging Face mirror of the sherpa-onnx release asset of the same name. + // The GitHub release only offers a tar.bz2, which the phone can't unpack; + // this mirror serves the same files (sha256-verified) individually. + baseUrl: + "https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/", + files: [ + { + name: "decoder_model_merged.ort", + sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d", + sizeBytes: 109424400, + }, + { + name: "encoder_model.ort", + sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf", + sizeBytes: 31326816, + }, + { + name: "tokens.txt", + sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049", + sizeBytes: 549350, + }, + ], + totalBytes: 141300566, + }, + "whisper-base-en": { + label: "Whisper (English, base)", + dirName: "sherpa-onnx-whisper-base-en-int8", + // csukuangfj/sherpa-onnx-whisper-base.en on Hugging Face: the sherpa-onnx + // maintainer's own mirror of the project's whisper export, individual + // files (no tar.bz2-unpack problem to begin with, but kept as the same + // per-file-with-pinned-hash shape as Moonshine above for consistency). + baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base.en/resolve/main/", + files: [ + { + name: "base.en-encoder.int8.onnx", + sha256: "ef6b936f4c9b1d90a3b68634b60c4ed8576b26172b33c2535ec0e933c9edb823", + sizeBytes: 29120534, + }, + { + name: "base.en-decoder.int8.onnx", + sha256: "f7162ad6db2dbef16cfaeaa7f945b9d7dd9c1b8d472f6aca82f2273d185e4d41", + sizeBytes: 130669978, + }, + { + name: "base.en-tokens.txt", + sha256: "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930", + sizeBytes: 835554, + }, + ], + totalBytes: 160626066, + }, }; export type AsrModelState = { @@ -56,121 +113,141 @@ export type AsrModelState = { totalBytes: number; }; -let downloader: any = null; -// Bytes of files already fully downloaded in this run, plus progress within -// the file currently downloading; drives the aggregate percentage. -let completedBytes = 0; -let currentFileBytes = 0; -const stateListeners = new Set<(state: AsrModelState) => void>(); +type ModelRuntime = { + downloader: any; + // Bytes of files already fully downloaded in this run, plus progress within + // the file currently downloading; drives the aggregate percentage. + completedBytes: number; + currentFileBytes: number; + stateListeners: Set<(state: AsrModelState) => void>; +}; -function modelDirPath(): string { +function freshRuntime(): ModelRuntime { + return { downloader: null, completedBytes: 0, currentFileBytes: 0, stateListeners: new Set() }; +} + +const runtimes: Record = { + moonshine: freshRuntime(), + "whisper-base-en": freshRuntime(), +}; + +function modelDirPath(id: AsrModelId): string { const context = Utils.android.getApplicationContext(); - return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`; + return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODELS[id].dirName}`; } -function filePath(file: AsrModelFile): string { - return `${modelDirPath()}/${file.name}`; +function filePath(id: AsrModelId, file: AsrModelFile): string { + return `${modelDirPath(id)}/${file.name}`; } -function isFilePresent(file: AsrModelFile): boolean { +function isFilePresent(id: AsrModelId, file: AsrModelFile): boolean { try { - const javaFile = new java.io.File(filePath(file)); + const javaFile = new java.io.File(filePath(id, file)); return javaFile.exists() && javaFile.length() > 0; } catch { return false; } } -export function isAsrModelReady(): boolean { +export function isAsrModelReady(id: AsrModelId): boolean { if (!global.isAndroid) return false; - return ASR_MODEL.files.every(isFilePresent); + return ASR_MODELS[id].files.every((file) => isFilePresent(id, file)); } -export function asrModelState(): AsrModelState { - if (downloader) { +export function asrModelState(id: AsrModelId): AsrModelState { + const runtime = runtimes[id]; + const totalBytes = ASR_MODELS[id].totalBytes; + if (runtime.downloader) { return { status: "downloading", - bytesDownloaded: completedBytes + currentFileBytes, - totalBytes: ASR_MODEL.totalBytes, + bytesDownloaded: runtime.completedBytes + runtime.currentFileBytes, + totalBytes, }; } return { - status: isAsrModelReady() ? "ready" : "absent", + status: isAsrModelReady(id) ? "ready" : "absent", bytesDownloaded: 0, - totalBytes: ASR_MODEL.totalBytes, + totalBytes, }; } -export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void { - stateListeners.add(listener); - return () => stateListeners.delete(listener); +export function onAsrModelStateChanged(id: AsrModelId, listener: (state: AsrModelState) => void): () => void { + const runtime = runtimes[id]; + runtime.stateListeners.add(listener); + return () => runtime.stateListeners.delete(listener); } -function notifyStateChanged(): void { - const state = asrModelState(); - stateListeners.forEach((listener) => listener(state)); +function notifyStateChanged(id: AsrModelId): void { + const state = asrModelState(id); + runtimes[id].stateListeners.forEach((listener) => listener(state)); } -export function startAsrModelDownload(): void { - if (!global.isAndroid || downloader || isAsrModelReady()) return; - completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0); - downloadNextFile(); - notifyStateChanged(); +export function startAsrModelDownload(id: AsrModelId): void { + const runtime = runtimes[id]; + if (!global.isAndroid || runtime.downloader || isAsrModelReady(id)) return; + runtime.completedBytes = ASR_MODELS[id].files + .filter((file) => isFilePresent(id, file)) + .reduce((sum, f) => sum + f.sizeBytes, 0); + downloadNextFile(id); + notifyStateChanged(id); } -function downloadNextFile(): void { - const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file)); +function downloadNextFile(id: AsrModelId): void { + const runtime = runtimes[id]; + const def = ASR_MODELS[id]; + const nextFile = def.files.find((file) => !isFilePresent(id, file)); if (!nextFile) { - downloader = null; - notifyStateChanged(); + runtime.downloader = null; + notifyStateChanged(id); return; } - currentFileBytes = 0; + runtime.currentFileBytes = 0; const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({ onProgress: (bytes: number, _total: number) => { - currentFileBytes = Number(bytes); - notifyStateChanged(); + runtime.currentFileBytes = Number(bytes); + notifyStateChanged(id); }, onDone: () => { - completedBytes += nextFile.sizeBytes; - currentFileBytes = 0; - downloadNextFile(); + runtime.completedBytes += nextFile.sizeBytes; + runtime.currentFileBytes = 0; + downloadNextFile(id); }, onError: (message: string) => { - console.error(`Voice model download failed (${nextFile.name}): ${message}`); - downloader = null; - notifyStateChanged(); + console.error(`Voice model download failed (${id}/${nextFile.name}): ${message}`); + runtime.downloader = null; + notifyStateChanged(id); }, }); - downloader = new com.faceclaw.app.FaceclawModelDownloader( - `${ASR_MODEL.baseUrl}${nextFile.name}`, - filePath(nextFile), + runtime.downloader = new com.faceclaw.app.FaceclawModelDownloader( + `${def.baseUrl}${nextFile.name}`, + filePath(id, nextFile), nextFile.sha256, nextFile.sizeBytes, listener, ); - downloader.start(); + runtime.downloader.start(); } /** Stops the download; already-fetched bytes are kept and resumed next time. */ -export function cancelAsrModelDownload(): void { - if (!downloader) return; - downloader.cancel(); - downloader = null; - notifyStateChanged(); +export function cancelAsrModelDownload(id: AsrModelId): void { + const runtime = runtimes[id]; + if (!runtime.downloader) return; + runtime.downloader.cancel(); + runtime.downloader = null; + notifyStateChanged(id); } -export function deleteAsrModel(): void { +export function deleteAsrModel(id: AsrModelId): void { if (!global.isAndroid) return; - cancelAsrModelDownload(); + cancelAsrModelDownload(id); try { - for (const file of ASR_MODEL.files) { - new java.io.File(filePath(file)).delete(); - new java.io.File(`${filePath(file)}.part`).delete(); + for (const file of ASR_MODELS[id].files) { + new java.io.File(filePath(id, file)).delete(); + new java.io.File(`${filePath(id, file)}.part`).delete(); } - new java.io.File(modelDirPath()).delete(); + new java.io.File(modelDirPath(id)).delete(); } catch (error) { - console.error(`Voice model delete failed: ${String(error)}`); + console.error(`Voice model delete failed (${id}): ${String(error)}`); } - notifyStateChanged(); + notifyStateChanged(id); } diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 2874d5a9..acfd9f31 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -26,7 +26,12 @@ export type VoiceControlState = { detail: string; }; -export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox"; +// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper, +// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model +// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx +// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than +// renamed, since it's a persisted settings value on real installs already. +export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox"; export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** @@ -314,6 +319,9 @@ export class FaceclawVoiceControlBridge { this.cloudClient = null; this.started = true; + // Which on-device model to load; a no-op setter for every provider except + // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). + this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); this.controller?.start("onboard"); } @@ -323,7 +331,7 @@ export class FaceclawVoiceControlBridge { * than failing the capture outright. */ private createCloudClient(options: PushToTalkOptions): CloudSttClient | null { - if (options.provider === "onboard") return null; + if (options.provider === "onboard" || options.provider === "onboard-whisper") return null; const sttOptions = { apiKey: "", onTranscript: (event: CloudSttTranscriptEvent) => diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 5ac7ba8e..aad0019e 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -523,12 +523,19 @@ export const ringConnectionModeSetting = new ConfigSettingEnum = { - onboard: "On-device", + onboard: "On-device (Moonshine)", + "onboard-whisper": "On-device (Whisper)", elevenlabs: "ElevenLabs", - whisper: "Whisper", + whisper: "OpenAI (Whisper)", soniox: "Soniox", }; @@ -537,7 +544,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ label: "Transcription Provider", storageKey: "voice.provider", defaultValue: "onboard", - values: ["onboard", "elevenlabs", "whisper", "soniox"], + values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"], formatValue: (value) => voiceProviderLabels[value] ?? value, isDisabled: (value) => { if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0; @@ -545,7 +552,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0; return false; }, - description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).", + description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.", }); const wakeWordActionLabels: Record = { diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index f67a1066..2dd7a745 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -13,12 +13,13 @@ import { startLocalModelDownload, } from "../../native/llama"; import { - ASR_MODEL, + ASR_MODELS, asrModelState, cancelAsrModelDownload, deleteAsrModel, onAsrModelStateChanged, startAsrModelDownload, + type AsrModelId, } from "../../native/asr-model"; import { TextViewerLayer } from "../../apps/files/text-viewer"; import type { LayerContext } from "../layers"; @@ -119,7 +120,8 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), - asrModelMenuItem(), + asrModelMenuItem("moonshine"), + asrModelMenuItem("whisper-base-en"), ], }, { @@ -344,48 +346,52 @@ function localModelMenuItem(): MenuItem { }; } -const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`; +function asrModelMb(id: AsrModelId): string { + return `${Math.round(ASR_MODELS[id].totalBytes / 1e6)}MB`; +} -let asrModelRenderUnsub: (() => void) | null = null; +const asrModelRenderUnsub: Partial void>> = {}; -function watchAsrModelDownload(ctx: LayerContext): void { - asrModelRenderUnsub?.(); - asrModelRenderUnsub = onAsrModelStateChanged((state) => { +function watchAsrModelDownload(id: AsrModelId, ctx: LayerContext): void { + asrModelRenderUnsub[id]?.(); + asrModelRenderUnsub[id] = onAsrModelStateChanged(id, (state) => { ctx.actions.requestRender(); if (state.status !== "downloading") { - asrModelRenderUnsub?.(); - asrModelRenderUnsub = null; + asrModelRenderUnsub[id]?.(); + asrModelRenderUnsub[id] = undefined; } }); } -function asrModelStatusText(): string { - const state = asrModelState(); +function asrModelStatusText(id: AsrModelId): string { + const state = asrModelState(id); if (state.status === "ready") return "downloaded"; if (state.status === "downloading") { const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0; - return `${pct}% of ${ASR_MODEL_MB}`; + return `${pct}% of ${asrModelMb(id)}`; } return "not downloaded"; } -/** Download/cancel/delete management for the on-device transcription model. */ -function asrModelMenuItem(): MenuItem { +/** Download/cancel/delete management for one on-device transcription model. */ +function asrModelMenuItem(id: AsrModelId): MenuItem { + const def = ASR_MODELS[id]; + const rowLabel = `On-device model: ${def.label}`; return { - label: "On-device voice model", + label: rowLabel, description: - `${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` + + `${def.label} (${asrModelMb(id)} download). ` + "Transcribes voice input on the phone itself, with no API key or cloud service. " + - "Required for the On-device transcription provider; the cloud providers work without it. " + + "Required for its matching Transcription Provider option; the other providers work without it. " + "An interrupted download resumes where it left off.", onSelect: (ctx) => { - const state = asrModelState(); + const state = asrModelState(id); const action: MenuItem = state.status === "downloading" ? { label: "Cancel download", onSelect: (innerCtx) => { - cancelAsrModelDownload(); + cancelAsrModelDownload(id); innerCtx.stack.pop(); }, } @@ -393,22 +399,22 @@ function asrModelMenuItem(): MenuItem { ? { label: "Delete model", onSelect: (innerCtx) => { - deleteAsrModel(); + deleteAsrModel(id); innerCtx.stack.pop(); }, } : { - label: `Download (${ASR_MODEL_MB})`, + label: `Download (${asrModelMb(id)})`, onSelect: (innerCtx) => { - startAsrModelDownload(); - watchAsrModelDownload(innerCtx); + startAsrModelDownload(id); + watchAsrModelDownload(id, innerCtx); innerCtx.stack.pop(); }, }; - openModalMenu(ctx, "On-device voice model", [action], 0); + openModalMenu(ctx, rowLabel, [action], 0); }, render: ({ image, x, y, width }) => { - drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText()); + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, rowLabel, asrModelStatusText(id)); }, }; }