forked from Blendman974/faceclaw
Add on-device Whisper as a second offline transcription option
sherpa-onnx's native library already reads an offline Whisper config
(verified against sherpa-onnx/jni/offline-recognizer.cc at v1.13.0);
only the Java wrapper was a package-private stub. This adds a public
OfflineWhisperModelConfig builder and wires base.en (int8) in as a
second on-device model next to Moonshine.
- FaceclawVoiceController: setOnboardModelKind() selects the model
before start("onboard"); defaults to Moonshine. Whisper skips the
700ms live-partial redecode (each call re-encodes the whole utterance
plus ~10s of tail padding) and gates near-silent segments before
decode.
- asr-model.ts: ASR_MODELS registry with per-model download state.
- Settings > Voice: "On-device (Moonshine)", "On-device (Whisper)",
and "OpenAI (Whisper)" for the existing cloud provider (stored
value unchanged); one model row per on-device model.
- Microphones live captions pass "moonshine" explicitly (unchanged).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
dcb071621d
commit
2d717a9cdb
9 files changed
+459
-162
No files matched your search
+172
-95
@@ -4,50 +4,107 @@ declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* Download management for the on-device transcription model (Moonshine, via
|
||||
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
|
||||
* demand into the same filesDir location that earlier releases copied the
|
||||
* bundled files to, so upgraded installs that already used on-device
|
||||
* transcription need no download. FaceclawVoiceController reads the files
|
||||
* from this directory.
|
||||
* Download management for the on-device transcription models (sherpa-onnx
|
||||
* offline recognizers). Models are fetched on demand into filesDir rather
|
||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||
* directory each model's `dirName` names.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
|
||||
* three files rather than one; they download sequentially through
|
||||
* Two models, same shape:
|
||||
* - "moonshine": the original on-device option (three files). The model is
|
||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||
* location that earlier releases copied the bundled files to, so
|
||||
* upgraded installs that already used on-device transcription need no
|
||||
* download.
|
||||
* - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en,
|
||||
* int8-quantized. Picked over tiny.en for materially better accuracy
|
||||
* (Whisper's own tiny/base WER gap is real and well documented) on the
|
||||
* reasoning that FaceclawVoiceController.java no longer re-decodes a
|
||||
* Whisper segment on every live-partial tick — see its ASR_WHISPER_*
|
||||
* comment — so base.en's extra compute is a one-time cost per utterance,
|
||||
* not a cost paid ~1.4x/second while the user is speaking. tiny.en is an
|
||||
* easy swap (this file's numbers only) if base.en proves too slow on a
|
||||
* real phone; nobody has measured that yet. Files are csukuangfj's own
|
||||
* (the sherpa-onnx maintainer's) HF mirror of the upstream sherpa-onnx
|
||||
* release asset, matching the mirror-not-the-GH-tarball approach already
|
||||
* used for Moonshine below. Every hash here was computed locally from a
|
||||
* freshly downloaded copy of the file, not copied from a doc page.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||
* here is multiple files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en";
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODEL = {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
] as AsrModelFile[],
|
||||
totalBytes: 141300566,
|
||||
type AsrModelDef = {
|
||||
label: string;
|
||||
dirName: string;
|
||||
baseUrl: string;
|
||||
files: AsrModelFile[];
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
||||
moonshine: {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
],
|
||||
totalBytes: 141300566,
|
||||
},
|
||||
"whisper-base-en": {
|
||||
label: "Whisper (English, base)",
|
||||
dirName: "sherpa-onnx-whisper-base-en-int8",
|
||||
// csukuangfj/sherpa-onnx-whisper-base.en on Hugging Face: the sherpa-onnx
|
||||
// maintainer's own mirror of the project's whisper export, individual
|
||||
// files (no tar.bz2-unpack problem to begin with, but kept as the same
|
||||
// per-file-with-pinned-hash shape as Moonshine above for consistency).
|
||||
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base.en/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "base.en-encoder.int8.onnx",
|
||||
sha256: "ef6b936f4c9b1d90a3b68634b60c4ed8576b26172b33c2535ec0e933c9edb823",
|
||||
sizeBytes: 29120534,
|
||||
},
|
||||
{
|
||||
name: "base.en-decoder.int8.onnx",
|
||||
sha256: "f7162ad6db2dbef16cfaeaa7f945b9d7dd9c1b8d472f6aca82f2273d185e4d41",
|
||||
sizeBytes: 130669978,
|
||||
},
|
||||
{
|
||||
name: "base.en-tokens.txt",
|
||||
sha256: "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
|
||||
sizeBytes: 835554,
|
||||
},
|
||||
],
|
||||
totalBytes: 160626066,
|
||||
},
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
@@ -56,121 +113,141 @@ export type AsrModelState = {
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
let downloader: any = null;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
let completedBytes = 0;
|
||||
let currentFileBytes = 0;
|
||||
const stateListeners = new Set<(state: AsrModelState) => void>();
|
||||
type ModelRuntime = {
|
||||
downloader: any;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
completedBytes: number;
|
||||
currentFileBytes: number;
|
||||
stateListeners: Set<(state: AsrModelState) => void>;
|
||||
};
|
||||
|
||||
function modelDirPath(): string {
|
||||
function freshRuntime(): ModelRuntime {
|
||||
return { downloader: null, completedBytes: 0, currentFileBytes: 0, stateListeners: new Set() };
|
||||
}
|
||||
|
||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||
moonshine: freshRuntime(),
|
||||
"whisper-base-en": freshRuntime(),
|
||||
};
|
||||
|
||||
function modelDirPath(id: AsrModelId): string {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODELS[id].dirName}`;
|
||||
}
|
||||
|
||||
function filePath(file: AsrModelFile): string {
|
||||
return `${modelDirPath()}/${file.name}`;
|
||||
function filePath(id: AsrModelId, file: AsrModelFile): string {
|
||||
return `${modelDirPath(id)}/${file.name}`;
|
||||
}
|
||||
|
||||
function isFilePresent(file: AsrModelFile): boolean {
|
||||
function isFilePresent(id: AsrModelId, file: AsrModelFile): boolean {
|
||||
try {
|
||||
const javaFile = new java.io.File(filePath(file));
|
||||
const javaFile = new java.io.File(filePath(id, file));
|
||||
return javaFile.exists() && javaFile.length() > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isAsrModelReady(): boolean {
|
||||
export function isAsrModelReady(id: AsrModelId): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
return ASR_MODEL.files.every(isFilePresent);
|
||||
return ASR_MODELS[id].files.every((file) => isFilePresent(id, file));
|
||||
}
|
||||
|
||||
export function asrModelState(): AsrModelState {
|
||||
if (downloader) {
|
||||
export function asrModelState(id: AsrModelId): AsrModelState {
|
||||
const runtime = runtimes[id];
|
||||
const totalBytes = ASR_MODELS[id].totalBytes;
|
||||
if (runtime.downloader) {
|
||||
return {
|
||||
status: "downloading",
|
||||
bytesDownloaded: completedBytes + currentFileBytes,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
bytesDownloaded: runtime.completedBytes + runtime.currentFileBytes,
|
||||
totalBytes,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: isAsrModelReady() ? "ready" : "absent",
|
||||
status: isAsrModelReady(id) ? "ready" : "absent",
|
||||
bytesDownloaded: 0,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
totalBytes,
|
||||
};
|
||||
}
|
||||
|
||||
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
|
||||
stateListeners.add(listener);
|
||||
return () => stateListeners.delete(listener);
|
||||
export function onAsrModelStateChanged(id: AsrModelId, listener: (state: AsrModelState) => void): () => void {
|
||||
const runtime = runtimes[id];
|
||||
runtime.stateListeners.add(listener);
|
||||
return () => runtime.stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
function notifyStateChanged(): void {
|
||||
const state = asrModelState();
|
||||
stateListeners.forEach((listener) => listener(state));
|
||||
function notifyStateChanged(id: AsrModelId): void {
|
||||
const state = asrModelState(id);
|
||||
runtimes[id].stateListeners.forEach((listener) => listener(state));
|
||||
}
|
||||
|
||||
export function startAsrModelDownload(): void {
|
||||
if (!global.isAndroid || downloader || isAsrModelReady()) return;
|
||||
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile();
|
||||
notifyStateChanged();
|
||||
export function startAsrModelDownload(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
if (!global.isAndroid || runtime.downloader || isAsrModelReady(id)) return;
|
||||
runtime.completedBytes = ASR_MODELS[id].files
|
||||
.filter((file) => isFilePresent(id, file))
|
||||
.reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
|
||||
function downloadNextFile(): void {
|
||||
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
|
||||
function downloadNextFile(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
const def = ASR_MODELS[id];
|
||||
const nextFile = def.files.find((file) => !isFilePresent(id, file));
|
||||
if (!nextFile) {
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
return;
|
||||
}
|
||||
currentFileBytes = 0;
|
||||
runtime.currentFileBytes = 0;
|
||||
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
|
||||
onProgress: (bytes: number, _total: number) => {
|
||||
currentFileBytes = Number(bytes);
|
||||
notifyStateChanged();
|
||||
runtime.currentFileBytes = Number(bytes);
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
onDone: () => {
|
||||
completedBytes += nextFile.sizeBytes;
|
||||
currentFileBytes = 0;
|
||||
downloadNextFile();
|
||||
runtime.completedBytes += nextFile.sizeBytes;
|
||||
runtime.currentFileBytes = 0;
|
||||
downloadNextFile(id);
|
||||
},
|
||||
onError: (message: string) => {
|
||||
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
console.error(`Voice model download failed (${id}/${nextFile.name}): ${message}`);
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
});
|
||||
downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${ASR_MODEL.baseUrl}${nextFile.name}`,
|
||||
filePath(nextFile),
|
||||
runtime.downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${def.baseUrl}${nextFile.name}`,
|
||||
filePath(id, nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloader.start();
|
||||
runtime.downloader.start();
|
||||
}
|
||||
|
||||
/** Stops the download; already-fetched bytes are kept and resumed next time. */
|
||||
export function cancelAsrModelDownload(): void {
|
||||
if (!downloader) return;
|
||||
downloader.cancel();
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
export function cancelAsrModelDownload(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
if (!runtime.downloader) return;
|
||||
runtime.downloader.cancel();
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
|
||||
export function deleteAsrModel(): void {
|
||||
export function deleteAsrModel(id: AsrModelId): void {
|
||||
if (!global.isAndroid) return;
|
||||
cancelAsrModelDownload();
|
||||
cancelAsrModelDownload(id);
|
||||
try {
|
||||
for (const file of ASR_MODEL.files) {
|
||||
new java.io.File(filePath(file)).delete();
|
||||
new java.io.File(`${filePath(file)}.part`).delete();
|
||||
for (const file of ASR_MODELS[id].files) {
|
||||
new java.io.File(filePath(id, file)).delete();
|
||||
new java.io.File(`${filePath(id, file)}.part`).delete();
|
||||
}
|
||||
new java.io.File(modelDirPath()).delete();
|
||||
new java.io.File(modelDirPath(id)).delete();
|
||||
} catch (error) {
|
||||
console.error(`Voice model delete failed: ${String(error)}`);
|
||||
console.error(`Voice model delete failed (${id}): ${String(error)}`);
|
||||
}
|
||||
notifyStateChanged();
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
@@ -26,7 +26,12 @@ export type VoiceControlState = {
|
||||
detail: string;
|
||||
};
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||
// renamed, since it's a persisted settings value on real installs already.
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
@@ -314,6 +319,9 @@ export class FaceclawVoiceControlBridge {
|
||||
|
||||
this.cloudClient = null;
|
||||
this.started = true;
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
this.controller?.start("onboard");
|
||||
}
|
||||
|
||||
@@ -323,7 +331,7 @@ export class FaceclawVoiceControlBridge {
|
||||
* than failing the capture outright.
|
||||
*/
|
||||
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
|
||||
if (options.provider === "onboard") return null;
|
||||
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
|
||||
Reference in new issue
Block a user