forked from Blendman974/faceclaw
> Most serious fixes > - Recordings kept forever: a recording made with captions off, or with "Save captions" > off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so > the phone UI never showed it and the retention settings never deleted it. Recording > now only happens alongside a saved caption session, and turning captions off ends > that session's recording properly. > - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio > forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic > array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends > the config when it has been lost. The normal single-stream mode already recovered on > its own. > - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream > but never switched to the normal stream. Setup changes now take effect live, > including switching between the two modes. > - Setup changes were undone: after the assistant closed, the app re-sent the config it > loaded at launch to the glasses, reverting any changes you'd made since. > - Startup failures: if the caption engine failed to start, the session got stuck > half-running. If the glasses weren't connected, listeners piled up and an empty > conversation could be left open. Startup now brings up the microphone first and > cleans up properly on failure. > > Other fixes > - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms > frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test > that fails on the old code. > - Stale direction data: an arm that stopped sending kept steering the direction > estimate and could hold the caption feed on a silent arm. > - Compass: closing Microphones switched off the Compass app's compass, because both > used the same owner name. > - Speaker database failures: > - A failed speaker reload wiped the in-memory list, so the next voice became a > duplicate "Speaker N". > - A failed speaker insert crashed caption handling; in re-diarize it reassigned > transcript lines to a nonexistent speaker. > - Translation: > - A crash path inside the model-download callback is now caught. > - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of > old-style language codes. > - Ask questions: once the speaker list grew past the size limit (every unknown voice > adds a "Speaker N"), it pushed all transcript lines out of the context sent to the > model. When trimming is needed, the oldest lines now go first. > - Smaller fixes: > - The captions view stopped jumping back to live speech after 60 lines. > - "This is Dana calling…" was treated as introducing someone else. > - A model download that failed to start was stuck at "downloading" until restart. > - The developer mic monitor grew without limit while hidden.
215 lines
6.4 KiB
TypeScript
215 lines
6.4 KiB
TypeScript
import { Utils } from "@nativescript/core";
|
|
|
|
declare const com: any;
|
|
declare const java: any;
|
|
|
|
/**
|
|
* Download management for the Microphones app's on-device models, mirroring
|
|
* the Moonshine flow in app/native/asr-model.ts (FaceclawModelDownloader,
|
|
* resume + pinned sha256 per file):
|
|
*
|
|
* - Speaker embedding (WeSpeaker CAM++ English, 512-dim): voice-prints for
|
|
* caption attribution, wearer recognition, and re-diarization.
|
|
* - Pyannote segmentation 3.0: offline re-diarization of saved recordings.
|
|
*
|
|
* The caption ASR model itself is the shared Moonshine download managed by
|
|
* asr-model.ts; this module only adds the speaker models.
|
|
*/
|
|
|
|
export type MicModelId = "speaker-embedding" | "diarization-segmentation";
|
|
|
|
type MicModelFile = {
|
|
/** Local file name under the model directory. */
|
|
name: string;
|
|
/** Source file name in the mirror repo (may differ from the local name). */
|
|
remoteName: string;
|
|
sha256: string;
|
|
sizeBytes: number;
|
|
};
|
|
|
|
type MicModel = {
|
|
id: MicModelId;
|
|
label: string;
|
|
baseUrl: string;
|
|
files: MicModelFile[];
|
|
totalBytes: number;
|
|
};
|
|
|
|
// Hugging Face mirrors of the sherpa-onnx release assets (served per-file,
|
|
// sha256 pinned from the repos' LFS pointers).
|
|
export const MIC_MODELS: MicModel[] = [
|
|
{
|
|
id: "speaker-embedding",
|
|
label: "Speaker voices (WeSpeaker English)",
|
|
baseUrl: "https://huggingface.co/csukuangfj/speaker-embedding-models/resolve/main/",
|
|
files: [
|
|
{
|
|
name: "speaker-embedding.onnx",
|
|
remoteName: "wespeaker_en_voxceleb_CAM%2B%2B.onnx",
|
|
sha256: "c46fad10b5f81e1aa4a60c162714208577093655076c5450f8c469e522ec54ef",
|
|
sizeBytes: 29292684,
|
|
},
|
|
],
|
|
totalBytes: 29292684,
|
|
},
|
|
{
|
|
id: "diarization-segmentation",
|
|
label: "Speaker turns (pyannote segmentation)",
|
|
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-pyannote-segmentation-3-0/resolve/main/",
|
|
files: [
|
|
{
|
|
name: "segmentation.onnx",
|
|
remoteName: "model.onnx",
|
|
sha256: "220ad67ca923bef2fa91f2390c786097bf305bceb5e261d4af67b38e938e1079",
|
|
sizeBytes: 5992913,
|
|
},
|
|
],
|
|
totalBytes: 5992913,
|
|
},
|
|
];
|
|
|
|
export type MicModelState = {
|
|
status: "absent" | "downloading" | "ready";
|
|
bytesDownloaded: number;
|
|
totalBytes: number;
|
|
};
|
|
|
|
const downloaders = new Map<MicModelId, any>();
|
|
const progress = new Map<MicModelId, number>();
|
|
const stateListeners = new Set<(id: MicModelId, state: MicModelState) => void>();
|
|
|
|
function model(id: MicModelId): MicModel {
|
|
const found = MIC_MODELS.find((entry) => entry.id === id);
|
|
if (!found) throw new Error(`unknown mic model ${id}`);
|
|
return found;
|
|
}
|
|
|
|
function modelDirPath(id: MicModelId): string {
|
|
const context = Utils.android.getApplicationContext();
|
|
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-mic-models/${id}`;
|
|
}
|
|
|
|
function filePath(id: MicModelId, file: MicModelFile): string {
|
|
return `${modelDirPath(id)}/${file.name}`;
|
|
}
|
|
|
|
function isFilePresent(id: MicModelId, file: MicModelFile): boolean {
|
|
try {
|
|
const javaFile = new java.io.File(filePath(id, file));
|
|
return javaFile.exists() && javaFile.length() > 0;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export function isMicModelReady(id: MicModelId): boolean {
|
|
if (!global.isAndroid) return false;
|
|
return model(id).files.every((file) => isFilePresent(id, file));
|
|
}
|
|
|
|
/** Absolute path of a ready model's primary file, or null when absent. */
|
|
export function micModelPath(id: MicModelId): string | null {
|
|
if (!isMicModelReady(id)) return null;
|
|
return filePath(id, model(id).files[0]!);
|
|
}
|
|
|
|
export function micModelState(id: MicModelId): MicModelState {
|
|
if (downloaders.has(id)) {
|
|
return {
|
|
status: "downloading",
|
|
bytesDownloaded: progress.get(id) ?? 0,
|
|
totalBytes: model(id).totalBytes,
|
|
};
|
|
}
|
|
return {
|
|
status: isMicModelReady(id) ? "ready" : "absent",
|
|
bytesDownloaded: 0,
|
|
totalBytes: model(id).totalBytes,
|
|
};
|
|
}
|
|
|
|
export function onMicModelStateChanged(
|
|
listener: (id: MicModelId, state: MicModelState) => void,
|
|
): () => void {
|
|
stateListeners.add(listener);
|
|
return () => stateListeners.delete(listener);
|
|
}
|
|
|
|
function notifyStateChanged(id: MicModelId): void {
|
|
const state = micModelState(id);
|
|
stateListeners.forEach((listener) => listener(id, state));
|
|
}
|
|
|
|
export function startMicModelDownload(id: MicModelId): void {
|
|
if (!global.isAndroid || downloaders.has(id) || isMicModelReady(id)) return;
|
|
progress.delete(id);
|
|
downloadNextFile(id);
|
|
notifyStateChanged(id);
|
|
}
|
|
|
|
function downloadNextFile(id: MicModelId): void {
|
|
const definition = model(id);
|
|
const nextFile = definition.files.find((file) => !isFilePresent(id, file));
|
|
if (!nextFile) {
|
|
downloaders.delete(id);
|
|
notifyStateChanged(id);
|
|
return;
|
|
}
|
|
const completedBytes = definition.files
|
|
.filter((file) => isFilePresent(id, file))
|
|
.reduce((sum, file) => sum + file.sizeBytes, 0);
|
|
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
|
|
onProgress: (bytes: number, _total: number) => {
|
|
progress.set(id, completedBytes + Number(bytes));
|
|
notifyStateChanged(id);
|
|
},
|
|
onDone: () => {
|
|
downloadNextFile(id);
|
|
},
|
|
onError: (message: string) => {
|
|
console.error(`Mic model download failed (${id}/${nextFile.name}): ${message}`);
|
|
downloaders.delete(id);
|
|
notifyStateChanged(id);
|
|
},
|
|
});
|
|
try {
|
|
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
|
`${definition.baseUrl}${nextFile.remoteName}`,
|
|
filePath(id, nextFile),
|
|
nextFile.sha256,
|
|
nextFile.sizeBytes,
|
|
listener,
|
|
);
|
|
downloaders.set(id, downloader);
|
|
downloader.start();
|
|
} catch (error) {
|
|
// Without this, a dead entry would pin the model at "downloading".
|
|
console.error(`Mic model download failed to start (${id}/${nextFile.name}): ${String(error)}`);
|
|
downloaders.delete(id);
|
|
notifyStateChanged(id);
|
|
}
|
|
}
|
|
|
|
export function cancelMicModelDownload(id: MicModelId): void {
|
|
const downloader = downloaders.get(id);
|
|
if (!downloader) return;
|
|
downloader.cancel();
|
|
downloaders.delete(id);
|
|
notifyStateChanged(id);
|
|
}
|
|
|
|
export function deleteMicModel(id: MicModelId): void {
|
|
if (!global.isAndroid) return;
|
|
cancelMicModelDownload(id);
|
|
try {
|
|
for (const file of model(id).files) {
|
|
new java.io.File(filePath(id, file)).delete();
|
|
new java.io.File(`${filePath(id, file)}.part`).delete();
|
|
}
|
|
new java.io.File(modelDirPath(id)).delete();
|
|
} catch (error) {
|
|
console.error(`Mic model delete failed: ${String(error)}`);
|
|
}
|
|
notifyStateChanged(id);
|
|
}
|