Files
faceclaw/app/apps/microphones/speakers.ts
T
James Babcock b50898ea02 Fix a bunch of microphone-related bugs
> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
2026-10-14 00:56:54 -07:00

316 lines
11 KiB
TypeScript

import { Utils } from "@nativescript/core";
import { encodeBase64 } from "../../native/cloud-stt";
import { toUint8Array } from "../../util/array-util";
import { micModelPath } from "./mic-models";
import { wearerCommandsOnlySetting } from "./mic-settings";
declare const com: any;
declare const android: any;
/**
* Speaker voice-print registry over FaceclawConversationStore. Matching is
* cosine similarity on L2-normalized embeddings: >= 0.80 auto-assign ("known"), 0.50-0.80 "uncertain" (assigned
* but flagged for review), below that a new speaker is enrolled. Centroids
* update as a capped running mean so one long session cannot drown an
* enrolled voice.
*/
export const SPEAKER_KNOWN_THRESHOLD = 0.8;
export const SPEAKER_UNCERTAIN_THRESHOLD = 0.5;
/** Cap on the running-mean weight of an existing centroid. */
export const CENTROID_UPDATE_CAP = 24;
const SPEAKER_COLORS = [
"#4FC3F7",
"#FFB74D",
"#81C784",
"#E57373",
"#BA68C8",
"#FFD54F",
"#4DB6AC",
"#F06292",
"#A1887F",
"#90A4AE",
];
export type SpeakerProfile = {
id: number;
name: string;
color: string;
isWearer: boolean;
embedding: Float32Array | null;
embeddingCount: number;
lastHeardAt: number;
tag: string;
segmentCount: number;
/** LLM recap of this speaker's most recent conversation ("" until one is generated). */
lastRecap: string;
/** Open action items from that conversation. */
actionItems: string[];
/** Accumulated inferred facts about this person. */
facts: string[];
insightsUpdatedAt: number;
insightsSessionId: number;
};
export type SpeakerMatch = {
profile: SpeakerProfile;
similarity: number;
classification: "known" | "uncertain" | "new";
/**
* The profile's last_heard_at BEFORE this utterance updated it — i.e. when
* this person's voice was previously heard. 0 for a newly enrolled speaker.
*/
previousLastHeardAt: number;
};
function store(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
/** Parse a stored JSON string array leniently; anything malformed reads as empty. */
function parseStringArray(json: unknown): string[] {
try {
const parsed = JSON.parse(String(json || "[]"));
return Array.isArray(parsed) ? parsed.map((item) => String(item)).filter(Boolean) : [];
} catch {
return [];
}
}
function decodeEmbedding(base64: string): Float32Array | null {
if (!base64) return null;
try {
const bytes = toUint8Array(android.util.Base64.decode(base64, android.util.Base64.NO_WRAP));
if (bytes.length < 4 || bytes.length % 4 !== 0) return null;
return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.length));
} catch {
return null;
}
}
export function encodeEmbedding(embedding: Float32Array): string {
return encodeBase64(new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength));
}
export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
if (a.length !== b.length || a.length === 0) return 0;
let dot = 0;
for (let i = 0; i < a.length; i++) dot += a[i]! * b[i]!;
// Embeddings are L2-normalized at the source, so the dot product is the
// cosine; renormalizing here would only amplify float noise.
return dot;
}
export class SpeakerRegistry {
private profiles: SpeakerProfile[] = [];
private loaded = false;
private readonly changeListeners = new Set<() => void>();
onChanged(listener: () => void): () => void {
this.changeListeners.add(listener);
return () => this.changeListeners.delete(listener);
}
private notifyChanged(): void {
this.changeListeners.forEach((listener) => listener());
}
reload(): void {
try {
const rows = JSON.parse(String(store().querySpeakers())) as Array<Record<string, unknown>>;
this.profiles = rows.map((row) => ({
id: Number(row.id),
name: String(row.name ?? ""),
color: String(row.color ?? SPEAKER_COLORS[0]),
isWearer: Boolean(row.isWearer),
embedding: decodeEmbedding(String(row.embeddingBase64 ?? "")),
embeddingCount: Number(row.embeddingCount ?? 0),
lastHeardAt: Number(row.lastHeardAt ?? 0),
tag: String(row.tag ?? ""),
segmentCount: Number(row.segmentCount ?? 0),
lastRecap: String(row.lastRecap ?? ""),
actionItems: parseStringArray(row.actionItemsJson),
facts: parseStringArray(row.factsJson),
insightsUpdatedAt: Number(row.insightsUpdatedAt ?? 0),
insightsSessionId: Number(row.insightsSessionId ?? 0),
}));
this.loaded = true;
} catch (error) {
// Keep the last good cache: emptying it would make the next utterance
// enroll an already-known voice as a duplicate "Speaker N".
console.error(`speaker reload failed: ${String(error)}`);
}
}
all(): SpeakerProfile[] {
if (!this.loaded) this.reload();
return this.profiles;
}
byId(id: number): SpeakerProfile | null {
return this.all().find((profile) => profile.id === id) ?? null;
}
wearer(): SpeakerProfile | null {
return this.all().find((profile) => profile.isWearer) ?? null;
}
/** Best cosine match against every enrolled profile, or null when none have prints. */
bestMatch(embedding: Float32Array): { profile: SpeakerProfile; similarity: number } | null {
let best: { profile: SpeakerProfile; similarity: number } | null = null;
for (const profile of this.all()) {
if (!profile.embedding || profile.embedding.length !== embedding.length) continue;
const similarity = cosineSimilarity(embedding, profile.embedding);
if (!best || similarity > best.similarity) {
best = { profile, similarity };
}
}
return best;
}
/**
* Match an utterance's voice-print to a profile, enrolling a new
* "Speaker N" when nothing crosses the uncertain threshold. Known and
* uncertain matches also fold the print into the centroid.
*/
assign(embedding: Float32Array): SpeakerMatch {
const best = this.bestMatch(embedding);
if (best && best.similarity >= SPEAKER_UNCERTAIN_THRESHOLD) {
const previousLastHeardAt = best.profile.lastHeardAt;
store().updateSpeakerEmbedding(best.profile.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
this.reload();
const profile = this.byId(best.profile.id) ?? best.profile;
return {
profile,
similarity: best.similarity,
classification: best.similarity >= SPEAKER_KNOWN_THRESHOLD ? "known" : "uncertain",
previousLastHeardAt,
};
}
const name = this.nextAutoName();
const color = SPEAKER_COLORS[this.all().length % SPEAKER_COLORS.length]!;
const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding)));
this.reload();
this.notifyChanged();
const profile = this.byId(id);
if (!profile) throw new Error(`speaker "${name}" was not created (id ${id})`);
return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 };
}
private nextAutoName(): string {
const taken = new Set(this.all().map((profile) => profile.name));
for (let index = 1; ; index++) {
const candidate = `Speaker ${index}`;
if (!taken.has(candidate)) return candidate;
}
}
/** Enroll (or refresh) the wearer's own voice from a training utterance. */
enrollWearer(embedding: Float32Array): SpeakerProfile {
const existing = this.wearer();
if (existing) {
store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
this.reload();
this.notifyChanged();
return this.byId(existing.id) ?? existing;
}
const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding)));
this.reload();
this.notifyChanged();
const profile = this.byId(id);
if (!profile) throw new Error(`wearer profile was not created (id ${id})`);
return profile;
}
/** Is this utterance the wearer's own voice? Used to gate command recognition. */
isWearerVoice(embedding: Float32Array): boolean {
const wearer = this.wearer();
if (!wearer?.embedding || wearer.embedding.length !== embedding.length) return false;
return cosineSimilarity(embedding, wearer.embedding) >= SPEAKER_KNOWN_THRESHOLD;
}
rename(id: number, name: string): void {
store().renameSpeaker(id, name);
this.reload();
this.notifyChanged();
}
setColor(id: number, color: string): void {
store().setSpeakerColor(id, color);
this.reload();
this.notifyChanged();
}
setTag(id: number, tag: string): void {
store().setSpeakerTag(id, tag);
this.reload();
this.notifyChanged();
}
/** Store LLM-derived insights: recap + action items of `sessionId`, merged fact list. */
setInsights(id: number, recap: string, actionItems: string[], facts: string[], sessionId: number): void {
store().setSpeakerInsights(id, recap, JSON.stringify(actionItems), JSON.stringify(facts), sessionId);
this.reload();
this.notifyChanged();
}
setWearer(id: number, isWearer: boolean): void {
store().setSpeakerWearer(id, isWearer);
this.reload();
this.notifyChanged();
}
merge(fromId: number, intoId: number): void {
store().mergeSpeakers(fromId, intoId);
this.reload();
this.notifyChanged();
}
/**
* Split support: give one session's segments of a speaker their own new
* profile (e.g. when two people were fused into one). Returns the new id.
*/
splitFromSession(sessionId: number, speakerId: number): number {
const source = this.byId(speakerId);
const name = this.nextAutoName();
const color = SPEAKER_COLORS[this.all().length % SPEAKER_COLORS.length]!;
const newId = Number(store().createSpeaker(name, color, false, ""));
store().reassignSessionSpeaker(sessionId, speakerId, newId);
if (source) {
this.reload();
}
this.notifyChanged();
return newId;
}
remove(id: number): void {
store().deleteSpeaker(id);
this.reload();
this.notifyChanged();
}
}
export const speakerRegistry = new SpeakerRegistry();
/**
* The speaker-verification config for "my voice only" command gating, or
* null when the setting is off, no wearer voice is enrolled, or the speaker
* model isn't downloaded. Consumed by the voice-capture path so assistant /
* dictation commands from other speakers are discarded.
*/
export function wearerVerificationOptions(): {
speakerModelPath: string;
wearerEmbedding: Float32Array;
threshold: number;
} | null {
if (!wearerCommandsOnlySetting.get()) return null;
const wearer = speakerRegistry.wearer();
if (!wearer?.embedding) return null;
const speakerModelPath = micModelPath("speaker-embedding");
if (!speakerModelPath) return null;
return { speakerModelPath, wearerEmbedding: wearer.embedding, threshold: SPEAKER_KNOWN_THRESHOLD };
}