forked from Blendman974/faceclaw
> Most serious fixes > - Recordings kept forever: a recording made with captions off, or with "Save captions" > off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so > the phone UI never showed it and the retention settings never deleted it. Recording > now only happens alongside a saved caption session, and turning captions off ends > that session's recording properly. > - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio > forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic > array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends > the config when it has been lost. The normal single-stream mode already recovered on > its own. > - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream > but never switched to the normal stream. Setup changes now take effect live, > including switching between the two modes. > - Setup changes were undone: after the assistant closed, the app re-sent the config it > loaded at launch to the glasses, reverting any changes you'd made since. > - Startup failures: if the caption engine failed to start, the session got stuck > half-running. If the glasses weren't connected, listeners piled up and an empty > conversation could be left open. Startup now brings up the microphone first and > cleans up properly on failure. > > Other fixes > - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms > frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test > that fails on the old code. > - Stale direction data: an arm that stopped sending kept steering the direction > estimate and could hold the caption feed on a silent arm. > - Compass: closing Microphones switched off the Compass app's compass, because both > used the same owner name. > - Speaker database failures: > - A failed speaker reload wiped the in-memory list, so the next voice became a > duplicate "Speaker N". > - A failed speaker insert crashed caption handling; in re-diarize it reassigned > transcript lines to a nonexistent speaker. > - Translation: > - A crash path inside the model-download callback is now caught. > - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of > old-style language codes. > - Ask questions: once the speaker list grew past the size limit (every unknown voice > adds a "Speaker N"), it pushed all transcript lines out of the context sent to the > model. When trimming is needed, the oldest lines now go first. > - Smaller fixes: > - The captions view stopped jumping back to live speech after 60 lines. > - "This is Dana calling…" was treated as introducing someone else. > - A model download that failed to start was stuck at "downloading" until restart. > - The developer mic monitor grew without limit while hidden.
316 lines
11 KiB
TypeScript
316 lines
11 KiB
TypeScript
import { Utils } from "@nativescript/core";
|
|
|
|
import { encodeBase64 } from "../../native/cloud-stt";
|
|
import { toUint8Array } from "../../util/array-util";
|
|
import { micModelPath } from "./mic-models";
|
|
import { wearerCommandsOnlySetting } from "./mic-settings";
|
|
|
|
declare const com: any;
|
|
declare const android: any;
|
|
|
|
/**
|
|
* Speaker voice-print registry over FaceclawConversationStore. Matching is
|
|
* cosine similarity on L2-normalized embeddings: >= 0.80 auto-assign ("known"), 0.50-0.80 "uncertain" (assigned
|
|
* but flagged for review), below that a new speaker is enrolled. Centroids
|
|
* update as a capped running mean so one long session cannot drown an
|
|
* enrolled voice.
|
|
*/
|
|
|
|
export const SPEAKER_KNOWN_THRESHOLD = 0.8;
|
|
export const SPEAKER_UNCERTAIN_THRESHOLD = 0.5;
|
|
/** Cap on the running-mean weight of an existing centroid. */
|
|
export const CENTROID_UPDATE_CAP = 24;
|
|
|
|
const SPEAKER_COLORS = [
|
|
"#4FC3F7",
|
|
"#FFB74D",
|
|
"#81C784",
|
|
"#E57373",
|
|
"#BA68C8",
|
|
"#FFD54F",
|
|
"#4DB6AC",
|
|
"#F06292",
|
|
"#A1887F",
|
|
"#90A4AE",
|
|
];
|
|
|
|
export type SpeakerProfile = {
|
|
id: number;
|
|
name: string;
|
|
color: string;
|
|
isWearer: boolean;
|
|
embedding: Float32Array | null;
|
|
embeddingCount: number;
|
|
lastHeardAt: number;
|
|
tag: string;
|
|
segmentCount: number;
|
|
/** LLM recap of this speaker's most recent conversation ("" until one is generated). */
|
|
lastRecap: string;
|
|
/** Open action items from that conversation. */
|
|
actionItems: string[];
|
|
/** Accumulated inferred facts about this person. */
|
|
facts: string[];
|
|
insightsUpdatedAt: number;
|
|
insightsSessionId: number;
|
|
};
|
|
|
|
export type SpeakerMatch = {
|
|
profile: SpeakerProfile;
|
|
similarity: number;
|
|
classification: "known" | "uncertain" | "new";
|
|
/**
|
|
* The profile's last_heard_at BEFORE this utterance updated it — i.e. when
|
|
* this person's voice was previously heard. 0 for a newly enrolled speaker.
|
|
*/
|
|
previousLastHeardAt: number;
|
|
};
|
|
|
|
function store(): any {
|
|
const context = Utils.android.getApplicationContext();
|
|
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
|
|
}
|
|
|
|
/** Parse a stored JSON string array leniently; anything malformed reads as empty. */
|
|
function parseStringArray(json: unknown): string[] {
|
|
try {
|
|
const parsed = JSON.parse(String(json || "[]"));
|
|
return Array.isArray(parsed) ? parsed.map((item) => String(item)).filter(Boolean) : [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function decodeEmbedding(base64: string): Float32Array | null {
|
|
if (!base64) return null;
|
|
try {
|
|
const bytes = toUint8Array(android.util.Base64.decode(base64, android.util.Base64.NO_WRAP));
|
|
if (bytes.length < 4 || bytes.length % 4 !== 0) return null;
|
|
return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.length));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function encodeEmbedding(embedding: Float32Array): string {
|
|
return encodeBase64(new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength));
|
|
}
|
|
|
|
export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
|
|
if (a.length !== b.length || a.length === 0) return 0;
|
|
let dot = 0;
|
|
for (let i = 0; i < a.length; i++) dot += a[i]! * b[i]!;
|
|
// Embeddings are L2-normalized at the source, so the dot product is the
|
|
// cosine; renormalizing here would only amplify float noise.
|
|
return dot;
|
|
}
|
|
|
|
export class SpeakerRegistry {
|
|
private profiles: SpeakerProfile[] = [];
|
|
private loaded = false;
|
|
private readonly changeListeners = new Set<() => void>();
|
|
|
|
onChanged(listener: () => void): () => void {
|
|
this.changeListeners.add(listener);
|
|
return () => this.changeListeners.delete(listener);
|
|
}
|
|
|
|
private notifyChanged(): void {
|
|
this.changeListeners.forEach((listener) => listener());
|
|
}
|
|
|
|
reload(): void {
|
|
try {
|
|
const rows = JSON.parse(String(store().querySpeakers())) as Array<Record<string, unknown>>;
|
|
this.profiles = rows.map((row) => ({
|
|
id: Number(row.id),
|
|
name: String(row.name ?? ""),
|
|
color: String(row.color ?? SPEAKER_COLORS[0]),
|
|
isWearer: Boolean(row.isWearer),
|
|
embedding: decodeEmbedding(String(row.embeddingBase64 ?? "")),
|
|
embeddingCount: Number(row.embeddingCount ?? 0),
|
|
lastHeardAt: Number(row.lastHeardAt ?? 0),
|
|
tag: String(row.tag ?? ""),
|
|
segmentCount: Number(row.segmentCount ?? 0),
|
|
lastRecap: String(row.lastRecap ?? ""),
|
|
actionItems: parseStringArray(row.actionItemsJson),
|
|
facts: parseStringArray(row.factsJson),
|
|
insightsUpdatedAt: Number(row.insightsUpdatedAt ?? 0),
|
|
insightsSessionId: Number(row.insightsSessionId ?? 0),
|
|
}));
|
|
this.loaded = true;
|
|
} catch (error) {
|
|
// Keep the last good cache: emptying it would make the next utterance
|
|
// enroll an already-known voice as a duplicate "Speaker N".
|
|
console.error(`speaker reload failed: ${String(error)}`);
|
|
}
|
|
}
|
|
|
|
all(): SpeakerProfile[] {
|
|
if (!this.loaded) this.reload();
|
|
return this.profiles;
|
|
}
|
|
|
|
byId(id: number): SpeakerProfile | null {
|
|
return this.all().find((profile) => profile.id === id) ?? null;
|
|
}
|
|
|
|
wearer(): SpeakerProfile | null {
|
|
return this.all().find((profile) => profile.isWearer) ?? null;
|
|
}
|
|
|
|
/** Best cosine match against every enrolled profile, or null when none have prints. */
|
|
bestMatch(embedding: Float32Array): { profile: SpeakerProfile; similarity: number } | null {
|
|
let best: { profile: SpeakerProfile; similarity: number } | null = null;
|
|
for (const profile of this.all()) {
|
|
if (!profile.embedding || profile.embedding.length !== embedding.length) continue;
|
|
const similarity = cosineSimilarity(embedding, profile.embedding);
|
|
if (!best || similarity > best.similarity) {
|
|
best = { profile, similarity };
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
/**
|
|
* Match an utterance's voice-print to a profile, enrolling a new
|
|
* "Speaker N" when nothing crosses the uncertain threshold. Known and
|
|
* uncertain matches also fold the print into the centroid.
|
|
*/
|
|
assign(embedding: Float32Array): SpeakerMatch {
|
|
const best = this.bestMatch(embedding);
|
|
if (best && best.similarity >= SPEAKER_UNCERTAIN_THRESHOLD) {
|
|
const previousLastHeardAt = best.profile.lastHeardAt;
|
|
store().updateSpeakerEmbedding(best.profile.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
|
|
this.reload();
|
|
const profile = this.byId(best.profile.id) ?? best.profile;
|
|
return {
|
|
profile,
|
|
similarity: best.similarity,
|
|
classification: best.similarity >= SPEAKER_KNOWN_THRESHOLD ? "known" : "uncertain",
|
|
previousLastHeardAt,
|
|
};
|
|
}
|
|
const name = this.nextAutoName();
|
|
const color = SPEAKER_COLORS[this.all().length % SPEAKER_COLORS.length]!;
|
|
const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding)));
|
|
this.reload();
|
|
this.notifyChanged();
|
|
const profile = this.byId(id);
|
|
if (!profile) throw new Error(`speaker "${name}" was not created (id ${id})`);
|
|
return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 };
|
|
}
|
|
|
|
private nextAutoName(): string {
|
|
const taken = new Set(this.all().map((profile) => profile.name));
|
|
for (let index = 1; ; index++) {
|
|
const candidate = `Speaker ${index}`;
|
|
if (!taken.has(candidate)) return candidate;
|
|
}
|
|
}
|
|
|
|
/** Enroll (or refresh) the wearer's own voice from a training utterance. */
|
|
enrollWearer(embedding: Float32Array): SpeakerProfile {
|
|
const existing = this.wearer();
|
|
if (existing) {
|
|
store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
return this.byId(existing.id) ?? existing;
|
|
}
|
|
const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding)));
|
|
this.reload();
|
|
this.notifyChanged();
|
|
const profile = this.byId(id);
|
|
if (!profile) throw new Error(`wearer profile was not created (id ${id})`);
|
|
return profile;
|
|
}
|
|
|
|
/** Is this utterance the wearer's own voice? Used to gate command recognition. */
|
|
isWearerVoice(embedding: Float32Array): boolean {
|
|
const wearer = this.wearer();
|
|
if (!wearer?.embedding || wearer.embedding.length !== embedding.length) return false;
|
|
return cosineSimilarity(embedding, wearer.embedding) >= SPEAKER_KNOWN_THRESHOLD;
|
|
}
|
|
|
|
rename(id: number, name: string): void {
|
|
store().renameSpeaker(id, name);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
}
|
|
|
|
setColor(id: number, color: string): void {
|
|
store().setSpeakerColor(id, color);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
}
|
|
|
|
setTag(id: number, tag: string): void {
|
|
store().setSpeakerTag(id, tag);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
}
|
|
|
|
/** Store LLM-derived insights: recap + action items of `sessionId`, merged fact list. */
|
|
setInsights(id: number, recap: string, actionItems: string[], facts: string[], sessionId: number): void {
|
|
store().setSpeakerInsights(id, recap, JSON.stringify(actionItems), JSON.stringify(facts), sessionId);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
}
|
|
|
|
setWearer(id: number, isWearer: boolean): void {
|
|
store().setSpeakerWearer(id, isWearer);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
}
|
|
|
|
merge(fromId: number, intoId: number): void {
|
|
store().mergeSpeakers(fromId, intoId);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
}
|
|
|
|
/**
|
|
* Split support: give one session's segments of a speaker their own new
|
|
* profile (e.g. when two people were fused into one). Returns the new id.
|
|
*/
|
|
splitFromSession(sessionId: number, speakerId: number): number {
|
|
const source = this.byId(speakerId);
|
|
const name = this.nextAutoName();
|
|
const color = SPEAKER_COLORS[this.all().length % SPEAKER_COLORS.length]!;
|
|
const newId = Number(store().createSpeaker(name, color, false, ""));
|
|
store().reassignSessionSpeaker(sessionId, speakerId, newId);
|
|
if (source) {
|
|
this.reload();
|
|
}
|
|
this.notifyChanged();
|
|
return newId;
|
|
}
|
|
|
|
remove(id: number): void {
|
|
store().deleteSpeaker(id);
|
|
this.reload();
|
|
this.notifyChanged();
|
|
}
|
|
}
|
|
|
|
export const speakerRegistry = new SpeakerRegistry();
|
|
|
|
/**
|
|
* The speaker-verification config for "my voice only" command gating, or
|
|
* null when the setting is off, no wearer voice is enrolled, or the speaker
|
|
* model isn't downloaded. Consumed by the voice-capture path so assistant /
|
|
* dictation commands from other speakers are discarded.
|
|
*/
|
|
export function wearerVerificationOptions(): {
|
|
speakerModelPath: string;
|
|
wearerEmbedding: Float32Array;
|
|
threshold: number;
|
|
} | null {
|
|
if (!wearerCommandsOnlySetting.get()) return null;
|
|
const wearer = speakerRegistry.wearer();
|
|
if (!wearer?.embedding) return null;
|
|
const speakerModelPath = micModelPath("speaker-embedding");
|
|
if (!speakerModelPath) return null;
|
|
return { speakerModelPath, wearerEmbedding: wearer.embedding, threshold: SPEAKER_KNOWN_THRESHOLD };
|
|
}
|