Files
faceclaw/app/apps/microphones/captions-layer.ts
T
James Babcock b50898ea02 Fix a bunch of microphone-related bugs
> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
2026-10-14 00:56:54 -07:00

141 lines
4.8 KiB
TypeScript

import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, type InputEvent } from "../../ui/gestures";
import { Layer, type LayerContext } from "../../ui/layers";
import { micSession, type CaptionLine, type MicSessionState } from "./mic-session";
/**
* On-glasses captions: each line is prefixed with the speaker's name (their
* id until tagged); foreign-language lines show the translation directly
* beneath the recognized original. Scroll reviews recent lines; the view
* snaps back to live on new speech.
*/
export class CaptionsLayer implements Layer {
private state: MicSessionState = micSession.getState();
private unsubscribe: (() => void) | null = null;
private scrollback = 0;
private lastLine: CaptionLine | null = null;
start(requestRender: () => void): void {
this.unsubscribe = micSession.onState((state) => {
// Watch the newest line itself: once the session's line cap is reached
// the count stops changing even though new speech keeps arriving.
const last = state.captionLines[state.captionLines.length - 1] ?? null;
if (last !== this.lastLine) {
this.lastLine = last;
this.scrollback = 0;
}
this.state = state;
requestRender();
});
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const state = this.state;
image.drawText(font, 12, 4, "Captions", 220);
if (!state.captionsActive) {
image.drawText(font, 12, 26, "Captions are off (enable in the Microphones menu).", 130);
}
const rows = this.buildRows(font, width - 36, state.captionLines);
const step = lineStep(font) + 1;
const bodyTop = 24;
const footerY = height - font.lineHeight - 4;
const visibleRows = Math.max(1, Math.floor((footerY - bodyTop) / step));
const maxScrollback = Math.max(0, rows.length - visibleRows);
this.scrollback = Math.min(this.scrollback, maxScrollback);
const firstRow = Math.max(0, rows.length - visibleRows - this.scrollback);
for (let index = firstRow; index < Math.min(rows.length, firstRow + visibleRows); index++) {
const row = rows[index]!;
const y = bodyTop + (index - firstRow) * step;
if (row.prefix) {
image.drawText(font, 16, y, row.prefix, row.prefixValue);
image.drawText(font, 16 + font.measureText(row.prefix), y, row.text, row.value);
} else {
image.drawText(font, 16 + row.indent, y, row.text, row.value);
}
}
image.drawText(
font,
12,
footerY,
`${GESTURE_SCROLL} history ${GESTURE_DOUBLE_CLICK} back` +
(this.scrollback > 0 ? ` (${this.scrollback} back)` : ""),
110,
);
return image;
}
private buildRows(
font: UiFont,
maxWidth: number,
lines: CaptionLine[],
): Array<{ prefix?: string; prefixValue: number; text: string; value: number; indent: number }> {
const rows: Array<{ prefix?: string; prefixValue: number; text: string; value: number; indent: number }> = [];
for (const line of lines) {
const prefix = `${line.speakerName}: `;
const prefixWidth = font.measureText(prefix);
const wrapped = wrapWords(font, line.text, maxWidth - prefixWidth);
wrapped.forEach((text, index) => {
rows.push({
prefix: index === 0 ? prefix : undefined,
prefixValue: line.isWearer ? 255 : 170,
text,
value: 230,
indent: index === 0 ? 0 : prefixWidth,
});
});
if (line.translation) {
for (const text of wrapWords(font, `→ ${line.translation}`, maxWidth - prefixWidth)) {
rows.push({ prefixValue: 0, text, value: 140, indent: prefixWidth });
}
}
}
return rows;
}
handleInput(event: InputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
this.scrollback += 1;
return;
case "scroll-down":
this.scrollback = Math.max(0, this.scrollback - 1);
return;
case "double-click":
ctx.stack.pop();
return;
default:
return;
}
}
onRemoved(): void {
this.unsubscribe?.();
this.unsubscribe = null;
}
}
function wrapWords(font: UiFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) lines.push(line);
return lines.length ? lines : [""];
}