forked from Blendman974/faceclaw
> Most serious fixes > - Recordings kept forever: a recording made with captions off, or with "Save captions" > off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so > the phone UI never showed it and the retention settings never deleted it. Recording > now only happens alongside a saved caption session, and turning captions off ends > that session's recording properly. > - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio > forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic > array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends > the config when it has been lost. The normal single-stream mode already recovered on > its own. > - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream > but never switched to the normal stream. Setup changes now take effect live, > including switching between the two modes. > - Setup changes were undone: after the assistant closed, the app re-sent the config it > loaded at launch to the glasses, reverting any changes you'd made since. > - Startup failures: if the caption engine failed to start, the session got stuck > half-running. If the glasses weren't connected, listeners piled up and an empty > conversation could be left open. Startup now brings up the microphone first and > cleans up properly on failure. > > Other fixes > - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms > frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test > that fails on the old code. > - Stale direction data: an arm that stopped sending kept steering the direction > estimate and could hold the caption feed on a silent arm. > - Compass: closing Microphones switched off the Compass app's compass, because both > used the same owner name. > - Speaker database failures: > - A failed speaker reload wiped the in-memory list, so the next voice became a > duplicate "Speaker N". > - A failed speaker insert crashed caption handling; in re-diarize it reassigned > transcript lines to a nonexistent speaker. > - Translation: > - A crash path inside the model-download callback is now caught. > - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of > old-style language codes. > - Ask questions: once the speaker list grew past the size limit (every unknown voice > adds a "Speaker N"), it pushed all transcript lines out of the context sent to the > model. When trimming is needed, the oldest lines now go first. > - Smaller fixes: > - The captions view stopped jumping back to live speech after 60 lines. > - "This is Dana calling…" was treated as introducing someone else. > - A model download that failed to start was stuck at "downloading" until restart. > - The developer mic monitor grew without limit while hidden.
141 lines
4.8 KiB
TypeScript
141 lines
4.8 KiB
TypeScript
import { GrayImage, type UiFont } from "../../graphics/image";
|
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
|
import { lineStep } from "../../ui/metrics";
|
|
import { GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, type InputEvent } from "../../ui/gestures";
|
|
import { Layer, type LayerContext } from "../../ui/layers";
|
|
import { micSession, type CaptionLine, type MicSessionState } from "./mic-session";
|
|
|
|
/**
|
|
* On-glasses captions: each line is prefixed with the speaker's name (their
|
|
* id until tagged); foreign-language lines show the translation directly
|
|
* beneath the recognized original. Scroll reviews recent lines; the view
|
|
* snaps back to live on new speech.
|
|
*/
|
|
export class CaptionsLayer implements Layer {
|
|
private state: MicSessionState = micSession.getState();
|
|
private unsubscribe: (() => void) | null = null;
|
|
private scrollback = 0;
|
|
private lastLine: CaptionLine | null = null;
|
|
|
|
start(requestRender: () => void): void {
|
|
this.unsubscribe = micSession.onState((state) => {
|
|
// Watch the newest line itself: once the session's line cap is reached
|
|
// the count stops changing even though new speech keeps arriving.
|
|
const last = state.captionLines[state.captionLines.length - 1] ?? null;
|
|
if (last !== this.lastLine) {
|
|
this.lastLine = last;
|
|
this.scrollback = 0;
|
|
}
|
|
this.state = state;
|
|
requestRender();
|
|
});
|
|
}
|
|
|
|
paint(ctx: LayerContext): GrayImage {
|
|
const font = getDefaultSmallFont();
|
|
const { width, height } = ctx.stack.getBaseSize();
|
|
const image = new GrayImage(width, height, 0);
|
|
const state = this.state;
|
|
|
|
image.drawText(font, 12, 4, "Captions", 220);
|
|
if (!state.captionsActive) {
|
|
image.drawText(font, 12, 26, "Captions are off (enable in the Microphones menu).", 130);
|
|
}
|
|
|
|
const rows = this.buildRows(font, width - 36, state.captionLines);
|
|
const step = lineStep(font) + 1;
|
|
const bodyTop = 24;
|
|
const footerY = height - font.lineHeight - 4;
|
|
const visibleRows = Math.max(1, Math.floor((footerY - bodyTop) / step));
|
|
const maxScrollback = Math.max(0, rows.length - visibleRows);
|
|
this.scrollback = Math.min(this.scrollback, maxScrollback);
|
|
const firstRow = Math.max(0, rows.length - visibleRows - this.scrollback);
|
|
for (let index = firstRow; index < Math.min(rows.length, firstRow + visibleRows); index++) {
|
|
const row = rows[index]!;
|
|
const y = bodyTop + (index - firstRow) * step;
|
|
if (row.prefix) {
|
|
image.drawText(font, 16, y, row.prefix, row.prefixValue);
|
|
image.drawText(font, 16 + font.measureText(row.prefix), y, row.text, row.value);
|
|
} else {
|
|
image.drawText(font, 16 + row.indent, y, row.text, row.value);
|
|
}
|
|
}
|
|
|
|
image.drawText(
|
|
font,
|
|
12,
|
|
footerY,
|
|
`${GESTURE_SCROLL} history ${GESTURE_DOUBLE_CLICK} back` +
|
|
(this.scrollback > 0 ? ` (${this.scrollback} back)` : ""),
|
|
110,
|
|
);
|
|
return image;
|
|
}
|
|
|
|
private buildRows(
|
|
font: UiFont,
|
|
maxWidth: number,
|
|
lines: CaptionLine[],
|
|
): Array<{ prefix?: string; prefixValue: number; text: string; value: number; indent: number }> {
|
|
const rows: Array<{ prefix?: string; prefixValue: number; text: string; value: number; indent: number }> = [];
|
|
for (const line of lines) {
|
|
const prefix = `${line.speakerName}: `;
|
|
const prefixWidth = font.measureText(prefix);
|
|
const wrapped = wrapWords(font, line.text, maxWidth - prefixWidth);
|
|
wrapped.forEach((text, index) => {
|
|
rows.push({
|
|
prefix: index === 0 ? prefix : undefined,
|
|
prefixValue: line.isWearer ? 255 : 170,
|
|
text,
|
|
value: 230,
|
|
indent: index === 0 ? 0 : prefixWidth,
|
|
});
|
|
});
|
|
if (line.translation) {
|
|
for (const text of wrapWords(font, `→ ${line.translation}`, maxWidth - prefixWidth)) {
|
|
rows.push({ prefixValue: 0, text, value: 140, indent: prefixWidth });
|
|
}
|
|
}
|
|
}
|
|
return rows;
|
|
}
|
|
|
|
handleInput(event: InputEvent, ctx: LayerContext): void {
|
|
switch (event.type) {
|
|
case "scroll-up":
|
|
this.scrollback += 1;
|
|
return;
|
|
case "scroll-down":
|
|
this.scrollback = Math.max(0, this.scrollback - 1);
|
|
return;
|
|
case "double-click":
|
|
ctx.stack.pop();
|
|
return;
|
|
default:
|
|
return;
|
|
}
|
|
}
|
|
|
|
onRemoved(): void {
|
|
this.unsubscribe?.();
|
|
this.unsubscribe = null;
|
|
}
|
|
}
|
|
|
|
function wrapWords(font: UiFont, text: string, maxWidth: number): string[] {
|
|
const words = text.split(/\s+/).filter(Boolean);
|
|
const lines: string[] = [];
|
|
let line = "";
|
|
for (const word of words) {
|
|
const candidate = line ? `${line} ${word}` : word;
|
|
if (line && font.measureText(candidate) > maxWidth) {
|
|
lines.push(line);
|
|
line = word;
|
|
} else {
|
|
line = candidate;
|
|
}
|
|
}
|
|
if (line) lines.push(line);
|
|
return lines.length ? lines : [""];
|
|
}
|