Files
faceclaw/app/apps/microphones/microphones-app.ts
T
James Babcock b50898ea02 Fix a bunch of microphone-related bugs
> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
2026-10-14 00:56:54 -07:00

498 lines
17 KiB
TypeScript

import { imageFromAsciiArt } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { appViewportSize } from "../../ui/shell/geometry";
import {
MenuLayer,
TextPageLayer,
drawRightValueMenuItem,
drawToggleMenuItem,
openModalMenu,
submenuItem,
type MenuItem,
} from "../../ui/menu";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import {
createInProcessWindow,
YieldAtRootLayer,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { shell } from "../../ui/shell/shell";
import { isAsrModelReady, startAsrModelDownload, asrModelState } from "../../native/asr-model";
import { CaptionsLayer } from "./captions-layer";
import { LevelsLayer } from "./levels-layer";
import { RadarLayer } from "./radar-layer";
import { loadMicConfig, micArrayController, micControlSupported } from "./mic-control";
import { isMicModelReady, micModelState, startMicModelDownload } from "./mic-models";
import { micChannelLabel, type MicChannelKey, type MicConfig } from "./mic-protocol";
import { micSession } from "./mic-session";
import { speakerRegistry, type SpeakerProfile } from "./speakers";
import { formatRelativeTime } from "../../util/date-util";
import {
ancEnabledSetting,
beamFilterSetting,
captionsEnabledSetting,
captionsRetentionSetting,
recordingsRetentionSetting,
saveCaptionsSetting,
saveRecordingsSetting,
translateEnabledSetting,
wearerCommandsOnlySetting,
} from "./mic-settings";
export const MICROPHONES_WINDOW_ID = "microphones";
export const MICROPHONES_SURFACE_ID = "window:microphones";
const TRAY_ICON_ID = "microphones";
// 14x15 dual-mic glyph for the top-bar tray.
const TRAY_ICON = imageFromAsciiArt(
[
" ## ## ",
" # # # # ",
" # # # # ",
" # # # # ",
" # # # # ",
"# ## # # ## #",
" # # # # ",
" ## ## ",
" # # ",
" ### ### ",
],
{ "#": 220 },
);
const MENU_LAYOUT = {
x: 8,
y: 8,
width: 292,
showBorder: false,
minHeight: 0,
maxHeight: appViewportSize("min").height - 16,
opaque: true,
};
/** Persist a config change; the running session applies it live. */
function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void {
micSession.updateConfig(mutate);
}
function micChannelToggleItem(
key: MicChannelKey,
description: string,
disabled: () => boolean,
): MenuItem {
return configToggleItem(
`${micChannelLabel(key)} mic`,
() => loadMicConfig().hostMics[key],
() => micSession.setMicChannelEnabled(key, !loadMicConfig().hostMics[key]),
disabled,
description,
);
}
function configToggleItem(
label: string,
isOn: () => boolean,
toggle: () => void,
disabled?: () => boolean,
description?: string,
): MenuItem {
return {
label,
description,
disabled,
onSelect: () => toggle(),
render: ({ image, x, y, width, selected }) => {
drawToggleMenuItem(image, getDefaultSmallFont(), x, y, width, label, isOn(), selected);
},
};
}
function configPickerItem<T extends string | number>(
label: string,
values: readonly T[],
format: (value: T) => string,
current: () => T,
apply: (value: T) => void,
disabled?: () => boolean,
description?: string,
): MenuItem {
return {
label,
description,
disabled,
onSelect: (ctx) => {
const items = values.map(
(value): MenuItem => ({
label: format(value) + (current() === value ? " *" : ""),
onSelect: (innerCtx) => {
apply(value);
innerCtx.stack.pop();
},
}),
);
openModalMenu(ctx, label, items, Math.max(0, values.indexOf(current())));
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, format(current()));
},
};
}
function micSetupMenu(): MenuLayer {
const unsupported = () => !micControlSupported();
return new MenuLayer(
"Microphone setup",
[
{
label: "Array status",
description:
"Per-temple mic-control status. Requires the Use microphone control developer setting.",
onSelect: (ctx) => {
micArrayController.query();
const { left, right } = micArrayController.getStatuses();
const describe = (side: string, status: typeof left) =>
status
? `${side}: ${status.active ? "active" : "idle"}, mics ${status.channelMask & 1 ? "front" : ""}${
status.channelMask & 2 ? "+rear" : ""
}, ${status.effectiveRateHz / 1000} kHz, ${status.hardwareArmed ? "armed" : "not armed"}, ${
status.framesEmitted
} frames`
: `${side}: no status yet`;
const consistency = micArrayController.arrayConsistent()
? "Array consistent (L/R matched)."
: "Array not yet consistent.";
ctx.stack.push(
new TextPageLayer(
"Array status",
micControlSupported()
? `${describe("Left", left)}\n${describe("Right", right)}\n${consistency}`
: "Mic-control is disabled. Turn on Settings > Developer > Use microphone control to control the four mics individually; while it is off the glasses stream one mixed channel with a firmware-computed direction of arrival.",
),
);
},
},
// Each temple streams its full front+rear pair; these four toggles are
// host-side selection over the split channels, so any combination of
// the four microphones can be used independently.
micChannelToggleItem("leftFront", "Enable the left temple's front microphone (near the hinge).", unsupported),
micChannelToggleItem("leftRear", "Enable the left temple's rear microphone (by the touchpad).", unsupported),
micChannelToggleItem("rightFront", "Enable the right temple's front microphone (near the hinge).", unsupported),
micChannelToggleItem("rightRear", "Enable the right temple's rear microphone (by the touchpad).", unsupported),
configPickerItem(
"Source",
["pdm", "codec"] as const,
(value) => (value === "pdm" ? "PDM mics" : "Codec DMIC"),
() => loadMicConfig().source,
(value) => updateMicConfig((config) => ({ ...config, source: value })),
unsupported,
"PDM streams one channel; the codec front end streams both mics of a temple.",
),
configPickerItem(
"Codec",
["raw", "lc3"] as const,
(value) => (value === "raw" ? "Raw PCM" : "LC3"),
() => loadMicConfig().codec,
(value) => updateMicConfig((config) => ({ ...config, codec: value })),
unsupported,
"Raw PCM preserves inter-mic phase for beamforming. (Current firmware always streams raw.)",
),
configPickerItem(
"Sample rate",
[8000, 16000, 24000, 32000, 48000] as const,
(value) => `${value / 1000} kHz`,
() => loadMicConfig().sampleRateHz,
(value) => updateMicConfig((config) => ({ ...config, sampleRateHz: value })),
unsupported,
"Requested capture rate per mic pair. Current firmware pins the effective rate at 16 kHz and echoes the request in the status.",
),
configPickerItem(
"LC3 bitrate",
[0, 16000, 32000, 48000, 64000] as const,
(value) => (value === 0 ? "Default" : `${value / 1000} kbps`),
() => loadMicConfig().lc3BitrateBps,
(value) => updateMicConfig((config) => ({ ...config, lc3BitrateBps: value })),
unsupported,
"Requested LC3 bitrate per mic pair (takes effect once the firmware's LC3 path lands).",
),
configToggleItem(
"Direction detection",
() => loadMicConfig().beamform,
() => updateMicConfig((config) => ({ ...config, beamform: !config.beamform })),
unsupported,
"Have each temple compute and stream its own sound direction (TDOA angle + signal strength).",
),
{
label: "4-mic capture (advanced)",
description:
"Arms the microphone hardware and multi-channel streaming on the glasses. Leave off until the firmware validation checklist has passed on this hardware.",
disabled: unsupported,
onSelect: (ctx) => {
const config = loadMicConfig();
if (config.armHardware) {
updateMicConfig((current) => ({ ...current, armHardware: false }));
micArrayController.stopStreaming();
return;
}
openModalMenu(ctx, "Arm mic hardware?", [
{
label: "Cancel",
onSelect: (innerCtx) => innerCtx.stack.pop(),
},
{
label: "Enable (validated hardware only)",
onSelect: (innerCtx) => {
updateMicConfig((current) => ({ ...current, armHardware: true }));
innerCtx.stack.pop();
},
},
]);
},
render: ({ image, x, y, width, selected }) => {
drawToggleMenuItem(
image,
getDefaultSmallFont(),
x,
y,
width,
"4-mic capture (advanced)",
loadMicConfig().armHardware,
selected,
);
},
},
],
MENU_LAYOUT,
);
}
function modelItem(
label: string,
ready: () => boolean,
stateText: () => string,
download: () => void,
): MenuItem {
return {
label,
onSelect: () => {
if (!ready()) download();
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, stateText());
},
};
}
function voiceSpeakersMenu(): MenuLayer {
return new MenuLayer(
"Voice & speakers",
[
{
label: "Enroll my voice",
description:
"Speak a sentence after selecting; the next utterance becomes (or refreshes) your voice-print, used to label you in captions and to restrict commands to your voice.",
disabled: () => !micSession.getState().captionsActive,
onSelect: () => micSession.armWearerEnrollment(),
},
toggleSettingMenuItem(wearerCommandsOnlySetting),
modelItem(
"Speaker voice model",
() => isMicModelReady("speaker-embedding"),
() => {
const state = micModelState("speaker-embedding");
if (state.status === "ready") return "ready";
if (state.status === "downloading") {
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
}
return "download (28 MB)";
},
() => startMicModelDownload("speaker-embedding"),
),
modelItem(
// Live captions always use Moonshine (see the matching note in
// mic-session.ts); the second on-device model (Whisper) is a
// push-to-talk dictation option only, managed in Settings > Voice.
"Caption ASR model",
() => isAsrModelReady("moonshine"),
() => {
const state = asrModelState("moonshine");
if (state.status === "ready") return "ready";
if (state.status === "downloading") {
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
}
return "download (141 MB)";
},
() => startAsrModelDownload("moonshine"),
),
modelItem(
"Re-diarization model",
() => isMicModelReady("diarization-segmentation"),
() => {
const state = micModelState("diarization-segmentation");
if (state.status === "ready") return "ready";
if (state.status === "downloading") {
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
}
return "download (6 MB)";
},
() => startMicModelDownload("diarization-segmentation"),
),
],
MENU_LAYOUT,
);
}
/** The full contact card behind a People row (and behind the encounter popup). */
function speakerDetailBody(profile: SpeakerProfile): string {
const parts: string[] = [
`Last heard ${profile.lastHeardAt ? `${formatRelativeTime(profile.lastHeardAt)} ago` : "never"}.`,
];
if (profile.tag) parts.push(`Note: ${profile.tag}`);
if (profile.lastRecap) parts.push(`Last conversation: ${profile.lastRecap}`);
if (profile.actionItems.length) {
parts.push(`To do:\n${profile.actionItems.map((item) => `- ${item}`).join("\n")}`);
}
if (profile.facts.length) {
parts.push(`Facts:\n${profile.facts.map((fact) => `- ${fact}`).join("\n")}`);
}
if (parts.length === 1) {
parts.push(
"No conversation insights yet. A recap, action items, and remembered facts are " +
"generated when a captioned conversation with this person ends (requires the " +
"on-phone assistant model).",
);
}
return parts.join("\n\n");
}
/**
* People: the speaker-recognition contacts, most recently heard first — the
* review surface behind the encounter popup. Selecting a person shows when
* they were last heard, their note, the recap and action items of the last
* conversation, and the facts remembered about them.
*/
function peopleMenu(): MenuLayer {
const profiles = speakerRegistry.all().filter((profile) => !profile.isWearer);
const items: MenuItem[] = profiles.length
? profiles.map((profile) => ({
label: profile.name,
onSelect: (ctx) => {
ctx.stack.push(new TextPageLayer(profile.name, speakerDetailBody(profile)));
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(
image,
getDefaultSmallFont(),
x,
y,
width,
profile.name,
profile.lastHeardAt ? `${formatRelativeTime(profile.lastHeardAt)} ago` : "",
);
},
}))
: [
{
label: "No people heard yet",
disabled: true,
onSelect: () => {},
},
];
return new MenuLayer("People", items, MENU_LAYOUT);
}
function storageMenu(): MenuLayer {
return new MenuLayer(
"Storage",
[
toggleSettingMenuItem(saveCaptionsSetting),
toggleSettingMenuItem(saveRecordingsSetting),
enumSettingMenuItem(captionsRetentionSetting),
enumSettingMenuItem(recordingsRetentionSetting),
{
label: "Review on phone",
description:
"Saved conversations, transcripts, speakers, and recordings are reviewed in the Faceclaw phone app: Conversations in the top-right menu.",
onSelect: (ctx) => {
ctx.stack.push(
new TextPageLayer(
"Review on phone",
"Open the Faceclaw app on your phone and choose Conversations from the top-right menu to search captions, filter by speaker, play linked recordings, rename/merge speakers, re-diarize sessions, and see the tone/sentiment graph of each conversation.",
),
);
},
},
],
MENU_LAYOUT,
);
}
export function createMicrophonesAppWindow(options: InProcessAppOptions): InProcessWindow {
const menu = new MenuLayer(
"Microphones",
[
submenuItem("Sonic Radar", (ctx) => {
const layer = new RadarLayer();
ctx.stack.push(layer);
layer.start(ctx.actions.requestRender);
}),
submenuItem("Levels", (ctx) => {
const layer = new LevelsLayer();
ctx.stack.push(layer);
layer.start(ctx.actions.requestRender);
}),
submenuItem("Captions view", (ctx) => {
const layer = new CaptionsLayer();
ctx.stack.push(layer);
layer.start(ctx.actions.requestRender);
}),
submenuItem("People", (ctx) => {
speakerRegistry.reload();
ctx.stack.push(peopleMenu());
}),
configToggleItem(
"Captions",
() => captionsEnabledSetting.get(),
() => micSession.setCaptionsEnabled(!captionsEnabledSetting.get()),
undefined,
captionsEnabledSetting.description,
),
toggleSettingMenuItem(translateEnabledSetting),
toggleSettingMenuItem(beamFilterSetting),
toggleSettingMenuItem(ancEnabledSetting),
submenuItem("Microphone setup", (ctx) => {
ctx.stack.push(micSetupMenu());
}),
submenuItem("Voice & speakers", (ctx) => {
ctx.stack.push(voiceSpeakersMenu());
}),
submenuItem("Storage", (ctx) => {
ctx.stack.push(storageMenu());
}),
],
MENU_LAYOUT,
);
const app = createInProcessWindow({
appId: "microphones",
windowId: MICROPHONES_WINDOW_ID,
title: "Microphones",
iconLetter: "Mi",
icon: "mic",
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(menu),
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
micSession.stop();
shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed();
},
});
micSession.start();
shell.setTrayIcon(TRAY_ICON_ID, TRAY_ICON);
return app;
}