forked from Blendman974/faceclaw
> Most serious fixes > - Recordings kept forever: a recording made with captions off, or with "Save captions" > off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so > the phone UI never showed it and the retention settings never deleted it. Recording > now only happens alongside a saved caption session, and turning captions off ends > that session's recording properly. > - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio > forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic > array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends > the config when it has been lost. The normal single-stream mode already recovered on > its own. > - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream > but never switched to the normal stream. Setup changes now take effect live, > including switching between the two modes. > - Setup changes were undone: after the assistant closed, the app re-sent the config it > loaded at launch to the glasses, reverting any changes you'd made since. > - Startup failures: if the caption engine failed to start, the session got stuck > half-running. If the glasses weren't connected, listeners piled up and an empty > conversation could be left open. Startup now brings up the microphone first and > cleans up properly on failure. > > Other fixes > - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms > frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test > that fails on the old code. > - Stale direction data: an arm that stopped sending kept steering the direction > estimate and could hold the caption feed on a silent arm. > - Compass: closing Microphones switched off the Compass app's compass, because both > used the same owner name. > - Speaker database failures: > - A failed speaker reload wiped the in-memory list, so the next voice became a > duplicate "Speaker N". > - A failed speaker insert crashed caption handling; in re-diarize it reassigned > transcript lines to a nonexistent speaker. > - Translation: > - A crash path inside the model-download callback is now caught. > - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of > old-style language codes. > - Ask questions: once the speaker list grew past the size limit (every unknown voice > adds a "Speaker N"), it pushed all transcript lines out of the context sent to the > model. When trimming is needed, the oldest lines now go first. > - Smaller fixes: > - The captions view stopped jumping back to live speech after 60 lines. > - "This is Dana calling…" was treated as introducing someone else. > - A model download that failed to start was stuck at "downloading" until restart. > - The developer mic monitor grew without limit while hidden.
196 lines
6.2 KiB
TypeScript
196 lines
6.2 KiB
TypeScript
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
|
import { useMicControlSetting } from "../../ui/dashboard-settings";
|
|
import { toJavaBytes } from "../../native/cloud-stt";
|
|
import { toUint8Array } from "../../util/array-util";
|
|
import {
|
|
MIC_LEASE_MS,
|
|
decodeMicStatus,
|
|
defaultMicConfig,
|
|
encodeMicControl,
|
|
templeActive,
|
|
type MicConfig,
|
|
type MicOp,
|
|
type MicSide,
|
|
type MicStatus,
|
|
} from "./mic-protocol";
|
|
|
|
declare const com: any;
|
|
|
|
/**
|
|
* Host-side controller for the CFW mic_control array: sends the same config
|
|
* to both temples (each temple is its own endpoint and its own 2-mic array),
|
|
* tracks per-temple field-104 status, and keeps the 90 s streaming lease
|
|
* renewed while capture is armed. The feature is part of the required
|
|
* firmware revision, so the only gate is the developer setting:
|
|
* micControlSupported() is false and apply() refuses to arm when it is off.
|
|
*/
|
|
|
|
const CONFIG_SETTING_KEY = "microphones.array-config";
|
|
const LEASE_RENEW_MS = Math.floor(MIC_LEASE_MS / 3);
|
|
|
|
export type MicArrayStatuses = { left: MicStatus | null; right: MicStatus | null };
|
|
|
|
function activeCommunicator(): any {
|
|
if (!global.isAndroid) return null;
|
|
try {
|
|
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* True when the "Use microphone control" developer setting is on. With the
|
|
* setting off, Faceclaw behaves exactly as it would on firmware without the
|
|
* mic-control channel (one mixed stream).
|
|
*/
|
|
export function micControlSupported(): boolean {
|
|
return useMicControlSetting.get();
|
|
}
|
|
|
|
export function loadMicConfig(): MicConfig {
|
|
const raw = getStringSetting(CONFIG_SETTING_KEY, "");
|
|
if (!raw) return defaultMicConfig();
|
|
try {
|
|
const parsed = JSON.parse(raw) as Partial<MicConfig>;
|
|
const defaults = defaultMicConfig();
|
|
return {
|
|
...defaults,
|
|
...parsed,
|
|
hostMics: { ...defaults.hostMics, ...parsed.hostMics },
|
|
};
|
|
} catch {
|
|
return defaultMicConfig();
|
|
}
|
|
}
|
|
|
|
export function saveMicConfig(config: MicConfig): void {
|
|
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
|
|
}
|
|
|
|
class MicArrayController {
|
|
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
|
private statuses: MicArrayStatuses = { left: null, right: null };
|
|
// The Java listener proxy must stay referenced or it gets GC'd.
|
|
private statusProxy: any = null;
|
|
private proxyCommunicator: any = null;
|
|
private leaseTimer: ReturnType<typeof setInterval> | null = null;
|
|
private armed = false;
|
|
|
|
getStatuses(): MicArrayStatuses {
|
|
return this.statuses;
|
|
}
|
|
|
|
onStatus(listener: (statuses: MicArrayStatuses) => void): () => void {
|
|
this.statusListeners.add(listener);
|
|
listener(this.statuses);
|
|
return () => this.statusListeners.delete(listener);
|
|
}
|
|
|
|
/**
|
|
* True when both temples report the same active configuration; only then
|
|
* should cross-temple bearing fusion be trusted.
|
|
*/
|
|
arrayConsistent(): boolean {
|
|
const { left, right } = this.statuses;
|
|
if (!left || !right) return false;
|
|
return (
|
|
left.active &&
|
|
right.active &&
|
|
left.effectiveRateHz === right.effectiveRateHz &&
|
|
left.codec === right.codec &&
|
|
left.format === right.format
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Push the config to the temples. Each active temple streams its full
|
|
* front+rear pair (per-mic selection happens host-side after the split);
|
|
* a temple whose mics are all toggled off gets STOP instead so its stream
|
|
* doesn't spend BLE bandwidth.
|
|
*/
|
|
apply(config: MicConfig): boolean {
|
|
if (config.armHardware && !micControlSupported()) return false;
|
|
const leftActive = templeActive(config, "left");
|
|
const rightActive = templeActive(config, "right");
|
|
if (leftActive || rightActive) {
|
|
if (!this.send("configure", config, "configure", rightActive, leftActive)) return false;
|
|
}
|
|
if (!leftActive || !rightActive) {
|
|
this.send("stop", undefined, "stop-disabled-temple", !rightActive, !leftActive);
|
|
}
|
|
saveMicConfig(config);
|
|
this.armed = config.armHardware && (leftActive || rightActive);
|
|
this.syncLeaseTimer();
|
|
return true;
|
|
}
|
|
|
|
query(): boolean {
|
|
return this.send("query", undefined, "query");
|
|
}
|
|
|
|
stopStreaming(): void {
|
|
this.armed = false;
|
|
this.syncLeaseTimer();
|
|
this.send("stop", undefined, "stop");
|
|
}
|
|
|
|
private renewLease(): void {
|
|
if (!this.armed) return;
|
|
this.send("renew", undefined, "renew");
|
|
}
|
|
|
|
private send(
|
|
op: MicOp,
|
|
config: MicConfig | undefined,
|
|
label: string,
|
|
rightTemple = true,
|
|
leftTemple = true,
|
|
): boolean {
|
|
const communicator = activeCommunicator();
|
|
if (!communicator) return false;
|
|
try {
|
|
this.ensureStatusProxy(communicator);
|
|
const record = encodeMicControl(op, config);
|
|
communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple);
|
|
return true;
|
|
} catch (error) {
|
|
console.warn(`mic control ${label} failed: ${error}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private ensureStatusProxy(communicator: any): void {
|
|
if (this.statusProxy && this.proxyCommunicator === communicator) return;
|
|
if (this.statusProxy && this.proxyCommunicator) {
|
|
try {
|
|
this.proxyCommunicator.removeMicStatusListener(this.statusProxy);
|
|
} catch {
|
|
// The old communicator instance may be gone after a reconnect.
|
|
}
|
|
}
|
|
this.statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({
|
|
onMicStatus: (body: any, arm: string) => {
|
|
const status = decodeMicStatus(toUint8Array(body));
|
|
if (!status) return;
|
|
const side: MicSide = String(arm) === "L" ? "left" : "right";
|
|
this.statuses = { ...this.statuses, [side]: { ...status, side } };
|
|
this.statusListeners.forEach((listener) => listener(this.statuses));
|
|
},
|
|
});
|
|
this.proxyCommunicator = communicator;
|
|
communicator.addMicStatusListener(this.statusProxy);
|
|
}
|
|
|
|
private syncLeaseTimer(): void {
|
|
if (this.armed && !this.leaseTimer) {
|
|
this.leaseTimer = setInterval(() => this.renewLease(), LEASE_RENEW_MS);
|
|
} else if (!this.armed && this.leaseTimer) {
|
|
clearInterval(this.leaseTimer);
|
|
this.leaseTimer = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
export const micArrayController = new MicArrayController();
|