Merge branch 'calculator' of https://github.com/kalanihelekunihi/faceclaw into calculator

This commit is contained in:
James Babcock committed 2026-08-26 00:49:20 -07:00
commit f3ad55ca3d
126 files changed
+22253 -58

No files matched your search

+91
View File
@@ -36,6 +36,23 @@ export type PushToTalkOptions = {
* Ignored when the mic is already running for another holder.
*/
endpointing?: boolean;
/**
* "My voice only": verify the captured utterance against the enrolled
* wearer voice-print, and suppress the final transcript when someone else
* spoke. wearerEmbedding is the L2-normalized profile centroid.
*/
speakerVerification?: {
speakerModelPath: string;
wearerEmbedding: Float32Array;
threshold: number;
};
/** Spectral noise suppression on the decoded stream (Microphones config). */
noiseSuppression?: boolean;
/**
* Sonic Radar beam gating: only listen to the configured wedge (device
* frame). Null/absent leaves the mic omnidirectional.
*/
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
};
/** Who currently wants the mic running. */
@@ -43,6 +60,16 @@ type CaptureHolder = "ptt" | "continuous";
export type RawPcmListener = (pcm: Uint8Array) => void;
/**
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
* packet: direction-of-arrival in signed degrees (0 = straight ahead,
* positive to the wearer's right) and a signal-strength ratio used as a
* confidence/energy proxy. Computed on the raw stereo capture before the
* firmware's mono downmix, so it survives even though only mono PCM arrives.
*/
export type MicFrameMeta = { angleDegrees: number; ssr: number };
export type FrameMetaListener = (meta: MicFrameMeta) => void;
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
@@ -63,7 +90,13 @@ export class FaceclawVoiceControlBridge {
// listeners. STT capture preempts it — a live assistant/transcribe session
// owns the single G2 mic and the raw tap is stopped for its duration.
private readonly rawPcmListeners = new Set<RawPcmListener>();
private readonly frameMetaListeners = new Set<FrameMetaListener>();
private rawActive = false;
// Set when the current capture session runs "my voice only" verification;
// a non-wearer verdict arrives just before the final transcript and makes
// the bridge swallow it.
private verificationActive = false;
private verificationRejected = false;
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener);
@@ -110,6 +143,12 @@ export class FaceclawVoiceControlBridge {
return () => this.rawPcmListeners.delete(listener);
}
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
onFrameMeta(listener: FrameMetaListener): () => void {
this.frameMetaListeners.add(listener);
return () => this.frameMetaListeners.delete(listener);
}
/**
* Start the decode-only raw-PCM tap for an EvenHub mic app. Fails (returns
* false) if an STT capture already owns the mic; the caller re-tries when its
@@ -123,6 +162,11 @@ export class FaceclawVoiceControlBridge {
this.controller?.setCommunicator(options.communicator);
this.controller?.setSaveRecordings(false);
this.controller?.setEndpointing(false);
// Raw means raw: the Microphones session (and EvenHub apps) get the
// unprocessed stream and run their own beam/suppression processing.
this.controller?.setNoiseSuppression(false);
this.controller?.setBeamFilter(false, 0, 180);
this.controller?.clearSpeakerVerification();
this.cloudClient = null;
this.rawActive = true;
this.started = true;
@@ -155,6 +199,24 @@ export class FaceclawVoiceControlBridge {
this.controller?.setCommunicator(options.communicator);
this.controller?.setSaveRecordings(options.saveRecording);
this.controller?.setEndpointing(Boolean(options.endpointing));
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
if (options.beamFilter) {
this.controller?.setBeamFilter(true, Math.round(options.beamFilter.centerDeg), Math.round(options.beamFilter.halfWidthDeg));
} else {
this.controller?.setBeamFilter(false, 0, 180);
}
this.verificationRejected = false;
if (options.speakerVerification) {
this.verificationActive = true;
this.controller?.setSpeakerVerification(
options.speakerVerification.speakerModelPath,
toJavaFloats(options.speakerVerification.wearerEmbedding),
options.speakerVerification.threshold,
);
} else {
this.verificationActive = false;
this.controller?.clearSpeakerVerification();
}
const cloudClient = this.createCloudClient(options);
if (cloudClient) {
@@ -270,16 +332,37 @@ export class FaceclawVoiceControlBridge {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
},
onFrameMeta: (angleDegrees: number, ssr: number) => {
if (this.frameMetaListeners.size === 0) return;
const meta = { angleDegrees: Number(angleDegrees), ssr: Number(ssr) };
this.frameMetaListeners.forEach((listener) => listener(meta));
},
onSpeechEnd: () => {
for (const listener of this.speechEndListeners) {
listener();
}
},
onSpeakerVerified: (isWearer: boolean, similarity: number) => {
if (!this.verificationActive) return;
this.verificationRejected = !isWearer;
if (!isWearer) {
this.setStatus(
`Ignored — not your enrolled voice (match ${Math.round(Number(similarity) * 100)}%).`,
);
}
},
});
this.controller.setListener(this.listenerProxy);
}
private emitTranscript(text: string, isFinal: boolean): void {
// "My voice only": a rejected session's final transcript is emptied so
// nothing downstream (assistant, dictation) acts on another speaker's
// words. The verification verdict is posted before the final, and cloud
// finals arrive over the network later still, so the flag is settled.
if (isFinal && this.verificationActive && this.verificationRejected) {
text = "";
}
const event = { text, isFinal };
for (const listener of this.transcriptListeners) {
listener(event);
@@ -294,4 +377,12 @@ export class FaceclawVoiceControlBridge {
}
}
function toJavaFloats(values: Float32Array): any {
const javaFloats = Array.create("float", values.length);
for (let i = 0; i < values.length; i++) {
javaFloats[i] = values[i]!;
}
return javaFloats;
}
export const voiceControlBridge = new FaceclawVoiceControlBridge();