import type { OpenAiSttModel } from "./openai-stt"; import { Utils } from "@nativescript/core"; import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt"; import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider"; import { newRecordingPath } from "./mic-recording"; import { SpeechPauseDetector } from "./speech-pause"; import { toUint8Array } from "../util/array-util"; import { pcmLevel } from "../util/audio-level"; declare const com: any; export type VoiceControlState = { /** One-line state of the capture: progress, "Listening...", or an error. */ status: string; /** * Whether audio is actually being captured and transcribed right now. False * while the mic is starting, while the phone is showing the permission * prompt, and after an error, so a dialog must not claim to be listening. */ listening: boolean; /** * Longer guidance to show in place of the (still empty) transcript, e.g. * what the user has to do on the phone. Empty when there is nothing to say. */ detail: string; }; // "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model, // see openai-stt.ts) -- an unfortunate pre-existing name collision with the model // family, not with "onboard-whisper" below, which is the on-device sherpa-onnx // Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than // renamed, since it's a persisted settings value on real installs already. // "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP // (whisper-server-stt.ts). export type VoiceProviderKind = "onboard" | "onboard-whisper" | "onboard-whisper-fr" | "elevenlabs" | "whisper" | "soniox" | "whisper-server"; export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** * Complete best transcript of the current utterance. REPLACE semantics — * render as-is, replacing any previous partial. Not a delta. */ text: string; isFinal: boolean; }; export type PushToTalkOptions = { /** Native G2 communicator, or null in preview-only mode (see usePhoneMic). */ communicator: any; /** * Capture from the phone's own microphone instead of the G2 over BLE. * Preview-only mode, where no glasses are connected; everything downstream * (providers, endpointing, verification) works the same on either source. */ usePhoneMic?: boolean; provider: VoiceProviderKind; elevenLabsApiKey: string; openAiApiKey: string; openAiTranscriptionModel?: OpenAiSttModel; sonioxApiKey: string; whisperServerUrl?: string; whisperServerModel?: string; whisperServerApiKey?: string; /** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */ language?: string; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free * ("Hey Even") capture, which has no button release to end the utterance. * Ignored when the mic is already running for another holder. */ endpointing?: boolean; /** * "My voice only": verify the captured utterance against the enrolled * wearer voice-print, and suppress the final transcript when someone else * spoke. wearerEmbedding is the L2-normalized profile centroid. */ speakerVerification?: { speakerModelPath: string; wearerEmbedding: Float32Array; threshold: number; }; /** Spectral noise suppression on the decoded stream (Microphones config). */ noiseSuppression?: boolean; /** * Sonic Radar beam gating: only listen to the configured wedge (device * frame). Null/absent leaves the mic omnidirectional. */ beamFilter?: { centerDeg: number; halfWidthDeg: number } | null; /** * Capture from the temples' own microphones (the custom firmware's * mic_control stream) instead of the glasses' mixed stream. Each temple * streams its microphones on its own link and clock; the phone mixes them * into the mono feed that is transcribed. Null/absent keeps the mixed * stream; ignored with usePhoneMic. */ micArray?: MicArrayCapture | null; /** * Play this recording (a .mka file from Save Recording) in real time in * place of any microphone. The capture needs no glasses, and ends by * itself when the recording does. */ replay?: { path: string; startMs: number } | null; /** What the caller knows of the capture, for the description stored in a recording of it. */ captureInfo?: Record; }; export type MicArrayCapture = { /** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */ configure: Uint8Array; }; /** Progress of a capture that replays a recording; positions are ms into it. */ export type VoiceReplayEvent = { /** * `start` once the file is open, `position` about once per second played, * `seek` when a jump took effect, `end` when the recording ran out, and * `error` when it could not be opened (the status says why). */ kind: "start" | "position" | "seek" | "end" | "error"; positionMs: number; durationMs: number; }; /** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */ export type VoiceEventSource = { onStatus(listener: (state: VoiceControlState) => void): () => void; onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void; onSpeechPause(listener: () => void): () => void; onMicLevel(listener: (level: number) => void): () => void; readonly recording: boolean; readonly recordingFailed: boolean; }; /** Who currently wants the mic running. */ type CaptureHolder = "ptt" | "continuous"; export type RawPcmListener = (pcm: Uint8Array) => void; /** * Per-packet DSP metadata the glasses firmware appends to each 50 ms audio * packet: direction-of-arrival in signed degrees (0 = straight ahead, * positive to the wearer's right) and a signal-strength ratio used as a * confidence/energy proxy. Computed on the raw stereo capture before the * firmware's mono downmix, so it survives even though only mono PCM arrives. */ export type MicFrameMeta = { angleDegrees: number; ssr: number }; export type FrameMetaListener = (meta: MicFrameMeta) => void; export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>(); private readonly speechPauseListeners = new Set<() => void>(); private readonly speechPause = new SpeechPauseDetector(); private readonly speechEndListeners = new Set<() => void>(); private controller: any = null; private listenerProxy: any = null; private status = "Voice control stopped."; private listening = false; private detail = ""; private started = false; private nativeCaptureFinished: Promise = Promise.resolve(); private nextNativeCaptureId = 1; private readonly nativeCaptureResolvers = new Map void>(); // The mic is a single shared stream; these are the reasons it is running. // The first holder starts capture (choosing the provider); the mic stops // when the last one releases. Transcript events broadcast to every // listener, so push-to-talk and the Transcribe window both receive text. private readonly captureHolders = new Set(); // Holders whose stream died with the glasses session (transport drop, // charging case, EvenHub suspend). They still want the mic, so the next // session restarts it for them; see handleSessionEnded/resumeCapture. private readonly suspendedHolders = new Set(); private suspendedRaw = false; // Endpointing of the live capture, replayed when it is resumed. private captureEndpointing = false; // Non-null while a cloud provider owns the transcript; Java only decodes PCM. private cloudClient: CloudSttClient | null = null; // What the running STT stream captures from and transcribes with // (streamKey); a holder asking for something else restarts the stream // rather than sharing it. private streamProvider: string | null = null; // The options of the running stream, for the description of a recording of it. private streamOptions: PushToTalkOptions | null = null; private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>(); // Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the // controller runs in the decode-only "cloud" mode and every decoded PCM // frame is broadcast to these listeners. STT capture preempts it — a live // assistant/transcribe session owns the single G2 mic and the raw tap is // stopped for its duration. private readonly rawPcmListeners = new Set(); private readonly frameMetaListeners = new Set(); private readonly micLevelListeners = new Set<(level: number) => void>(); private rawActive = false; // Who wants the raw tap. It runs while any owner does, so one owner letting // go (an EvenHub app backgrounding) doesn't silence another. private readonly rawOwners = new Set(); // Set when the current capture session runs "my voice only" verification; // a non-wearer verdict arrives just before the final transcript and makes // the bridge swallow it. private verificationActive = false; private verificationRejected = false; // Continuous captures' audio goes to Downloads/.mka while it is // set: every source's chunks as they arrive, before any processing, written // by the controller. The file outlives a capture, so a restarted one keeps // adding to it, and it is created by the first audio, so a failed start // leaves no file. private recordingName = ""; // The file is open in the controller. private recordingOpen = false; // The file could not be created: one attempt per recording, not one per packet. private recordingFailure = false; private readonly recordingListeners = new Set<() => void>(); onStatus(listener: (state: VoiceControlState) => void): () => void { this.statusListeners.add(listener); listener(this.state()); return () => this.statusListeners.delete(listener); } onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void { this.transcriptListeners.add(listener); return () => this.transcriptListeners.delete(listener); } /** * The speaker stopped, in a hands-free session. Only fires when the capture * was started with `endpointing: true`. */ onSpeechEnd(listener: () => void): () => void { this.speechEndListeners.add(listener); return () => this.speechEndListeners.delete(listener); } /** Pauses in on-device recognition, for Transcribe's presentation only. */ onSpeechPause(listener: () => void): () => void { this.speechPauseListeners.add(listener); return () => this.speechPauseListeners.delete(listener); } /** * The capture is blocked on the phone's system microphone-permission * prompt. Reported by the controller before it asks Android, so the dialog * on the glasses can send the user to the phone instead of claiming to * listen; the next status (from the capture start, or reportPermissionDenied) * replaces it. */ reportPermissionPrompt(): void { this.setStatus( "Waiting for microphone permission...", "Allow microphone access in the prompt on your phone.", ); } /** The user declined the microphone permission (or the prompt could not be shown). */ reportPermissionDenied(): void { this.setStatus( "Microphone access denied.", "Enable the microphone for Faceclaw in the phone's app settings, then try again.", ); } /** * Capture could not be started for a reason other than a denied permission. * Without this the dialog keeps showing the idle "Voice control stopped." */ reportStartFailed(reason: string): void { this.setStatus("Voice input couldn't start.", reason); } /** Begin push-to-talk capture (momentary; released with stopPushToTalk). */ startPushToTalk(options: PushToTalkOptions): void { this.acquireCapture("ptt", options); } /** End push-to-talk: for cloud, commit for a final result if it was the last holder. */ stopPushToTalk(): Promise | void { // Shared continuous capture keeps running; cloud providers finalize via // their own callbacks. Only a stopped native recognizer must be awaited. const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1 ? this.nativeCaptureFinished : undefined; this.releaseCapture("ptt", true); return finished; } /** Begin continuous capture (Transcribe): the mic stays on until released. */ startContinuousCapture(options: PushToTalkOptions): void { this.acquireCapture("continuous", options); } stopContinuousCapture(): void { this.releaseCapture("continuous", false); } /** * Restart the running continuous capture when it transcribes with another * provider than `options` asks for (a window that needs a multilingual one * opened or closed), keeping every holder on the new stream. A parked * capture is left alone: it takes the options when it resumes. */ retargetContinuousCapture(options: PushToTalkOptions): void { if (!this.captureHolders.has("continuous") || !this.micIsLive()) return; this.acquireCapture("continuous", options); } /** * End the continuous capture as an utterance: stop it and have the * transcript of its last audio come as a final. For a capture whose audio * ran out by itself (a replay). */ finishContinuousCapture(): void { this.releaseCapture("continuous", true); } /** Progress of a capture that replays a recording (PushToTalkOptions.replay). */ onReplay(listener: (event: VoiceReplayEvent) => void): () => void { this.replayListeners.add(listener); return () => this.replayListeners.delete(listener); } /** Continue the running replay from `positionMs` into its recording. */ seekReplay(positionMs: number): void { this.controller?.seekReplay(Math.max(0, Math.round(positionMs))); } /** Hold the running replay where it is, or let it go on. */ setReplayPaused(paused: boolean): void { this.controller?.setReplayPaused(paused); } /** Whether a capture is running and still has audio coming (a replay stops being so at its end). */ isCaptureLive(): boolean { return this.micIsLive(); } /** * Record continuous captures' audio to Downloads/.mka, across restarts, * until called with another name or "". */ recordTo(name: string): void { if (name === this.recordingName) return; this.closeRecording(); this.setRecording(name, false); } private closeRecording(): void { if (!this.recordingOpen) return; this.recordingOpen = false; try { this.controller?.stopRecording(); } catch (error) { console.warn(`Voice: closing the recording failed: ${error}`); } } /** Whether continuous captures' audio is being saved (see recordTo). */ get recording(): boolean { return this.recordingName !== "" && !this.recordingFailure; } /** Whether the recording asked for could not be saved (its file could not be created). */ get recordingFailed(): boolean { return this.recordingFailure; } /** `recording` or `recordingFailed` changed. */ onRecordingChange(listener: () => void): () => void { this.recordingListeners.add(listener); return () => this.recordingListeners.delete(listener); } private setRecording(name: string, failed: boolean): void { const wasRecording = this.recording; const hadFailed = this.recordingFailure; this.recordingName = name; this.recordingFailure = failed; if (wasRecording === this.recording && hadFailed === failed) return; for (const listener of Array.from(this.recordingListeners)) listener(); } /** * Audio arrived for a continuous capture: open the recording asked for, if * it isn't open yet. The controller writes it from then on, as the audio * arrives on its own thread. */ private ensureRecording(): void { if (!this.recording || this.recordingOpen) return; const path = newRecordingPath(this.recordingName); let opened = false; try { opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture()))); } catch (error) { console.warn(`Voice: could not create the recording: ${error}`); } if (!opened) { this.setRecording(this.recordingName, true); return; } this.recordingOpen = true; console.log(`Voice: recording audio to ${path}`); } /** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */ private syncRecordingEnabled(): void { this.controller?.setRecordingEnabled(this.captureHolders.has("continuous")); } /** The running capture, as the JSON stored in a recording of it. */ private describeCapture(): Record { const options = this.streamOptions; return { recording: this.recordingName, startedAt: new Date().toISOString(), source: options?.usePhoneMic ? "phone" : "glasses", micArray: Boolean(options?.micArray) && !options?.usePhoneMic, transcription: { provider: options?.provider ?? "", language: options?.language ?? "" }, processing: { noiseSuppression: Boolean(options?.noiseSuppression), beamFilter: options?.beamFilter ?? null, }, ...options?.captureInfo, }; } /** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */ onRawPcm(listener: RawPcmListener): () => void { this.rawPcmListeners.add(listener); return () => this.rawPcmListeners.delete(listener); } /** * Subscribe to the mic level (meter position in [0,1], see pcmLevel) of * each decoded audio chunk, about every 50 ms, from any capture: STT in * every mode, or the raw tap. */ onMicLevel(listener: (level: number) => void): () => void { this.micLevelListeners.add(listener); return () => this.micLevelListeners.delete(listener); } /** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */ onFrameMeta(listener: FrameMetaListener): () => void { this.frameMetaListeners.add(listener); return () => this.frameMetaListeners.delete(listener); } /** * Start the decode-only raw-PCM tap on behalf of `owner`. Fails (returns * false) if an STT capture already owns the mic; the owner stays registered * and re-tries later (the EvenHub router when eligibility is re-evaluated, * the Microphones app from its capture check). Idempotent while already * running. */ startRawCapture(options: { communicator: any; owner: string }): boolean { if (!global.isAndroid) return false; this.rawOwners.add(options.owner); return this.startRawTap(options.communicator); } /** `owner` no longer wants the raw tap; it stops once no owner does. */ stopRawCapture(owner: string): void { this.rawOwners.delete(owner); if (this.rawOwners.size === 0) this.endRawTap(); } /** * Whether the raw tap is running and its glasses mic enable still holds * (or is still coming up). False after an STT preemption or a session drop. */ isRawCaptureLive(): boolean { return this.rawActive && this.micIsLive(); } private startRawTap(communicator: any): boolean { if (!global.isAndroid) return false; if (this.rawActive && this.micIsLive()) return true; // An STT holder owns the mic even while its stream waits on a new glasses // session; the tap re-tries when the app's eligibility is re-evaluated. if (this.captureHolders.size > 0 || this.suspendedHolders.size > 0) return false; // A tap left over from a dead session is not something to share. if (this.started || this.rawActive) this.teardownCapture(); this.suspendedRaw = false; this.ensureController(); this.controller?.setCommunicator(communicator); // The raw tap is strictly the G2 stream; clear any phone-mic flag a // preview-mode capture may have left on the shared controller. this.controller?.setUsePhoneMic(false); this.controller?.setEndpointing(false); this.controller?.setMicArray(null); this.controller?.setReplay(null, 0); // Raw means raw: the Microphones session (and EvenHub apps) get the // unprocessed stream and run their own beam/suppression processing. this.controller?.setNoiseSuppression(false); this.controller?.setBeamFilter(false, 0, 180); this.controller?.clearSpeakerVerification(); this.cloudClient = null; this.rawActive = true; this.started = true; // "cloud" mode decodes LC3 to PCM and emits it via onPcm without loading // any recognizer; with no cloudClient the frames reach only the raw tap. this.controller?.start("cloud"); return true; } private endRawTap(): void { this.suspendedRaw = false; if (!this.rawActive) return; this.rawActive = false; this.started = false; if (global.isAndroid) this.controller?.stop(); } private acquireCapture(holder: CaptureHolder, options: PushToTalkOptions): void { if (!global.isAndroid) return; // STT preempts the raw-PCM tap: a live transcription session owns the mic, // and an EvenHub app gets no audio for its duration. if (this.rawActive) this.endRawTap(); // Whether the mic is already running is a question about the stream, not // about this set: a holder whose capture died with the glasses session // (or whose mic enable was refused) is still in it. Trusting the set here // let one stale holder silently swallow every later request for the // process's lifetime. const micLive = this.micIsLive(); this.suspendedHolders.delete(holder); this.captureHolders.add(holder); if (micLive && this.streamProvider === streamKey(options)) { // Mic already running; the new holder just shares the existing stream // (transcripts are already broadcast to its listeners). this.syncRecordingEnabled(); return; } // Any leftover capture is dead, or captures or transcribes otherwise than // this holder needs; drop it before starting the new one. if (this.started) this.teardownCapture(); this.streamProvider = streamKey(options); this.streamOptions = options; this.ensureController(); this.syncRecordingEnabled(); // A recording under way carries on into this capture: note what changed. if (this.recordingOpen) { try { this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() })); } catch (error) { console.warn(`Voice: recording event failed: ${error}`); } } this.controller?.setCommunicator(options.communicator); this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic)); this.controller?.setMicArray( options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null, ); this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0))); this.controller?.setEndpointing(Boolean(options.endpointing)); this.captureEndpointing = Boolean(options.endpointing); this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression)); if (options.beamFilter) { this.controller?.setBeamFilter(true, Math.round(options.beamFilter.centerDeg), Math.round(options.beamFilter.halfWidthDeg)); } else { this.controller?.setBeamFilter(false, 0, 180); } this.verificationRejected = false; if (options.speakerVerification) { this.verificationActive = true; this.controller?.setSpeakerVerification( options.speakerVerification.speakerModelPath, toJavaFloats(options.speakerVerification.wearerEmbedding), options.speakerVerification.threshold, ); } else { this.verificationActive = false; this.controller?.clearSpeakerVerification(); } this.speechPause.reset(); // Replaces whatever the last capture left behind ("Listening...", an // error, the permission prompt) until the controller reports progress. this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone..."); // A previous push-to-talk commit may still be awaiting its final result. this.cloudClient?.stop(); const cloudClient = this.createCloudClient(options); if (cloudClient) { this.cloudClient = cloudClient; cloudClient.start(); this.started = true; this.controller?.start("cloud"); return; } this.cloudClient = null; this.started = true; // Which on-device model to load; a no-op setter for every provider except // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). const frenchWhisper = options.provider === "onboard-whisper-fr"; this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" || frenchWhisper ? "whisper" : "moonshine"); this.controller?.setFrenchWhisper(frenchWhisper); this.controller?.setOnboardLanguage(frenchWhisper ? "fr" : options.language ?? "en"); const captureId = this.nextNativeCaptureId++; this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); }); this.controller?.start("onboard", captureId); } /** * The cloud provider for this session, or null to transcribe on-device. * A cloud provider whose API key is missing falls back to on-device rather * than failing the capture outright. */ private createCloudClient(options: PushToTalkOptions): CloudSttClient | null { return createCloudSttClient(options, { onTranscript: (event: CloudSttTranscriptEvent) => this.emitTranscript(event.text, event.isFinal, event), onStatus: (status: string) => this.setStatus(status), onError: (message: string) => this.setStatus(message), }, () => this.captureHolders.has("continuous")); } private releaseCapture(holder: CaptureHolder, commit: boolean): void { // A holder that gives up while its capture is parked must not be resumed // by the next session. this.suspendedHolders.delete(holder); if (!this.captureHolders.delete(holder)) { // Never held; still finish a dangling cloud commit if one is pending. if (commit) this.cloudClient?.finish(); return; } this.syncRecordingEnabled(); if (this.captureHolders.size > 0) { // Another holder still wants the mic; keep it running. return; } if (!this.started) { this.cloudClient?.finish(); return; } // Order matters for cloud: stopping the Java controller flushes any final // decode/PCM; then commit so the provider finalizes the transcript. this.controller?.stop(); this.started = false; if (commit) { this.cloudClient?.finish(); } else { this.cloudClient?.stop(); this.cloudClient = null; } } stop(): void { this.captureHolders.clear(); this.suspendedHolders.clear(); this.suspendedRaw = false; this.syncRecordingEnabled(); this.teardownCapture(); this.setStatus("Voice control stopped."); } /** * The glasses session ended under a live capture (transport drop, charging * case, EvenHub suspend). Its mic enable did not survive, so drop the local * capture state — leaving it in place would make the holder set claim a * stream that no longer exists — and remember who was holding so * resumeCapture() can bring the mic back on the next session. */ handleSessionEnded(): void { if (this.captureHolders.size === 0 && !this.started && !this.rawActive) return; for (const holder of this.captureHolders) { this.suspendedHolders.add(holder); } this.captureHolders.clear(); this.syncRecordingEnabled(); this.suspendedRaw = this.rawActive; this.teardownCapture(); if (this.suspendedHolders.size > 0) { this.setStatus("Waiting for the glasses..."); } } /** Whether a capture is parked waiting for a session to come back. */ hasSuspendedCapture(): boolean { return this.suspendedHolders.size > 0 || this.suspendedRaw; } /** Whether `holder` holds the mic, including across a session outage. */ holdsCapture(holder: CaptureHolder): boolean { return this.captureHolders.has(holder) || this.suspendedHolders.has(holder); } /** Whether anyone holds the mic, including across a session outage. */ isCaptureHeld(): boolean { return this.captureHolders.size > 0 || this.suspendedHolders.size > 0; } /** * A fresh glasses session is ready (its display path warmed up, so a mic * enable will be accepted): restart the mic for every holder that was * capturing when the last session ended. Mic permission was granted for the * original capture and cannot be revoked without restarting the process, so * this does not ask again. */ resumeCapture(options: Omit): void { const holders = Array.from(this.suspendedHolders); const resumeRaw = this.suspendedRaw; this.suspendedHolders.clear(); this.suspendedRaw = false; for (const holder of holders) { this.acquireCapture(holder, { ...options, endpointing: this.captureEndpointing }); } if (resumeRaw && !holders.length && this.rawOwners.size > 0) { this.startRawTap(options.communicator); } } /** * Whether the glasses mic is actually streaming. The holder set and * `started` are phone-side bookkeeping; the enable itself lives in the * glasses session, so the controller (via the communicator) is the only * authority on whether audio is still flowing. */ private micIsLive(): boolean { if (!global.isAndroid || !this.started) return false; return Boolean(this.controller?.isCapturing()); } /** Stop the native capture and any cloud client; leaves holders alone. */ private teardownCapture(): void { this.rawActive = false; this.streamProvider = null; this.streamOptions = null; if (global.isAndroid) { this.controller?.stop(); } this.started = false; this.cloudClient?.stop(); this.cloudClient = null; } private ensureController(): void { if (!global.isAndroid || this.controller) return; const context = Utils.android.getApplicationContext(); if (!context) { throw new Error("Android application context unavailable"); } this.controller = new com.faceclaw.app.FaceclawVoiceController(context); this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({ onStatus: (status: string) => { // The raw tap runs silently; its "Listening (cloud)..." chatter must // not surface on the glasses voice status. if (this.rawActive) return; this.setStatus(String(status)); }, onTranscript: (text: string, isFinal: boolean) => { this.emitTranscript(String(text), Boolean(isFinal)); }, onStopped: (captureId: number) => { this.nativeCaptureResolvers.get(captureId)?.(); this.nativeCaptureResolvers.delete(captureId); }, onPcm: (pcm: any) => { const bytes = toUint8Array(pcm); this.cloudClient?.acceptPcm(bytes); // Push-to-talk on a capture of its own is not recorded. if (this.captureHolders.has("continuous")) this.ensureRecording(); if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) { for (const listener of this.speechPauseListeners) listener(); } if (this.rawPcmListeners.size > 0) { this.rawPcmListeners.forEach((listener) => listener(bytes)); } if (this.micLevelListeners.size > 0) { const level = pcmLevel(bytes); this.micLevelListeners.forEach((listener) => listener(level)); } }, onFrameMeta: (angleDegrees: number, ssr: number) => { if (this.frameMetaListeners.size === 0) return; const meta = { angleDegrees: Number(angleDegrees), ssr: Number(ssr) }; this.frameMetaListeners.forEach((listener) => listener(meta)); }, onSpeechEnd: () => { for (const listener of this.speechEndListeners) { listener(); } }, onSpeakerVerified: (isWearer: boolean, similarity: number) => { if (!this.verificationActive) return; this.verificationRejected = !isWearer; if (!isWearer) { this.setStatus( `Ignored — not your enrolled voice (match ${Math.round(Number(similarity) * 100)}%).`, ); } }, onReplay: (kind: string, positionMs: number, durationMs: number) => { const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) }; for (const listener of Array.from(this.replayListeners)) listener(event); }, }); this.controller.setListener(this.listenerProxy); } private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void { // "My voice only": a rejected session's final transcript is emptied so // nothing downstream (assistant, dictation) acts on another speaker's // words. The verification verdict is posted before the final, and cloud // finals arrive over the network later still, so the flag is settled. if (isFinal && this.verificationActive && this.verificationRejected) { text = ""; metadata = undefined; } const event = { ...metadata, text, isFinal }; for (const listener of this.transcriptListeners) { listener(event); } } private setStatus(status: string, detail = ""): void { this.status = status; this.detail = detail; // Every "audio is flowing" report — from the Java controller and from the // cloud providers — is phrased "Listening..." / "Listening (X)...". this.listening = status.startsWith("Listening"); const state = this.state(); for (const listener of this.statusListeners) { listener(state); } } private state(): VoiceControlState { return { status: this.status, listening: this.listening, detail: this.detail }; } } /** * What a capture reads and transcribes with. A running capture is shared only * by holders asking for the same: the provider and language, the microphones * (the mixed stream, or the temples' own under a given configuration), or the * recording it replays. */ function streamKey(options: PushToTalkOptions): string { const source = options.replay ? `replay:${options.replay.path}` : options.micArray && !options.usePhoneMic ? `array:${Array.from(options.micArray.configure).join(",")}` : "mixed"; return `${transcriptionKey(options)}|${source}`; } function toJavaFloats(values: Float32Array): any { const javaFloats = Array.create("float", values.length); for (let i = 0; i < values.length; i++) { javaFloats[i] = values[i]!; } return javaFloats; } export const voiceControlBridge = new FaceclawVoiceControlBridge(); export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed"; /** * A recording played back as a capture of its own: transcribed by the same * pipeline as the microphone, with the same settings, on a bridge and native * controller separate from the microphone's. Nothing here touches the mic, * so voice input and a live Transcribe window carry on beside a replay. */ export class VoiceReplay { private readonly bridge = new FaceclawVoiceControlBridge(); private readonly listeners = new Set<(jumpedToMs?: number) => void>(); private readonly offReplay: () => void; private position = 0; private duration = 0; private current: VoiceReplayState = "opening"; private closed = false; /** * `options` gives the capture options (provider, keys, processing) each * time the replay starts, so a restart picks up changed settings. */ constructor(readonly path: string, private readonly options: () => PushToTalkOptions) { this.offReplay = this.bridge.onReplay((event) => this.handle(event)); } /** The replay's status, transcript and level events. */ get events(): VoiceEventSource { return this.bridge; } /** Milliseconds into the recording. */ get positionMs(): number { return this.position; } /** The recording's length in ms; 0 until it has been opened. */ get durationMs(): number { return this.duration; } get state(): VoiceReplayState { return this.current; } /** * The position, duration or state changed. `jumpedToMs` is given when * playback moved there other than by playing on: a jump, or a start * part-way in. It comes after the transcript of what played before. */ onChange(listener: (jumpedToMs?: number) => void): () => void { this.listeners.add(listener); return () => this.listeners.delete(listener); } /** Play from `positionMs` into the recording. */ start(positionMs = 0): void { if (this.closed) return; this.position = Math.max(0, positionMs); this.set("opening"); this.bridge.startContinuousCapture({ ...this.options(), communicator: null, usePhoneMic: false, micArray: null, replay: { path: this.path, startMs: this.position }, }); } /** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */ jumpTo(positionMs: number): void { if (this.closed) return; const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs); if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) { this.bridge.seekReplay(target); this.position = target; this.notify(); return; } this.start(target); } setPaused(paused: boolean): void { if (this.closed || (this.current !== "playing" && this.current !== "paused")) return; this.bridge.setReplayPaused(paused); this.set(paused ? "paused" : "playing"); } /** Stop for good and release the capture. */ close(): void { if (this.closed) return; this.closed = true; this.offReplay(); this.listeners.clear(); this.bridge.stop(); } private handle(event: VoiceReplayEvent): void { this.duration = event.durationMs; switch (event.kind) { case "start": this.position = event.positionMs; this.current = "playing"; this.notify(event.positionMs > 0 ? event.positionMs : undefined); return; case "seek": this.position = event.positionMs; this.notify(event.positionMs); return; case "end": this.position = event.durationMs; // Its last audio is still to be transcribed (a cloud provider's final). this.bridge.finishContinuousCapture(); this.set("ended"); return; case "error": this.set("failed"); return; default: this.position = event.positionMs; this.notify(); } } private set(state: VoiceReplayState): void { this.current = state; this.notify(); } private notify(jumpedToMs?: number): void { for (const listener of Array.from(this.listeners)) listener(jumpedToMs); } }