diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index e2f26a64..927d202b 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -360,6 +360,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + /** + * Whether the glasses mic is enabled right now. The enable lives in the + * current EvenHub session, so it dies with a transport drop, the charging + * case, or a suspend — silently, from the phone's point of view. Callers + * that track a capture across those events must check this rather than + * assume their earlier enable still holds. + */ + public boolean isAudioCaptureActive() { + synchronized (lock) { + return running && sessionReady && !shutdownRequested && audioCaptureActive; + } + } + /** * Acquire/renew or release CFW's volatile wake-takeover lease on both * arms. Delivery (not a protocol ACK) is awaited so a caller can ensure diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index e0a75ab3..2bdacece 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -76,6 +76,9 @@ public class FaceclawVoiceController { private volatile FaceclawBleCommunicator communicator; private Thread workerThread; private volatile boolean started; + // Set once the worker has the glasses mic enabled for this session. + // Read and written under `lock`, so it flips with `started` atomically. + private boolean audioStarted; private VoiceInputMode mode = VoiceInputMode.CLOUD; private OfflineRecognizer recognizer; private FaceclawLc3Decoder lc3Decoder; @@ -137,11 +140,36 @@ public class FaceclawVoiceController { } mode = parseMode(requestedMode); started = true; + audioStarted = false; workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); workerThread.start(); } } + /** + * Whether mic audio is actually flowing. {@link #start} only records + * intent: the enable lives in the glasses' EvenHub session, so a transport + * drop or a session suspend can leave this controller started with a + * worker that will never see another packet. Anything deciding whether to + * (re)start capture must ask this rather than assume its own bookkeeping. + */ + public boolean isCapturing() { + boolean audioUp; + synchronized (lock) { + if (!started) { + return false; + } + audioUp = audioStarted; + } + if (!audioUp) { + // The worker is still bringing the mic up; report it as running so + // a concurrent request shares it instead of restarting it. + return true; + } + FaceclawBleCommunicator currentCommunicator = communicator; + return currentCommunicator != null && currentCommunicator.isAudioCaptureActive(); + } + public void stop() { Thread threadToJoin; synchronized (lock) { @@ -200,6 +228,9 @@ public class FaceclawVoiceController { emitStatus("Could not start G2 microphone input."); return; } + synchronized (lock) { + audioStarted = true; + } emitStatus(currentMode == VoiceInputMode.CLOUD ? "Listening (cloud)..." @@ -220,6 +251,7 @@ public class FaceclawVoiceController { releaseLc3(); synchronized (lock) { started = false; + audioStarted = false; workerThread = null; } } diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 66565521..5ba35a8b 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -702,6 +702,14 @@ class DashboardController { return; } + if (voiceControlBridge.isCaptureHeld()) { + // Ending the plugin task ends the mic with it, so a live capture (the + // Transcribe window, typically) outranks the screen-off power save. + // Check before the lease refresh below, which costs a BLE message. + this.scheduleEvenHubSuspend(); + return; + } + void (async () => { if (this.faceclawWakeLeaseSupported) { // Refresh immediately before teardown so the first suspended wake @@ -727,6 +735,9 @@ class DashboardController { .then((suspended) => { if (suspended === undefined) return; if (suspended) { + // A capture that raced the suspend lost its mic with the plugin + // task; park it for the resume path. + voiceControlBridge.handleSessionEnded(); this.appendLog("EvenHub session suspended while screen is off"); return; } @@ -1036,6 +1047,10 @@ class DashboardController { // the transport comes back, so do not make lock decisions from a // stale pre-disconnect value in the meantime. this.glassesWorn = null; + // The mic enable was session-scoped too: park any live capture so + // the next session restarts it, instead of leaving a holder that + // makes every later request think the mic is already running. + voiceControlBridge.handleSessionEnded(); } this.setPhase(mappedPhase); this.setStatus(state.status); @@ -1098,6 +1113,10 @@ class DashboardController { this.schedulePreviewUpdate(); if (this.phase === "connected") { this.setStatus("Connected."); + // A rendered frame also means the display path is warm enough for + // the glasses to accept a mic enable, so this is where a capture + // parked by the previous session (or by an EvenHub suspend) resumes. + this.resumeVoiceCapture(); // A rendered frame means the session is warmed up (fixedLayoutCreated), // so the buzzer won't be dropped. Play the one-time welcome sound now. if (this.welcomeSoundArmed) { @@ -1390,7 +1409,7 @@ class DashboardController { // Faceclaw's final BLE message before the transport closes. await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); - voiceControlBridge.stop(); + voiceControlBridge.handleSessionEnded(); const cleanupAcked = skipFirmwareCleanup ? false @@ -1558,6 +1577,30 @@ class DashboardController { voiceControlBridge.stopContinuousCapture(); } + /** Provider and key settings for a capture on the given connection. */ + private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge) { + return { + communicator: communicator.getNativeCommunicator(), + provider: voiceProviderSetting.get(), + elevenLabsApiKey: elevenLabsApiKeySetting.get(), + openAiApiKey: openAiApiKeySetting.get(), + sonioxApiKey: sonioxApiKeySetting.get(), + saveRecording: saveVoiceRecordingsSetting.get(), + }; + } + + /** + * Restart the mic for a capture that a previous session left holding. Cheap + * to call per frame: it does nothing unless one is actually parked. + */ + private resumeVoiceCapture(): void { + if (!voiceControlBridge.hasSuspendedCapture()) return; + const communicator = this.communicator; + if (!communicator || this.phase !== "connected") return; + this.appendLog("resuming voice capture on the new glasses session"); + voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator)); + } + private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void { if (this.phase !== "connected" || !this.communicator) { return; @@ -1566,15 +1609,7 @@ class DashboardController { void ensureVoicePermissions() .then(() => { if (this.phase !== "connected" || this.communicator !== communicator) return; - const options = { - communicator: communicator.getNativeCommunicator(), - provider: voiceProviderSetting.get(), - elevenLabsApiKey: elevenLabsApiKeySetting.get(), - openAiApiKey: openAiApiKeySetting.get(), - sonioxApiKey: sonioxApiKeySetting.get(), - saveRecording: saveVoiceRecordingsSetting.get(), - endpointing, - }; + const options = { ...this.voiceCaptureOptions(communicator), endpointing }; if (kind === "ptt") { voiceControlBridge.startPushToTalk(options); } else { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 121acf09..74aeb812 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -56,6 +56,13 @@ export class FaceclawVoiceControlBridge { // when the last one releases. Transcript events broadcast to every // listener, so push-to-talk and the Transcribe window both receive text. private readonly captureHolders = new Set(); + // Holders whose stream died with the glasses session (transport drop, + // charging case, EvenHub suspend). They still want the mic, so the next + // session restarts it for them; see handleSessionEnded/resumeCapture. + private readonly suspendedHolders = new Set(); + private suspendedRaw = false; + // Endpointing of the live capture, replayed when it is resumed. + private captureEndpointing = false; // Non-null while a cloud provider owns the transcript; Java only decodes PCM. private cloudClient: CloudSttClient | null = null; // Raw-PCM tap (EvenHub mic apps): when active, the controller runs in the @@ -117,8 +124,13 @@ export class FaceclawVoiceControlBridge { */ startRawCapture(options: { communicator: any }): boolean { if (!global.isAndroid) return false; - if (this.rawActive) return true; - if (this.captureHolders.size > 0 || this.started) return false; + if (this.rawActive && this.micIsLive()) return true; + // An STT holder owns the mic even while its stream waits on a new glasses + // session; the tap re-tries when the app's eligibility is re-evaluated. + if (this.captureHolders.size > 0 || this.suspendedHolders.size > 0) return false; + // A tap left over from a dead session is not something to share. + if (this.started || this.rawActive) this.teardownCapture(); + this.suspendedRaw = false; this.ensureController(); this.controller?.setCommunicator(options.communicator); this.controller?.setSaveRecordings(false); @@ -133,6 +145,7 @@ export class FaceclawVoiceControlBridge { } stopRawCapture(): void { + this.suspendedRaw = false; if (!this.rawActive) return; this.rawActive = false; this.started = false; @@ -144,17 +157,26 @@ export class FaceclawVoiceControlBridge { // STT preempts the raw-PCM tap: a live transcription session owns the mic, // and an EvenHub app gets no audio for its duration. if (this.rawActive) this.stopRawCapture(); - const wasIdle = this.captureHolders.size === 0; + // Whether the mic is already running is a question about the stream, not + // about this set: a holder whose capture died with the glasses session + // (or whose mic enable was refused) is still in it. Trusting the set here + // let one stale holder silently swallow every later request for the + // process's lifetime. + const micLive = this.micIsLive(); + this.suspendedHolders.delete(holder); this.captureHolders.add(holder); - if (!wasIdle) { + if (micLive) { // Mic already running; the new holder just shares the existing stream // (transcripts are already broadcast to its listeners). return; } + // Any leftover capture is dead; drop it before starting the new one. + if (this.started) this.teardownCapture(); this.ensureController(); this.controller?.setCommunicator(options.communicator); this.controller?.setSaveRecordings(options.saveRecording); this.controller?.setEndpointing(Boolean(options.endpointing)); + this.captureEndpointing = Boolean(options.endpointing); const cloudClient = this.createCloudClient(options); if (cloudClient) { @@ -209,6 +231,9 @@ export class FaceclawVoiceControlBridge { } private releaseCapture(holder: CaptureHolder, commit: boolean): void { + // A holder that gives up while its capture is parked must not be resumed + // by the next session. + this.suspendedHolders.delete(holder); if (!this.captureHolders.delete(holder)) { // Never held; still finish a dangling cloud commit if one is pending. if (commit) this.cloudClient?.finish(); @@ -236,6 +261,75 @@ export class FaceclawVoiceControlBridge { stop(): void { this.captureHolders.clear(); + this.suspendedHolders.clear(); + this.suspendedRaw = false; + this.teardownCapture(); + this.setStatus("Voice control stopped."); + } + + /** + * The glasses session ended under a live capture (transport drop, charging + * case, EvenHub suspend). Its mic enable did not survive, so drop the local + * capture state — leaving it in place would make the holder set claim a + * stream that no longer exists — and remember who was holding so + * resumeCapture() can bring the mic back on the next session. + */ + handleSessionEnded(): void { + if (this.captureHolders.size === 0 && !this.started && !this.rawActive) return; + for (const holder of this.captureHolders) { + this.suspendedHolders.add(holder); + } + this.captureHolders.clear(); + this.suspendedRaw = this.rawActive; + this.teardownCapture(); + if (this.suspendedHolders.size > 0) { + this.setStatus("Waiting for the glasses..."); + } + } + + /** Whether a capture is parked waiting for a session to come back. */ + hasSuspendedCapture(): boolean { + return this.suspendedHolders.size > 0 || this.suspendedRaw; + } + + /** Whether anyone holds the mic, including across a session outage. */ + isCaptureHeld(): boolean { + return this.captureHolders.size > 0 || this.suspendedHolders.size > 0; + } + + /** + * A fresh glasses session is ready (its display path warmed up, so a mic + * enable will be accepted): restart the mic for every holder that was + * capturing when the last session ended. Mic permission was granted for the + * original capture and cannot be revoked without restarting the process, so + * this does not ask again. + */ + resumeCapture(options: Omit): void { + const holders = Array.from(this.suspendedHolders); + const resumeRaw = this.suspendedRaw; + this.suspendedHolders.clear(); + this.suspendedRaw = false; + for (const holder of holders) { + this.acquireCapture(holder, { ...options, endpointing: this.captureEndpointing }); + } + if (resumeRaw && !holders.length) { + this.startRawCapture({ communicator: options.communicator }); + } + } + + /** + * Whether the glasses mic is actually streaming. The holder set and + * `started` are phone-side bookkeeping; the enable itself lives in the + * glasses session, so the controller (via the communicator) is the only + * authority on whether audio is still flowing. + */ + private micIsLive(): boolean { + if (!global.isAndroid || !this.started) return false; + return Boolean(this.controller?.isCapturing()); + } + + /** Stop the native capture and any cloud client; leaves holders alone. */ + private teardownCapture(): void { this.rawActive = false; if (global.isAndroid) { this.controller?.stop(); @@ -243,7 +337,6 @@ export class FaceclawVoiceControlBridge { this.started = false; this.cloudClient?.stop(); this.cloudClient = null; - this.setStatus("Voice control stopped."); } private ensureController(): void {