diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 1de06ca9..12680324 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -79,6 +79,12 @@ public class FaceclawVoiceController { // Set once the worker has the glasses mic enabled for this session. // Read and written under `lock`, so it flips with `started` atomically. private boolean audioStarted; + // Capture from the phone's own microphone instead of the G2 over BLE + // (preview-only mode, where no glasses are connected). Latched into + // activePhoneMic at start() (under `lock`) so a mid-session setter call + // can't switch pipelines underneath the worker. + private volatile boolean usePhoneMic; + private boolean activePhoneMic; private VoiceInputMode mode = VoiceInputMode.CLOUD; private OfflineRecognizer recognizer; private FaceclawLc3Decoder lc3Decoder; @@ -134,6 +140,11 @@ public class FaceclawVoiceController { this.communicator = communicator; } + /** Source the next capture from the phone microphone (no glasses paired). */ + public void setUsePhoneMic(boolean usePhoneMic) { + this.usePhoneMic = usePhoneMic; + } + /** When true, the decoded mic PCM for each session is saved as a WAV. */ public void setSaveRecordings(boolean saveRecordings) { this.saveRecordings = saveRecordings; @@ -216,11 +227,12 @@ public class FaceclawVoiceController { emitStatus("Voice control is already listening."); return; } - if (communicator == null || !communicator.isSessionReady()) { + if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) { emitStatus("Voice control needs an active G2 connection."); return; } mode = parseMode(requestedMode); + activePhoneMic = usePhoneMic; started = true; audioStarted = false; workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); @@ -237,17 +249,23 @@ public class FaceclawVoiceController { */ public boolean isCapturing() { boolean audioUp; + boolean phoneMic; synchronized (lock) { if (!started) { return false; } audioUp = audioStarted; + phoneMic = activePhoneMic; } if (!audioUp) { // The worker is still bringing the mic up; report it as running so // a concurrent request shares it instead of restarting it. return true; } + if (phoneMic) { + // AudioRecord has no session to lose the enable to; it runs until stop(). + return true; + } FaceclawBleCommunicator currentCommunicator = communicator; return currentCommunicator != null && currentCommunicator.isAudioCaptureActive(); } @@ -303,7 +321,6 @@ public class FaceclawVoiceController { resetTranscriptState(); lastTranscript = ""; } - lc3Decoder = new FaceclawLc3Decoder(); endpointDetector.reset(); if (suppressor != null) { suppressor.reset(); @@ -312,18 +329,42 @@ public class FaceclawVoiceController { boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null; verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null; verifyCount = 0; - if (!startG2Audio()) { - emitStatus("Could not start G2 microphone input."); - return; + if (activePhoneMic) { + android.media.AudioRecord record = openPhoneMic(); + if (record == null) { + emitStatus("Could not start the phone microphone."); + return; + } + synchronized (lock) { + audioStarted = true; + } + emitStatus(currentMode == VoiceInputMode.CLOUD + ? "Listening (cloud)..." + : "Listening..."); + try { + processPhoneAudio(record); + } finally { + try { + record.stop(); + } catch (Throwable ignored) { + // Already stopped or never recording; release below either way. + } + record.release(); + } + } else { + lc3Decoder = new FaceclawLc3Decoder(); + if (!startG2Audio()) { + emitStatus("Could not start G2 microphone input."); + return; + } + synchronized (lock) { + audioStarted = true; + } + emitStatus(currentMode == VoiceInputMode.CLOUD + ? "Listening (cloud)..." + : "Listening..."); + processG2Audio(); } - synchronized (lock) { - audioStarted = true; - } - - emitStatus(currentMode == VoiceInputMode.CLOUD - ? "Listening (cloud)..." - : "Listening..."); - processG2Audio(); // Verification result must precede the final transcript so the // TS bridge can suppress a non-wearer command before it is acted // on (the callbacks are posted in order to the main handler). @@ -511,35 +552,110 @@ public class FaceclawVoiceController { maybeEmitAudioStats(false); continue; } - if (suppressionEnabled) { - applySuppression(pcm, count); - } - if (recordingPcm != null) { - appendRecording(pcm, count); - } - if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { - int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount); - System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied); - verifyCount += copied; - } - if (endpointing && endpointDetector.accept(pcm, count)) { - emitSpeechEnd(); - } - // PCM and frame metadata flow in every mode so levels, recording, - // and the Microphones radar keep working alongside onboard ASR. - emitPcm(pcm, count); - emitFrameMeta(angleDegrees, ssr); - if (mode != VoiceInputMode.CLOUD) { - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = pcm[i] / 32768.0f; - } - processRecognizer(samples); - } + processPcmChunk(pcm, count, angleDegrees, ssr, true); maybeEmitAudioStats(false); } } + /** + * Per-chunk processing shared by the G2 and phone-mic paths, downstream of + * decode and the beam filter. hasFrameMeta is false for the phone mic, + * which has no firmware DSP metadata to report. + */ + private void processPcmChunk(short[] pcm, int count, int angleDegrees, int ssr, boolean hasFrameMeta) { + if (suppressionEnabled) { + applySuppression(pcm, count); + } + if (recordingPcm != null) { + appendRecording(pcm, count); + } + if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { + int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount); + System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied); + verifyCount += copied; + } + if (endpointing && endpointDetector.accept(pcm, count)) { + emitSpeechEnd(); + } + // PCM and frame metadata flow in every mode so levels, recording, + // and the Microphones radar keep working alongside onboard ASR. + emitPcm(pcm, count); + if (hasFrameMeta) { + emitFrameMeta(angleDegrees, ssr); + } + if (mode != VoiceInputMode.CLOUD) { + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + samples[i] = pcm[i] / 32768.0f; + } + processRecognizer(samples); + } + } + + // 50 ms chunks match the G2 packet cadence the rest of the pipeline + // (endpointing, transcript pacing) is tuned for. + private static final int PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20; + + /** + * Open the phone's own microphone at the pipeline's native format + * (16 kHz mono PCM16), or null when it cannot start — the permission is + * missing (SecurityException) or the device refuses the configuration. + */ + private android.media.AudioRecord openPhoneMic() { + android.media.AudioRecord record = null; + try { + int minBytes = android.media.AudioRecord.getMinBufferSize( + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT); + int bufferBytes = Math.max(minBytes, PHONE_MIC_CHUNK_SAMPLES * 2 * 4); + record = new android.media.AudioRecord( + android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION, + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT, + bufferBytes); + if (record.getState() != android.media.AudioRecord.STATE_INITIALIZED) { + record.release(); + return null; + } + record.startRecording(); + if (record.getRecordingState() != android.media.AudioRecord.RECORDSTATE_RECORDING) { + record.release(); + return null; + } + return record; + } catch (Throwable t) { + Log.w(TAG, "phone mic open failed", t); + if (record != null) { + record.release(); + } + return null; + } + } + + /** + * Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame + * metadata — AudioRecord already delivers the pipeline's PCM format. The + * blocking read returns every chunk (50 ms), which bounds how long a + * stop() waits for the loop to notice `started` dropped. + */ + private void processPhoneAudio(android.media.AudioRecord record) { + short[] pcm = new short[PHONE_MIC_CHUNK_SAMPLES]; + while (started && !Thread.currentThread().isInterrupted()) { + int read = record.read(pcm, 0, pcm.length); + if (read < 0) { + Log.w(TAG, "phone mic read failed: " + read); + return; + } + if (read == 0) { + continue; + } + decodedSamples += read; + processPcmChunk(pcm, read, 0, 0, false); + } + } + private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); long now = SystemClock.elapsedRealtime(); diff --git a/CHANGELOG b/CHANGELOG index 28e8893f..7ed4df0e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,8 @@ ## 0.6.2 - Future * Preview Only mode works +* Voice input uses phone mic in preview mode +* Ring input on phone screen is swipe-style rather than separate buttons ## 0.6.1 - 2026-08-30 * Settings import/export scripts work on release builds diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index f2b74f65..b2ec2257 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1,7 +1,12 @@ import { Application, ImageSource } from "@nativescript/core"; import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName, WatchGestureType, WatchGestureTypeName } from "./events"; import { loadDeviceAddresses } from "./device-addresses"; -import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions"; +import { + ensureBlePermissions, + ensureVoicePermissions, + hasMicrophonePermission, + requestMicrophonePermission, +} from "./android-permissions"; import { FaceclawCommunicatorBridge, type RawInputEvent } from "../native/faceclaw-communicator"; import * as frameTimings from "../native/frame-timings"; import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service"; @@ -317,6 +322,7 @@ class DashboardController { }, getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()), requestShellRender: () => this.requestShellRender(), + prepareVoiceCapture: () => this.prepareVoiceCapture(), onWindowsChanged: () => { this.persistOpenApps(); // The foreground title is mirrored on both remote-control faces. @@ -1732,10 +1738,15 @@ class DashboardController { voiceControlBridge.stopContinuousCapture(); } - /** Provider and key settings for a capture on the given connection. */ - private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge) { + /** + * Provider and key settings for a capture on the given connection; a null + * communicator means preview-only mode, where the phone's own microphone + * stands in for the G2 mic. + */ + private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { return { - communicator: communicator.getNativeCommunicator(), + communicator: communicator?.getNativeCommunicator() ?? null, + usePhoneMic: communicator === null, provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), @@ -1763,14 +1774,39 @@ class DashboardController { voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator)); } + /** + * Gate for opening the voice dialog (the shell asks before pushing the + * layer). Connected sessions always proceed — the mic permission prompt, + * if needed, appears over the open dialog as it always has. Preview mode + * captures from the phone mic, so a missing permission turns the tap into + * the system permission prompt instead of a dialog that would listen to + * nothing; a grant lets the dialog open right away. + */ + private async prepareVoiceCapture(): Promise { + if (!this.isPreviewDisplayActive()) return true; + if (hasMicrophonePermission()) return true; + const granted = await requestMicrophonePermission(); + if (!granted) { + this.appendLog("voice input blocked: microphone permission denied"); + } + return granted; + } + private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void { - if (this.phase !== "connected" || !this.communicator) { + // Preview mode captures from the phone mic (voiceCaptureOptions with a + // null communicator); otherwise a live glasses session must be the source. + const previewCapture = this.isPreviewDisplayActive(); + if (!previewCapture && (this.phase !== "connected" || !this.communicator)) { return; } const communicator = this.communicator; void ensureVoicePermissions() .then(() => { - if (this.phase !== "connected" || this.communicator !== communicator) return; + if (previewCapture) { + if (!this.isPreviewDisplayActive()) return; + } else if (this.phase !== "connected" || this.communicator !== communicator) { + return; + } const options = { ...this.voiceCaptureOptions(communicator), endpointing }; if (kind === "ptt") { voiceControlBridge.startPushToTalk(options); diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 205f4316..63bf8f9c 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -24,7 +24,14 @@ export type VoiceTranscriptEvent = { }; export type PushToTalkOptions = { + /** Native G2 communicator, or null in preview-only mode (see usePhoneMic). */ communicator: any; + /** + * Capture from the phone's own microphone instead of the G2 over BLE. + * Preview-only mode, where no glasses are connected; everything downstream + * (providers, endpointing, verification) works the same on either source. + */ + usePhoneMic?: boolean; provider: VoiceProviderKind; elevenLabsApiKey: string; openAiApiKey: string; @@ -172,6 +179,9 @@ export class FaceclawVoiceControlBridge { this.suspendedRaw = false; this.ensureController(); this.controller?.setCommunicator(options.communicator); + // The raw tap is strictly the G2 stream; clear any phone-mic flag a + // preview-mode capture may have left on the shared controller. + this.controller?.setUsePhoneMic(false); this.controller?.setSaveRecordings(false); this.controller?.setEndpointing(false); // Raw means raw: the Microphones session (and EvenHub apps) get the @@ -218,6 +228,7 @@ export class FaceclawVoiceControlBridge { if (this.started) this.teardownCapture(); this.ensureController(); this.controller?.setCommunicator(options.communicator); + this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic)); this.controller?.setSaveRecordings(options.saveRecording); this.controller?.setEndpointing(Boolean(options.endpointing)); this.captureEndpointing = Boolean(options.endpointing); diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 33dc052a..564fe45a 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -129,6 +129,12 @@ export type ShellConfig = { actions: LayerActions; getScreenTimeoutMs: () => number | null; requestShellRender: () => void; + /** + * Asked before the voice dialog opens; resolving false swallows the open. + * Preview-only mode uses it to turn the tap into a mic-permission prompt + * when RECORD_AUDIO isn't granted yet (the phone mic is the source there). + */ + prepareVoiceCapture?: () => Promise; /** Screen on/off changed: the controller blanks/unblanks the compositor. */ onScreenStateChanged: (on: boolean) => void; /** Window registered/removed or foreground changed (persists the open-app list). */ @@ -795,10 +801,38 @@ class Shell { this.config.onWindowsChanged?.(); } + // True while a voice-dialog open waits on prepareVoiceCapture (e.g. the + // preview-mode permission prompt); a second tap must not queue another open. + private voiceDialogPending = false; + private openVoiceDialog(options: { finishOnClick?: boolean; handsFree?: boolean; defaultTarget: "assistant" | "app"; + }): void { + if (this.voiceDialogPending) return; + this.voiceDialogPending = true; + void (async () => { + let ready = true; + try { + ready = (await this.config.prepareVoiceCapture?.()) ?? true; + } catch { + ready = false; + } finally { + this.voiceDialogPending = false; + } + // Re-checked after the await: another path may have opened a dialog + // (or torn down the base state) while a permission prompt was up. + if (!ready || this.activeVoiceLayer) return; + this.openVoiceDialogNow(options); + this.config.requestShellRender(); + })(); + } + + private openVoiceDialogNow(options: { + finishOnClick?: boolean; + handsFree?: boolean; + defaultTarget: "assistant" | "app"; }): void { const targets = this.buildVoiceSendTargets(); let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);