diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 8c056c52..4fa5952a 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -13,7 +13,9 @@ import { TranscriptModel, TranscriptScroll, wrapTranscribeText } from "./transcr * it, and saves it to Downloads from the app menu. */ export class TranscribeLayer implements Layer { - private status = "Listening..."; + private status = "Starting microphone..."; + private listening = false; + private detail = ""; private readonly transcript = new TranscriptModel(); private readonly scroll = new TranscriptScroll(); private saveNotice = ""; @@ -29,6 +31,8 @@ export class TranscribeLayer implements Layer { this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause()); this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { this.status = state.status; + this.listening = state.listening; + this.detail = state.detail; this.saveNotice = ""; requestRender(); }); @@ -38,7 +42,7 @@ export class TranscribeLayer implements Layer { const font = getDefaultSmallFont(); const { width, height } = ctx.stack.getBaseSize(); const image = new GrayImage(width, height, 0); - const text = this.transcript.text || "Listening..."; + const text = this.transcript.text || (this.listening ? "Listening..." : this.detail); const wrapped = wrapTranscribeText((text) => font.measureText(text), text, width - 64); image.drawText(font, 24, 20, "Transcribe", 200); diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index fbd9b57d..84fb9548 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1988,6 +1988,9 @@ class DashboardController { return; } const communicator = this.communicator; + // The system prompt appears on the phone; tell the glasses dialog so it + // sends the user there rather than claiming to listen. + if (!hasMicrophonePermission()) voiceControlBridge.reportPermissionPrompt(); await ensureVoicePermissions() .then(() => { if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return; @@ -2005,6 +2008,7 @@ class DashboardController { }) .catch((error) => { this.appendLog(`voice permission failed: ${this.formatError(error)}`); + if (!hasMicrophonePermission()) voiceControlBridge.reportPermissionDenied(); }); } diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index f0e5669a..2874d5a9 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -11,7 +11,19 @@ import { toUint8Array } from "../util/array-util"; declare const com: any; export type VoiceControlState = { + /** One-line state of the capture: progress, "Listening...", or an error. */ status: string; + /** + * Whether audio is actually being captured and transcribed right now. False + * while the mic is starting, while the phone is showing the permission + * prompt, and after an error, so a dialog must not claim to be listening. + */ + listening: boolean; + /** + * Longer guidance to show in place of the (still empty) transcript, e.g. + * what the user has to do on the phone. Empty when there is nothing to say. + */ + detail: string; }; export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox"; @@ -88,6 +100,8 @@ export class FaceclawVoiceControlBridge { private controller: any | null = null; private listenerProxy: any | null = null; private status = "Voice control stopped."; + private listening = false; + private detail = ""; private started = false; // The mic is a single shared stream; these are the reasons it is running. // The first holder starts capture (choosing the provider); the mic stops @@ -118,7 +132,7 @@ export class FaceclawVoiceControlBridge { onStatus(listener: (state: VoiceControlState) => void): () => void { this.statusListeners.add(listener); - listener({ status: this.status }); + listener(this.state()); return () => this.statusListeners.delete(listener); } @@ -142,6 +156,28 @@ export class FaceclawVoiceControlBridge { return () => this.speechPauseListeners.delete(listener); } + /** + * The capture is blocked on the phone's system microphone-permission + * prompt. Reported by the controller before it asks Android, so the dialog + * on the glasses can send the user to the phone instead of claiming to + * listen; the next status (from the capture start, or reportPermissionDenied) + * replaces it. + */ + reportPermissionPrompt(): void { + this.setStatus( + "Waiting for microphone permission...", + "Allow microphone access in the prompt on your phone.", + ); + } + + /** The user declined the microphone permission (or the prompt could not be shown). */ + reportPermissionDenied(): void { + this.setStatus( + "Microphone access denied.", + "Enable the microphone for Faceclaw in the phone's app settings, then try again.", + ); + } + /** Begin push-to-talk capture (momentary; released with stopPushToTalk). */ startPushToTalk(options: PushToTalkOptions): void { this.acquireCapture("ptt", options); @@ -262,6 +298,9 @@ export class FaceclawVoiceControlBridge { } this.speechPause.reset(); + // Replaces whatever the last capture left behind ("Listening...", an + // error, the permission prompt) until the controller reports progress. + this.setStatus("Starting microphone..."); // A previous push-to-talk commit may still be awaiting its final result. this.cloudClient?.stop(); const cloudClient = this.createCloudClient(options); @@ -492,12 +531,21 @@ export class FaceclawVoiceControlBridge { } } - private setStatus(status: string): void { + private setStatus(status: string, detail = ""): void { this.status = status; + this.detail = detail; + // Every "audio is flowing" report — from the Java controller and from the + // cloud providers — is phrased "Listening..." / "Listening (X)...". + this.listening = status.startsWith("Listening"); + const state = this.state(); for (const listener of this.statusListeners) { - listener({ status }); + listener(state); } } + + private state(): VoiceControlState { + return { status: this.status, listening: this.listening, detail: this.detail }; + } } function toJavaFloats(values: Float32Array): any { diff --git a/app/ui/shell/voice-input.ts b/app/ui/shell/voice-input.ts index ed5f88c8..41498570 100644 --- a/app/ui/shell/voice-input.ts +++ b/app/ui/shell/voice-input.ts @@ -64,7 +64,11 @@ export type VoiceInputLayerOptions = { */ export class VoiceInputLayer implements Layer { private phase: VoicePhase = "capturing"; - private status = "Listening..."; + private status = "Starting microphone..."; + /** Mirrors the bridge: only true while audio is actually being transcribed. */ + private listening = false; + /** Bridge guidance shown in place of the empty transcript (e.g. "check your phone"). */ + private detail = ""; // The active utterance. displayText() is what the dialog shows and what // Send delivers; the refine flow also writes the merged result here. private finalizedText = ""; @@ -114,6 +118,8 @@ export class VoiceInputLayer implements Layer { // The refine stage owns the status line ("Refining...", error text). if (this.phase === "refining") return; this.status = state.status; + this.listening = state.listening; + this.detail = state.detail; this.actions.requestRender(); }); if (this.handsFree) { @@ -156,7 +162,9 @@ export class VoiceInputLayer implements Layer { this.pendingAutoSend = true; this.status = "Sending..."; this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS); - } else if (this.status.startsWith("Listening")) { + } else if (this.status.endsWith("...")) { + // A progress status ("Starting microphone...", "Listening...") gives + // way to the menu prompt; an error (ending in ".") stays visible. this.status = "Send, continue, or discard?"; } this.actions.requestRender(); @@ -213,7 +221,7 @@ export class VoiceInputLayer implements Layer { const image = paintBelow(); const inMenu = this.phase === "menu"; paintInputDialog(image, { - title: this.capturing ? "Voice ●" : "Voice", + title: this.capturing && this.listening ? "Voice ●" : "Voice", status: this.status, text: this.displayText() || this.placeholderText(), rows: inMenu ? this.menuRows() : [], @@ -391,7 +399,9 @@ export class VoiceInputLayer implements Layer { private placeholderText(): string { switch (this.phase) { case "capturing": - return "Listening..."; + // Only claim to listen when the mic is actually running; otherwise + // the bridge's guidance (permission prompt on the phone, etc.), if any. + return this.listening ? "Listening..." : this.detail; case "continuing": return "Say more, or describe an edit..."; case "refining":