forked from Blendman974/faceclaw
Voice input tells you if it can't record due to microphone permissions
This commit is contained in:
1 parent
476bb281a1
commit
dfde776016
4 files changed
+75
-9
No files matched your search
@@ -13,7 +13,9 @@ import { TranscriptModel, TranscriptScroll, wrapTranscribeText } from "./transcr
|
|||||||
* it, and saves it to Downloads from the app menu.
|
* it, and saves it to Downloads from the app menu.
|
||||||
*/
|
*/
|
||||||
export class TranscribeLayer implements Layer {
|
export class TranscribeLayer implements Layer {
|
||||||
private status = "Listening...";
|
private status = "Starting microphone...";
|
||||||
|
private listening = false;
|
||||||
|
private detail = "";
|
||||||
private readonly transcript = new TranscriptModel();
|
private readonly transcript = new TranscriptModel();
|
||||||
private readonly scroll = new TranscriptScroll();
|
private readonly scroll = new TranscriptScroll();
|
||||||
private saveNotice = "";
|
private saveNotice = "";
|
||||||
@@ -29,6 +31,8 @@ export class TranscribeLayer implements Layer {
|
|||||||
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
||||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||||
this.status = state.status;
|
this.status = state.status;
|
||||||
|
this.listening = state.listening;
|
||||||
|
this.detail = state.detail;
|
||||||
this.saveNotice = "";
|
this.saveNotice = "";
|
||||||
requestRender();
|
requestRender();
|
||||||
});
|
});
|
||||||
@@ -38,7 +42,7 @@ export class TranscribeLayer implements Layer {
|
|||||||
const font = getDefaultSmallFont();
|
const font = getDefaultSmallFont();
|
||||||
const { width, height } = ctx.stack.getBaseSize();
|
const { width, height } = ctx.stack.getBaseSize();
|
||||||
const image = new GrayImage(width, height, 0);
|
const image = new GrayImage(width, height, 0);
|
||||||
const text = this.transcript.text || "Listening...";
|
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||||
const wrapped = wrapTranscribeText((text) => font.measureText(text), text, width - 64);
|
const wrapped = wrapTranscribeText((text) => font.measureText(text), text, width - 64);
|
||||||
|
|
||||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||||
|
|||||||
@@ -1988,6 +1988,9 @@ class DashboardController {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const communicator = this.communicator;
|
const communicator = this.communicator;
|
||||||
|
// The system prompt appears on the phone; tell the glasses dialog so it
|
||||||
|
// sends the user there rather than claiming to listen.
|
||||||
|
if (!hasMicrophonePermission()) voiceControlBridge.reportPermissionPrompt();
|
||||||
await ensureVoicePermissions()
|
await ensureVoicePermissions()
|
||||||
.then(() => {
|
.then(() => {
|
||||||
if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return;
|
if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return;
|
||||||
@@ -2005,6 +2008,7 @@ class DashboardController {
|
|||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
this.appendLog(`voice permission failed: ${this.formatError(error)}`);
|
this.appendLog(`voice permission failed: ${this.formatError(error)}`);
|
||||||
|
if (!hasMicrophonePermission()) voiceControlBridge.reportPermissionDenied();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,19 @@ import { toUint8Array } from "../util/array-util";
|
|||||||
declare const com: any;
|
declare const com: any;
|
||||||
|
|
||||||
export type VoiceControlState = {
|
export type VoiceControlState = {
|
||||||
|
/** One-line state of the capture: progress, "Listening...", or an error. */
|
||||||
status: string;
|
status: string;
|
||||||
|
/**
|
||||||
|
* Whether audio is actually being captured and transcribed right now. False
|
||||||
|
* while the mic is starting, while the phone is showing the permission
|
||||||
|
* prompt, and after an error, so a dialog must not claim to be listening.
|
||||||
|
*/
|
||||||
|
listening: boolean;
|
||||||
|
/**
|
||||||
|
* Longer guidance to show in place of the (still empty) transcript, e.g.
|
||||||
|
* what the user has to do on the phone. Empty when there is nothing to say.
|
||||||
|
*/
|
||||||
|
detail: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||||
@@ -88,6 +100,8 @@ export class FaceclawVoiceControlBridge {
|
|||||||
private controller: any | null = null;
|
private controller: any | null = null;
|
||||||
private listenerProxy: any | null = null;
|
private listenerProxy: any | null = null;
|
||||||
private status = "Voice control stopped.";
|
private status = "Voice control stopped.";
|
||||||
|
private listening = false;
|
||||||
|
private detail = "";
|
||||||
private started = false;
|
private started = false;
|
||||||
// The mic is a single shared stream; these are the reasons it is running.
|
// The mic is a single shared stream; these are the reasons it is running.
|
||||||
// The first holder starts capture (choosing the provider); the mic stops
|
// The first holder starts capture (choosing the provider); the mic stops
|
||||||
@@ -118,7 +132,7 @@ export class FaceclawVoiceControlBridge {
|
|||||||
|
|
||||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||||
this.statusListeners.add(listener);
|
this.statusListeners.add(listener);
|
||||||
listener({ status: this.status });
|
listener(this.state());
|
||||||
return () => this.statusListeners.delete(listener);
|
return () => this.statusListeners.delete(listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,6 +156,28 @@ export class FaceclawVoiceControlBridge {
|
|||||||
return () => this.speechPauseListeners.delete(listener);
|
return () => this.speechPauseListeners.delete(listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The capture is blocked on the phone's system microphone-permission
|
||||||
|
* prompt. Reported by the controller before it asks Android, so the dialog
|
||||||
|
* on the glasses can send the user to the phone instead of claiming to
|
||||||
|
* listen; the next status (from the capture start, or reportPermissionDenied)
|
||||||
|
* replaces it.
|
||||||
|
*/
|
||||||
|
reportPermissionPrompt(): void {
|
||||||
|
this.setStatus(
|
||||||
|
"Waiting for microphone permission...",
|
||||||
|
"Allow microphone access in the prompt on your phone.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The user declined the microphone permission (or the prompt could not be shown). */
|
||||||
|
reportPermissionDenied(): void {
|
||||||
|
this.setStatus(
|
||||||
|
"Microphone access denied.",
|
||||||
|
"Enable the microphone for Faceclaw in the phone's app settings, then try again.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
||||||
startPushToTalk(options: PushToTalkOptions): void {
|
startPushToTalk(options: PushToTalkOptions): void {
|
||||||
this.acquireCapture("ptt", options);
|
this.acquireCapture("ptt", options);
|
||||||
@@ -262,6 +298,9 @@ export class FaceclawVoiceControlBridge {
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.speechPause.reset();
|
this.speechPause.reset();
|
||||||
|
// Replaces whatever the last capture left behind ("Listening...", an
|
||||||
|
// error, the permission prompt) until the controller reports progress.
|
||||||
|
this.setStatus("Starting microphone...");
|
||||||
// A previous push-to-talk commit may still be awaiting its final result.
|
// A previous push-to-talk commit may still be awaiting its final result.
|
||||||
this.cloudClient?.stop();
|
this.cloudClient?.stop();
|
||||||
const cloudClient = this.createCloudClient(options);
|
const cloudClient = this.createCloudClient(options);
|
||||||
@@ -492,12 +531,21 @@ export class FaceclawVoiceControlBridge {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private setStatus(status: string): void {
|
private setStatus(status: string, detail = ""): void {
|
||||||
this.status = status;
|
this.status = status;
|
||||||
|
this.detail = detail;
|
||||||
|
// Every "audio is flowing" report — from the Java controller and from the
|
||||||
|
// cloud providers — is phrased "Listening..." / "Listening (X)...".
|
||||||
|
this.listening = status.startsWith("Listening");
|
||||||
|
const state = this.state();
|
||||||
for (const listener of this.statusListeners) {
|
for (const listener of this.statusListeners) {
|
||||||
listener({ status });
|
listener(state);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private state(): VoiceControlState {
|
||||||
|
return { status: this.status, listening: this.listening, detail: this.detail };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function toJavaFloats(values: Float32Array): any {
|
function toJavaFloats(values: Float32Array): any {
|
||||||
|
|||||||
@@ -64,7 +64,11 @@ export type VoiceInputLayerOptions = {
|
|||||||
*/
|
*/
|
||||||
export class VoiceInputLayer implements Layer {
|
export class VoiceInputLayer implements Layer {
|
||||||
private phase: VoicePhase = "capturing";
|
private phase: VoicePhase = "capturing";
|
||||||
private status = "Listening...";
|
private status = "Starting microphone...";
|
||||||
|
/** Mirrors the bridge: only true while audio is actually being transcribed. */
|
||||||
|
private listening = false;
|
||||||
|
/** Bridge guidance shown in place of the empty transcript (e.g. "check your phone"). */
|
||||||
|
private detail = "";
|
||||||
// The active utterance. displayText() is what the dialog shows and what
|
// The active utterance. displayText() is what the dialog shows and what
|
||||||
// Send delivers; the refine flow also writes the merged result here.
|
// Send delivers; the refine flow also writes the merged result here.
|
||||||
private finalizedText = "";
|
private finalizedText = "";
|
||||||
@@ -114,6 +118,8 @@ export class VoiceInputLayer implements Layer {
|
|||||||
// The refine stage owns the status line ("Refining...", error text).
|
// The refine stage owns the status line ("Refining...", error text).
|
||||||
if (this.phase === "refining") return;
|
if (this.phase === "refining") return;
|
||||||
this.status = state.status;
|
this.status = state.status;
|
||||||
|
this.listening = state.listening;
|
||||||
|
this.detail = state.detail;
|
||||||
this.actions.requestRender();
|
this.actions.requestRender();
|
||||||
});
|
});
|
||||||
if (this.handsFree) {
|
if (this.handsFree) {
|
||||||
@@ -156,7 +162,9 @@ export class VoiceInputLayer implements Layer {
|
|||||||
this.pendingAutoSend = true;
|
this.pendingAutoSend = true;
|
||||||
this.status = "Sending...";
|
this.status = "Sending...";
|
||||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||||
} else if (this.status.startsWith("Listening")) {
|
} else if (this.status.endsWith("...")) {
|
||||||
|
// A progress status ("Starting microphone...", "Listening...") gives
|
||||||
|
// way to the menu prompt; an error (ending in ".") stays visible.
|
||||||
this.status = "Send, continue, or discard?";
|
this.status = "Send, continue, or discard?";
|
||||||
}
|
}
|
||||||
this.actions.requestRender();
|
this.actions.requestRender();
|
||||||
@@ -213,7 +221,7 @@ export class VoiceInputLayer implements Layer {
|
|||||||
const image = paintBelow();
|
const image = paintBelow();
|
||||||
const inMenu = this.phase === "menu";
|
const inMenu = this.phase === "menu";
|
||||||
paintInputDialog(image, {
|
paintInputDialog(image, {
|
||||||
title: this.capturing ? "Voice ●" : "Voice",
|
title: this.capturing && this.listening ? "Voice ●" : "Voice",
|
||||||
status: this.status,
|
status: this.status,
|
||||||
text: this.displayText() || this.placeholderText(),
|
text: this.displayText() || this.placeholderText(),
|
||||||
rows: inMenu ? this.menuRows() : [],
|
rows: inMenu ? this.menuRows() : [],
|
||||||
@@ -391,7 +399,9 @@ export class VoiceInputLayer implements Layer {
|
|||||||
private placeholderText(): string {
|
private placeholderText(): string {
|
||||||
switch (this.phase) {
|
switch (this.phase) {
|
||||||
case "capturing":
|
case "capturing":
|
||||||
return "Listening...";
|
// Only claim to listen when the mic is actually running; otherwise
|
||||||
|
// the bridge's guidance (permission prompt on the phone, etc.), if any.
|
||||||
|
return this.listening ? "Listening..." : this.detail;
|
||||||
case "continuing":
|
case "continuing":
|
||||||
return "Say more, or describe an edit...";
|
return "Say more, or describe an edit...";
|
||||||
case "refining":
|
case "refining":
|
||||||
|
|||||||
Reference in new issue
Block a user