forked from Blendman974/faceclaw
Preview mode uses phone mic
This commit is contained in:
1 parent
01527adda9
commit
54005ae95a
5 files changed
+243
-44
No files matched your search
@@ -1,7 +1,12 @@
|
||||
import { Application, ImageSource } from "@nativescript/core";
|
||||
import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName, WatchGestureType, WatchGestureTypeName } from "./events";
|
||||
import { loadDeviceAddresses } from "./device-addresses";
|
||||
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
|
||||
import {
|
||||
ensureBlePermissions,
|
||||
ensureVoicePermissions,
|
||||
hasMicrophonePermission,
|
||||
requestMicrophonePermission,
|
||||
} from "./android-permissions";
|
||||
import { FaceclawCommunicatorBridge, type RawInputEvent } from "../native/faceclaw-communicator";
|
||||
import * as frameTimings from "../native/frame-timings";
|
||||
import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service";
|
||||
@@ -317,6 +322,7 @@ class DashboardController {
|
||||
},
|
||||
getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()),
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
||||
onWindowsChanged: () => {
|
||||
this.persistOpenApps();
|
||||
// The foreground title is mirrored on both remote-control faces.
|
||||
@@ -1732,10 +1738,15 @@ class DashboardController {
|
||||
voiceControlBridge.stopContinuousCapture();
|
||||
}
|
||||
|
||||
/** Provider and key settings for a capture on the given connection. */
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge) {
|
||||
/**
|
||||
* Provider and key settings for a capture on the given connection; a null
|
||||
* communicator means preview-only mode, where the phone's own microphone
|
||||
* stands in for the G2 mic.
|
||||
*/
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
return {
|
||||
communicator: communicator.getNativeCommunicator(),
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider: voiceProviderSetting.get(),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
@@ -1763,14 +1774,39 @@ class DashboardController {
|
||||
voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator));
|
||||
}
|
||||
|
||||
/**
|
||||
* Gate for opening the voice dialog (the shell asks before pushing the
|
||||
* layer). Connected sessions always proceed — the mic permission prompt,
|
||||
* if needed, appears over the open dialog as it always has. Preview mode
|
||||
* captures from the phone mic, so a missing permission turns the tap into
|
||||
* the system permission prompt instead of a dialog that would listen to
|
||||
* nothing; a grant lets the dialog open right away.
|
||||
*/
|
||||
private async prepareVoiceCapture(): Promise<boolean> {
|
||||
if (!this.isPreviewDisplayActive()) return true;
|
||||
if (hasMicrophonePermission()) return true;
|
||||
const granted = await requestMicrophonePermission();
|
||||
if (!granted) {
|
||||
this.appendLog("voice input blocked: microphone permission denied");
|
||||
}
|
||||
return granted;
|
||||
}
|
||||
|
||||
private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void {
|
||||
if (this.phase !== "connected" || !this.communicator) {
|
||||
// Preview mode captures from the phone mic (voiceCaptureOptions with a
|
||||
// null communicator); otherwise a live glasses session must be the source.
|
||||
const previewCapture = this.isPreviewDisplayActive();
|
||||
if (!previewCapture && (this.phase !== "connected" || !this.communicator)) {
|
||||
return;
|
||||
}
|
||||
const communicator = this.communicator;
|
||||
void ensureVoicePermissions()
|
||||
.then(() => {
|
||||
if (this.phase !== "connected" || this.communicator !== communicator) return;
|
||||
if (previewCapture) {
|
||||
if (!this.isPreviewDisplayActive()) return;
|
||||
} else if (this.phase !== "connected" || this.communicator !== communicator) {
|
||||
return;
|
||||
}
|
||||
const options = { ...this.voiceCaptureOptions(communicator), endpointing };
|
||||
if (kind === "ptt") {
|
||||
voiceControlBridge.startPushToTalk(options);
|
||||
|
||||
@@ -24,7 +24,14 @@ export type VoiceTranscriptEvent = {
|
||||
};
|
||||
|
||||
export type PushToTalkOptions = {
|
||||
/** Native G2 communicator, or null in preview-only mode (see usePhoneMic). */
|
||||
communicator: any;
|
||||
/**
|
||||
* Capture from the phone's own microphone instead of the G2 over BLE.
|
||||
* Preview-only mode, where no glasses are connected; everything downstream
|
||||
* (providers, endpointing, verification) works the same on either source.
|
||||
*/
|
||||
usePhoneMic?: boolean;
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
@@ -172,6 +179,9 @@ export class FaceclawVoiceControlBridge {
|
||||
this.suspendedRaw = false;
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
// The raw tap is strictly the G2 stream; clear any phone-mic flag a
|
||||
// preview-mode capture may have left on the shared controller.
|
||||
this.controller?.setUsePhoneMic(false);
|
||||
this.controller?.setSaveRecordings(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||
@@ -218,6 +228,7 @@ export class FaceclawVoiceControlBridge {
|
||||
if (this.started) this.teardownCapture();
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||
this.controller?.setSaveRecordings(options.saveRecording);
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
this.captureEndpointing = Boolean(options.endpointing);
|
||||
|
||||
@@ -129,6 +129,12 @@ export type ShellConfig = {
|
||||
actions: LayerActions;
|
||||
getScreenTimeoutMs: () => number | null;
|
||||
requestShellRender: () => void;
|
||||
/**
|
||||
* Asked before the voice dialog opens; resolving false swallows the open.
|
||||
* Preview-only mode uses it to turn the tap into a mic-permission prompt
|
||||
* when RECORD_AUDIO isn't granted yet (the phone mic is the source there).
|
||||
*/
|
||||
prepareVoiceCapture?: () => Promise<boolean>;
|
||||
/** Screen on/off changed: the controller blanks/unblanks the compositor. */
|
||||
onScreenStateChanged: (on: boolean) => void;
|
||||
/** Window registered/removed or foreground changed (persists the open-app list). */
|
||||
@@ -795,10 +801,38 @@ class Shell {
|
||||
this.config.onWindowsChanged?.();
|
||||
}
|
||||
|
||||
// True while a voice-dialog open waits on prepareVoiceCapture (e.g. the
|
||||
// preview-mode permission prompt); a second tap must not queue another open.
|
||||
private voiceDialogPending = false;
|
||||
|
||||
private openVoiceDialog(options: {
|
||||
finishOnClick?: boolean;
|
||||
handsFree?: boolean;
|
||||
defaultTarget: "assistant" | "app";
|
||||
}): void {
|
||||
if (this.voiceDialogPending) return;
|
||||
this.voiceDialogPending = true;
|
||||
void (async () => {
|
||||
let ready = true;
|
||||
try {
|
||||
ready = (await this.config.prepareVoiceCapture?.()) ?? true;
|
||||
} catch {
|
||||
ready = false;
|
||||
} finally {
|
||||
this.voiceDialogPending = false;
|
||||
}
|
||||
// Re-checked after the await: another path may have opened a dialog
|
||||
// (or torn down the base state) while a permission prompt was up.
|
||||
if (!ready || this.activeVoiceLayer) return;
|
||||
this.openVoiceDialogNow(options);
|
||||
this.config.requestShellRender();
|
||||
})();
|
||||
}
|
||||
|
||||
private openVoiceDialogNow(options: {
|
||||
finishOnClick?: boolean;
|
||||
handsFree?: boolean;
|
||||
defaultTarget: "assistant" | "app";
|
||||
}): void {
|
||||
const targets = this.buildVoiceSendTargets();
|
||||
let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);
|
||||
|
||||
Reference in new issue
Block a user