forked from Blendman974/faceclaw
Merge branch 'main' into ble-bandwidth-test
This commit is contained in:
commit
0114c521ad
316 files changed
+32980
-882
No files matched your search
@@ -0,0 +1,19 @@
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* The Java object a worker isolate submits surface frames to: the live BLE
|
||||
* communicator, or the preview-only compositor standing in when no glasses
|
||||
* are paired (both expose the same submitSurfaceFrame signature). Located
|
||||
* through Java statics because those are shared across isolates, unlike any
|
||||
* JS-side state.
|
||||
*
|
||||
* Frame submission only. Glasses-hardware calls (e.g. playBuzzerSequence)
|
||||
* should keep using FaceclawBleCommunicator.getActive() directly, so they
|
||||
* skip cleanly when nothing is connected.
|
||||
*/
|
||||
export function getActiveDisplay(): any {
|
||||
return (
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive() ??
|
||||
com.faceclaw.app.FaceclawPreviewCompositor.getActive()
|
||||
);
|
||||
}
|
||||
@@ -90,6 +90,18 @@ export type RawInputEvent =
|
||||
systemExitReasonCode: number;
|
||||
frameId: number;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* Synthetic directional gesture from the Wear OS remote; `eventType` is
|
||||
* a WatchGestureType (app/g2/events.ts). Never produced by Java.
|
||||
*/
|
||||
kind: "watch-gesture";
|
||||
containerName: string;
|
||||
eventType: number;
|
||||
eventSource: number;
|
||||
systemExitReasonCode: number;
|
||||
frameId: number;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* Stock display-lifecycle wake observed after a ring or arm double tap
|
||||
@@ -444,10 +456,14 @@ export class FaceclawCommunicatorBridge {
|
||||
});
|
||||
}
|
||||
|
||||
/** Phone-UI preview of the current composited screen, or null if none yet. */
|
||||
getCompositePreview(): ImageSource | null {
|
||||
/**
|
||||
* Phone-UI preview of the current composited screen, or null if none yet.
|
||||
* `green` tints it green-on-black (the Preview color setting) instead of
|
||||
* grayscale.
|
||||
*/
|
||||
getCompositePreview(green = false): ImageSource | null {
|
||||
if (!global.isAndroid) return null;
|
||||
const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA);
|
||||
const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
|
||||
return bitmap ? new ImageSource(bitmap) : null;
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,9 @@ export type FlashPromptState =
|
||||
* stock-firmware-compatible BLE path that shows the pre-flash confirmation on
|
||||
* the glasses and reports the user's Yes/No choice. Separate from
|
||||
* FaceclawCommunicatorBridge on purpose (different, minimal protocol subset).
|
||||
*
|
||||
* Talks to the right arm only: the lenses relay messages between themselves,
|
||||
* and acks/events always come from the right arm.
|
||||
*/
|
||||
export class FlashPromptCommunicator {
|
||||
private readonly communicator: any;
|
||||
@@ -25,14 +28,13 @@ export class FlashPromptCommunicator {
|
||||
private readonly stateListeners = new Set<(state: FlashPromptState, detail: string) => void>();
|
||||
private readonly resultListeners = new Set<(approved: boolean) => void>();
|
||||
|
||||
constructor(addresses: { right: string; left: string }, warningText: string) {
|
||||
constructor(rightAddress: string, warningText: string) {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
if (!context) throw new Error("Android application context unavailable");
|
||||
|
||||
this.communicator = new com.faceclaw.app.FaceclawFlashPromptCommunicator(
|
||||
context,
|
||||
addresses.right,
|
||||
addresses.left,
|
||||
rightAddress,
|
||||
warningText,
|
||||
);
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawFlashPromptListener({
|
||||
|
||||
@@ -25,6 +25,13 @@ export type G2MirrorSession = {
|
||||
cwdHint: string;
|
||||
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
|
||||
lastBellAt: number | null;
|
||||
/**
|
||||
* Unix epoch ms (host clock) of the app's last output, or null if none
|
||||
* observed. Fresh to within the wrapper's 2s activity reporting interval
|
||||
* while we're connected; compare against other host timestamps, not the
|
||||
* phone clock (see the worker's local-receive-time activity tracking).
|
||||
*/
|
||||
lastOutputAt: number | null;
|
||||
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
|
||||
title: string | null;
|
||||
};
|
||||
@@ -95,6 +102,7 @@ export class G2MirrorClient {
|
||||
private readonly sessionAttachedListeners = new Set<(command: string) => void>();
|
||||
private readonly sessionDetachedListeners = new Set<(reason: string) => void>();
|
||||
private readonly bellListeners = new Set<(socket: string, lastBellAtMs: number) => void>();
|
||||
private readonly activityListeners = new Set<(socket: string, lastOutputAtMs: number) => void>();
|
||||
private readonly titleListeners = new Set<(socket: string, title: string) => void>();
|
||||
// Launch requests awaiting their reply, oldest first. The server answers
|
||||
// each `launch` with either `launched` or an `error` whose message starts
|
||||
@@ -144,6 +152,16 @@ export class G2MirrorClient {
|
||||
return () => this.bellListeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unsolicited output-activity notification for any monitored terminal
|
||||
* (rate-limited by the wrapper to one per 2s per terminal, leading edge
|
||||
* only — notifications stop as soon as the output does).
|
||||
*/
|
||||
onActivity(listener: (socket: string, lastOutputAtMs: number) => void): () => void {
|
||||
this.activityListeners.add(listener);
|
||||
return () => this.activityListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Unsolicited title change for any monitored terminal. */
|
||||
onTitle(listener: (socket: string, title: string) => void): () => void {
|
||||
this.titleListeners.add(listener);
|
||||
@@ -350,6 +368,7 @@ export class G2MirrorClient {
|
||||
pid: Number(item?.pid) || 0,
|
||||
cwdHint: String(item?.cwd_hint ?? ""),
|
||||
lastBellAt: typeof item?.last_bell_at === "number" ? item.last_bell_at : null,
|
||||
lastOutputAt: typeof item?.last_output_at === "number" ? item.last_output_at : null,
|
||||
title: typeof item?.title === "string" ? item.title : null,
|
||||
}))
|
||||
.filter((session: G2MirrorSession) => session.socket.length > 0);
|
||||
@@ -367,6 +386,19 @@ export class G2MirrorClient {
|
||||
}
|
||||
return;
|
||||
}
|
||||
case "activity": {
|
||||
const socket = String(message.socket ?? "");
|
||||
const lastOutputAt = Number(message.last_output_at) || 0;
|
||||
if (!socket || !lastOutputAt) return;
|
||||
const session = this.sessions.find((s) => s.socket === socket);
|
||||
if (session) session.lastOutputAt = lastOutputAt;
|
||||
// No emitState: activity is frequent while an app is busy, and the
|
||||
// terminal worker drives its own hub animation off the listener.
|
||||
for (const listener of Array.from(this.activityListeners)) {
|
||||
listener(socket, lastOutputAt);
|
||||
}
|
||||
return;
|
||||
}
|
||||
case "title": {
|
||||
const socket = String(message.socket ?? "");
|
||||
const title = String(message.title ?? "");
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import { ImageSource, Utils } from "@nativescript/core";
|
||||
import type { FaceclawCommunicatorBridge, SurfaceOptions } from "./faceclaw-communicator";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
|
||||
|
||||
/**
|
||||
* The display subset of the communicator bridge: everything the dashboard
|
||||
* controller needs to composite frames and read them back for the phone
|
||||
* mirror, screenshots, and recordings. FaceclawCommunicatorBridge satisfies
|
||||
* it by construction (Pick), and PreviewDisplayTarget implements it against
|
||||
* a headless compositor with no BLE transport behind it.
|
||||
*/
|
||||
export type DisplayTarget = Pick<
|
||||
FaceclawCommunicatorBridge,
|
||||
| "configureCompositorScreen"
|
||||
| "configureSurface"
|
||||
| "removeSurface"
|
||||
| "setSurfaceVisible"
|
||||
| "setScreenBlanked"
|
||||
| "submitSurfaceFrame"
|
||||
| "waitForFrameFinished"
|
||||
| "getCompositePreview"
|
||||
| "saveScreenshot"
|
||||
| "startScreenRecording"
|
||||
| "recordScreenFrame"
|
||||
| "stopScreenRecording"
|
||||
>;
|
||||
|
||||
function nonNegativeNumber(value: number): number {
|
||||
const numeric = Number(value);
|
||||
return Number.isFinite(numeric) ? Math.max(0, numeric) : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Headless display target for preview-only mode (onboarding's "Preview Only"
|
||||
* path): hosts the same Java SurfaceCompositor a live connection uses, so
|
||||
* the whole shell/app render pipeline works with no glasses paired. Frames
|
||||
* end at the compositor; the phone mirror polls them back as usual.
|
||||
*
|
||||
* Unlike the communicator bridge, calls go straight into Java: they are all
|
||||
* quick in-memory operations, so there is no call queue and no transmit
|
||||
* backpressure to wait on.
|
||||
*/
|
||||
export class PreviewDisplayTarget implements DisplayTarget {
|
||||
private readonly compositor: any;
|
||||
|
||||
constructor() {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
if (!context) throw new Error("Android application context unavailable");
|
||||
this.compositor = new com.faceclaw.app.FaceclawPreviewCompositor(context);
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish the compositor as the worker isolates' frame target (they locate
|
||||
* it through the Java static, like the BLE communicator) and receive a
|
||||
* main-thread callback per applied frame — the preview path's stand-in for
|
||||
* the connected path's frame-metrics callback.
|
||||
*/
|
||||
activate(onFrameComposited: () => void): void {
|
||||
this.compositor.setFrameListener(
|
||||
new java.lang.Runnable({ run: () => onFrameComposited() }),
|
||||
);
|
||||
this.compositor.makeActive();
|
||||
}
|
||||
|
||||
/** Withdraw from workers and drop the frame callback. */
|
||||
release(): void {
|
||||
this.compositor.release();
|
||||
}
|
||||
|
||||
async configureCompositorScreen(width: number, height: number): Promise<void> {
|
||||
this.compositor.configureCompositorScreen(Math.round(width), Math.round(height));
|
||||
}
|
||||
|
||||
async configureSurface(id: string, options: SurfaceOptions): Promise<void> {
|
||||
const transparency = options.transparency === "color-key" ? 1 : 0;
|
||||
this.compositor.configureSurface(
|
||||
id,
|
||||
Math.round(options.x),
|
||||
Math.round(options.y),
|
||||
Math.round(options.width),
|
||||
Math.round(options.height),
|
||||
Math.round(options.zOrder),
|
||||
transparency,
|
||||
);
|
||||
}
|
||||
|
||||
async removeSurface(id: string): Promise<void> {
|
||||
this.compositor.removeSurface(id);
|
||||
}
|
||||
|
||||
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
|
||||
this.compositor.setSurfaceVisible(id, Boolean(visible));
|
||||
}
|
||||
|
||||
async setScreenBlanked(blanked: boolean): Promise<void> {
|
||||
this.compositor.setScreenBlanked(Boolean(blanked));
|
||||
}
|
||||
|
||||
async submitSurfaceFrame(
|
||||
surfaceId: string,
|
||||
pixels8bpp: Uint8Array,
|
||||
rect: { x: number; y: number; width: number; height: number },
|
||||
fingerprint: string,
|
||||
paintMs = -1,
|
||||
frameId = 0,
|
||||
glyphs: ArrayBuffer | null = null,
|
||||
): Promise<void> {
|
||||
// Copy for the same reason the bridge does: the source may be a view on a
|
||||
// larger buffer, and Java receives the backing ArrayBuffer.
|
||||
const snapshot = new Uint8Array(pixels8bpp);
|
||||
this.compositor.submitSurfaceFrame(
|
||||
snapshot.buffer,
|
||||
surfaceId,
|
||||
Math.round(rect.x),
|
||||
Math.round(rect.y),
|
||||
Math.round(rect.width),
|
||||
Math.round(rect.height),
|
||||
fingerprint,
|
||||
Math.round(nonNegativeNumber(paintMs)),
|
||||
Math.round(nonNegativeNumber(frameId)),
|
||||
glyphs,
|
||||
);
|
||||
// The Java side finishes the frame: the composite is the end of the line,
|
||||
// and worker isolates submit to the same object without coming through here.
|
||||
}
|
||||
|
||||
/** Composites are synchronous and nothing transmits, so there is never backpressure. */
|
||||
waitForFrameFinished(): Promise<string | null> {
|
||||
return Promise.resolve("composited");
|
||||
}
|
||||
|
||||
getCompositePreview(green = false): ImageSource | null {
|
||||
if (!global.isAndroid) return null;
|
||||
const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
|
||||
return bitmap ? new ImageSource(bitmap) : null;
|
||||
}
|
||||
|
||||
saveScreenshot(crop?: { x: number; y: number; width: number; height: number }): string {
|
||||
if (!global.isAndroid) return "";
|
||||
if (!crop) return String(this.compositor.saveCompositePngScreenshot());
|
||||
return String(
|
||||
this.compositor.saveCompositePngScreenshot(
|
||||
Math.round(crop.x),
|
||||
Math.round(crop.y),
|
||||
Math.round(crop.width),
|
||||
Math.round(crop.height),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
startScreenRecording(): void {
|
||||
if (!global.isAndroid) return;
|
||||
this.compositor.startScreenRecording();
|
||||
}
|
||||
|
||||
recordScreenFrame(): void {
|
||||
if (!global.isAndroid) return;
|
||||
this.compositor.recordScreenFrame();
|
||||
}
|
||||
|
||||
stopScreenRecording(): string {
|
||||
if (!global.isAndroid) return "";
|
||||
return String(this.compositor.stopScreenRecording());
|
||||
}
|
||||
}
|
||||
+200
-5
@@ -24,7 +24,14 @@ export type VoiceTranscriptEvent = {
|
||||
};
|
||||
|
||||
export type PushToTalkOptions = {
|
||||
/** Native G2 communicator, or null in preview-only mode (see usePhoneMic). */
|
||||
communicator: any;
|
||||
/**
|
||||
* Capture from the phone's own microphone instead of the G2 over BLE.
|
||||
* Preview-only mode, where no glasses are connected; everything downstream
|
||||
* (providers, endpointing, verification) works the same on either source.
|
||||
*/
|
||||
usePhoneMic?: boolean;
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
@@ -36,6 +43,23 @@ export type PushToTalkOptions = {
|
||||
* Ignored when the mic is already running for another holder.
|
||||
*/
|
||||
endpointing?: boolean;
|
||||
/**
|
||||
* "My voice only": verify the captured utterance against the enrolled
|
||||
* wearer voice-print, and suppress the final transcript when someone else
|
||||
* spoke. wearerEmbedding is the L2-normalized profile centroid.
|
||||
*/
|
||||
speakerVerification?: {
|
||||
speakerModelPath: string;
|
||||
wearerEmbedding: Float32Array;
|
||||
threshold: number;
|
||||
};
|
||||
/** Spectral noise suppression on the decoded stream (Microphones config). */
|
||||
noiseSuppression?: boolean;
|
||||
/**
|
||||
* Sonic Radar beam gating: only listen to the configured wedge (device
|
||||
* frame). Null/absent leaves the mic omnidirectional.
|
||||
*/
|
||||
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
|
||||
};
|
||||
|
||||
/** Who currently wants the mic running. */
|
||||
@@ -43,6 +67,16 @@ type CaptureHolder = "ptt" | "continuous";
|
||||
|
||||
export type RawPcmListener = (pcm: Uint8Array) => void;
|
||||
|
||||
/**
|
||||
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
|
||||
* packet: direction-of-arrival in signed degrees (0 = straight ahead,
|
||||
* positive to the wearer's right) and a signal-strength ratio used as a
|
||||
* confidence/energy proxy. Computed on the raw stereo capture before the
|
||||
* firmware's mono downmix, so it survives even though only mono PCM arrives.
|
||||
*/
|
||||
export type MicFrameMeta = { angleDegrees: number; ssr: number };
|
||||
export type FrameMetaListener = (meta: MicFrameMeta) => void;
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||
@@ -56,6 +90,13 @@ export class FaceclawVoiceControlBridge {
|
||||
// when the last one releases. Transcript events broadcast to every
|
||||
// listener, so push-to-talk and the Transcribe window both receive text.
|
||||
private readonly captureHolders = new Set<CaptureHolder>();
|
||||
// Holders whose stream died with the glasses session (transport drop,
|
||||
// charging case, EvenHub suspend). They still want the mic, so the next
|
||||
// session restarts it for them; see handleSessionEnded/resumeCapture.
|
||||
private readonly suspendedHolders = new Set<CaptureHolder>();
|
||||
private suspendedRaw = false;
|
||||
// Endpointing of the live capture, replayed when it is resumed.
|
||||
private captureEndpointing = false;
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: CloudSttClient | null = null;
|
||||
// Raw-PCM tap (EvenHub mic apps): when active, the controller runs in the
|
||||
@@ -63,7 +104,13 @@ export class FaceclawVoiceControlBridge {
|
||||
// listeners. STT capture preempts it — a live assistant/transcribe session
|
||||
// owns the single G2 mic and the raw tap is stopped for its duration.
|
||||
private readonly rawPcmListeners = new Set<RawPcmListener>();
|
||||
private readonly frameMetaListeners = new Set<FrameMetaListener>();
|
||||
private rawActive = false;
|
||||
// Set when the current capture session runs "my voice only" verification;
|
||||
// a non-wearer verdict arrives just before the final transcript and makes
|
||||
// the bridge swallow it.
|
||||
private verificationActive = false;
|
||||
private verificationRejected = false;
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
@@ -110,6 +157,12 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.rawPcmListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
|
||||
onFrameMeta(listener: FrameMetaListener): () => void {
|
||||
this.frameMetaListeners.add(listener);
|
||||
return () => this.frameMetaListeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the decode-only raw-PCM tap for an EvenHub mic app. Fails (returns
|
||||
* false) if an STT capture already owns the mic; the caller re-tries when its
|
||||
@@ -117,12 +170,25 @@ export class FaceclawVoiceControlBridge {
|
||||
*/
|
||||
startRawCapture(options: { communicator: any }): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
if (this.rawActive) return true;
|
||||
if (this.captureHolders.size > 0 || this.started) return false;
|
||||
if (this.rawActive && this.micIsLive()) return true;
|
||||
// An STT holder owns the mic even while its stream waits on a new glasses
|
||||
// session; the tap re-tries when the app's eligibility is re-evaluated.
|
||||
if (this.captureHolders.size > 0 || this.suspendedHolders.size > 0) return false;
|
||||
// A tap left over from a dead session is not something to share.
|
||||
if (this.started || this.rawActive) this.teardownCapture();
|
||||
this.suspendedRaw = false;
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
// The raw tap is strictly the G2 stream; clear any phone-mic flag a
|
||||
// preview-mode capture may have left on the shared controller.
|
||||
this.controller?.setUsePhoneMic(false);
|
||||
this.controller?.setSaveRecordings(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||
// unprocessed stream and run their own beam/suppression processing.
|
||||
this.controller?.setNoiseSuppression(false);
|
||||
this.controller?.setBeamFilter(false, 0, 180);
|
||||
this.controller?.clearSpeakerVerification();
|
||||
this.cloudClient = null;
|
||||
this.rawActive = true;
|
||||
this.started = true;
|
||||
@@ -133,6 +199,7 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
stopRawCapture(): void {
|
||||
this.suspendedRaw = false;
|
||||
if (!this.rawActive) return;
|
||||
this.rawActive = false;
|
||||
this.started = false;
|
||||
@@ -144,17 +211,45 @@ export class FaceclawVoiceControlBridge {
|
||||
// STT preempts the raw-PCM tap: a live transcription session owns the mic,
|
||||
// and an EvenHub app gets no audio for its duration.
|
||||
if (this.rawActive) this.stopRawCapture();
|
||||
const wasIdle = this.captureHolders.size === 0;
|
||||
// Whether the mic is already running is a question about the stream, not
|
||||
// about this set: a holder whose capture died with the glasses session
|
||||
// (or whose mic enable was refused) is still in it. Trusting the set here
|
||||
// let one stale holder silently swallow every later request for the
|
||||
// process's lifetime.
|
||||
const micLive = this.micIsLive();
|
||||
this.suspendedHolders.delete(holder);
|
||||
this.captureHolders.add(holder);
|
||||
if (!wasIdle) {
|
||||
if (micLive) {
|
||||
// Mic already running; the new holder just shares the existing stream
|
||||
// (transcripts are already broadcast to its listeners).
|
||||
return;
|
||||
}
|
||||
// Any leftover capture is dead; drop it before starting the new one.
|
||||
if (this.started) this.teardownCapture();
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||
this.controller?.setSaveRecordings(options.saveRecording);
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
this.captureEndpointing = Boolean(options.endpointing);
|
||||
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
||||
if (options.beamFilter) {
|
||||
this.controller?.setBeamFilter(true, Math.round(options.beamFilter.centerDeg), Math.round(options.beamFilter.halfWidthDeg));
|
||||
} else {
|
||||
this.controller?.setBeamFilter(false, 0, 180);
|
||||
}
|
||||
this.verificationRejected = false;
|
||||
if (options.speakerVerification) {
|
||||
this.verificationActive = true;
|
||||
this.controller?.setSpeakerVerification(
|
||||
options.speakerVerification.speakerModelPath,
|
||||
toJavaFloats(options.speakerVerification.wearerEmbedding),
|
||||
options.speakerVerification.threshold,
|
||||
);
|
||||
} else {
|
||||
this.verificationActive = false;
|
||||
this.controller?.clearSpeakerVerification();
|
||||
}
|
||||
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
if (cloudClient) {
|
||||
@@ -209,6 +304,9 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
// A holder that gives up while its capture is parked must not be resumed
|
||||
// by the next session.
|
||||
this.suspendedHolders.delete(holder);
|
||||
if (!this.captureHolders.delete(holder)) {
|
||||
// Never held; still finish a dangling cloud commit if one is pending.
|
||||
if (commit) this.cloudClient?.finish();
|
||||
@@ -236,6 +334,75 @@ export class FaceclawVoiceControlBridge {
|
||||
|
||||
stop(): void {
|
||||
this.captureHolders.clear();
|
||||
this.suspendedHolders.clear();
|
||||
this.suspendedRaw = false;
|
||||
this.teardownCapture();
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
|
||||
/**
|
||||
* The glasses session ended under a live capture (transport drop, charging
|
||||
* case, EvenHub suspend). Its mic enable did not survive, so drop the local
|
||||
* capture state — leaving it in place would make the holder set claim a
|
||||
* stream that no longer exists — and remember who was holding so
|
||||
* resumeCapture() can bring the mic back on the next session.
|
||||
*/
|
||||
handleSessionEnded(): void {
|
||||
if (this.captureHolders.size === 0 && !this.started && !this.rawActive) return;
|
||||
for (const holder of this.captureHolders) {
|
||||
this.suspendedHolders.add(holder);
|
||||
}
|
||||
this.captureHolders.clear();
|
||||
this.suspendedRaw = this.rawActive;
|
||||
this.teardownCapture();
|
||||
if (this.suspendedHolders.size > 0) {
|
||||
this.setStatus("Waiting for the glasses...");
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a capture is parked waiting for a session to come back. */
|
||||
hasSuspendedCapture(): boolean {
|
||||
return this.suspendedHolders.size > 0 || this.suspendedRaw;
|
||||
}
|
||||
|
||||
/** Whether anyone holds the mic, including across a session outage. */
|
||||
isCaptureHeld(): boolean {
|
||||
return this.captureHolders.size > 0 || this.suspendedHolders.size > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* A fresh glasses session is ready (its display path warmed up, so a mic
|
||||
* enable will be accepted): restart the mic for every holder that was
|
||||
* capturing when the last session ended. Mic permission was granted for the
|
||||
* original capture and cannot be revoked without restarting the process, so
|
||||
* this does not ask again.
|
||||
*/
|
||||
resumeCapture(options: Omit<PushToTalkOptions, "endpointing">): void {
|
||||
const holders = Array.from(this.suspendedHolders);
|
||||
const resumeRaw = this.suspendedRaw;
|
||||
this.suspendedHolders.clear();
|
||||
this.suspendedRaw = false;
|
||||
for (const holder of holders) {
|
||||
this.acquireCapture(holder, { ...options, endpointing: this.captureEndpointing });
|
||||
}
|
||||
if (resumeRaw && !holders.length) {
|
||||
this.startRawCapture({ communicator: options.communicator });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the glasses mic is actually streaming. The holder set and
|
||||
* `started` are phone-side bookkeeping; the enable itself lives in the
|
||||
* glasses session, so the controller (via the communicator) is the only
|
||||
* authority on whether audio is still flowing.
|
||||
*/
|
||||
private micIsLive(): boolean {
|
||||
if (!global.isAndroid || !this.started) return false;
|
||||
return Boolean(this.controller?.isCapturing());
|
||||
}
|
||||
|
||||
/** Stop the native capture and any cloud client; leaves holders alone. */
|
||||
private teardownCapture(): void {
|
||||
this.rawActive = false;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
@@ -243,7 +410,6 @@ export class FaceclawVoiceControlBridge {
|
||||
this.started = false;
|
||||
this.cloudClient?.stop();
|
||||
this.cloudClient = null;
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
|
||||
private ensureController(): void {
|
||||
@@ -270,16 +436,37 @@ export class FaceclawVoiceControlBridge {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
},
|
||||
onFrameMeta: (angleDegrees: number, ssr: number) => {
|
||||
if (this.frameMetaListeners.size === 0) return;
|
||||
const meta = { angleDegrees: Number(angleDegrees), ssr: Number(ssr) };
|
||||
this.frameMetaListeners.forEach((listener) => listener(meta));
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
for (const listener of this.speechEndListeners) {
|
||||
listener();
|
||||
}
|
||||
},
|
||||
onSpeakerVerified: (isWearer: boolean, similarity: number) => {
|
||||
if (!this.verificationActive) return;
|
||||
this.verificationRejected = !isWearer;
|
||||
if (!isWearer) {
|
||||
this.setStatus(
|
||||
`Ignored — not your enrolled voice (match ${Math.round(Number(similarity) * 100)}%).`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
private emitTranscript(text: string, isFinal: boolean): void {
|
||||
// "My voice only": a rejected session's final transcript is emptied so
|
||||
// nothing downstream (assistant, dictation) acts on another speaker's
|
||||
// words. The verification verdict is posted before the final, and cloud
|
||||
// finals arrive over the network later still, so the flag is settled.
|
||||
if (isFinal && this.verificationActive && this.verificationRejected) {
|
||||
text = "";
|
||||
}
|
||||
const event = { text, isFinal };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
@@ -294,4 +481,12 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
}
|
||||
|
||||
function toJavaFloats(values: Float32Array): any {
|
||||
const javaFloats = Array.create("float", values.length);
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
javaFloats[i] = values[i]!;
|
||||
}
|
||||
return javaFloats;
|
||||
}
|
||||
|
||||
export const voiceControlBridge = new FaceclawVoiceControlBridge();
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* TS wrapper for the Java FaceclawWearBridge: the phone side of the Wear OS
|
||||
* watch remote. Inbound watch messages arrive as (path, JSON) pairs on the
|
||||
* main isolate's thread; outbound calls publish dashboard state and send
|
||||
* messages to the watch. The semantic mapping (what a watch "click" does)
|
||||
* lives in app/g2/wear-remote.ts; this file only crosses the Java boundary.
|
||||
*
|
||||
* Every call is a no-op when Google Play services is missing (the Wearable
|
||||
* Data Layer is part of it), so the rest of the app never has to care.
|
||||
*/
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/** Message paths shared with the watch app (wear/app/.../Protocol.kt). */
|
||||
export const WEAR_PATHS = {
|
||||
/** Watch -> phone: a ring-style gesture. */
|
||||
input: "/faceclaw/input",
|
||||
/** Watch -> phone: a shell command (launch app, wake, lock, ...). */
|
||||
command: "/faceclaw/command",
|
||||
/** Watch -> phone: a text query for the assistant. */
|
||||
assistant: "/faceclaw/assistant",
|
||||
/** Watch -> phone: text to type into the foreground app. */
|
||||
text: "/faceclaw/text",
|
||||
/** Watch -> phone: please re-publish the state item. */
|
||||
stateRequest: "/faceclaw/state/request",
|
||||
/** Phone -> watch: reply to any of the above. */
|
||||
ack: "/faceclaw/ack",
|
||||
/** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */
|
||||
event: "/faceclaw/event",
|
||||
/** Phone -> watch Data Layer item: the mirrored dashboard state. */
|
||||
state: "/faceclaw/state",
|
||||
} as const;
|
||||
|
||||
export type WearMessage = {
|
||||
path: string;
|
||||
/** Parsed JSON payload ({} when the watch sent none or it was malformed). */
|
||||
payload: Record<string, unknown>;
|
||||
nodeId: string;
|
||||
};
|
||||
|
||||
export type WearWatchConnection = {
|
||||
reachable: boolean;
|
||||
watchName: string;
|
||||
};
|
||||
|
||||
type MessageListener = (message: WearMessage) => void;
|
||||
type ConnectionListener = (connection: WearWatchConnection) => void;
|
||||
|
||||
class WearBridge {
|
||||
private java: any | null | undefined = undefined;
|
||||
// The Java-side listener proxy must stay referenced or it gets GC'd.
|
||||
private retainedListenerProxy: any = null;
|
||||
private readonly messageListeners = new Set<MessageListener>();
|
||||
private readonly connectionListeners = new Set<ConnectionListener>();
|
||||
private lastConnection: WearWatchConnection = { reachable: false, watchName: "" };
|
||||
|
||||
private getJava(): any | null {
|
||||
if (this.java !== undefined) return this.java;
|
||||
if (!global.isAndroid) {
|
||||
this.java = null;
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
const bridge = context ? com.faceclaw.app.FaceclawWearBridge.getInstance(context) : null;
|
||||
this.java = bridge && bridge.isAvailable() ? bridge : null;
|
||||
} catch (error) {
|
||||
console.warn("wear bridge unavailable", error);
|
||||
this.java = null;
|
||||
}
|
||||
return this.java;
|
||||
}
|
||||
|
||||
/** Whether the Wearable Data Layer exists on this phone (Play services). */
|
||||
isAvailable(): boolean {
|
||||
return this.getJava() !== null;
|
||||
}
|
||||
|
||||
private ensureListener(): void {
|
||||
if (this.retainedListenerProxy) return;
|
||||
const java = this.getJava();
|
||||
if (!java) return;
|
||||
this.retainedListenerProxy = new com.faceclaw.app.FaceclawWearListener({
|
||||
onMessage: (path: string, json: string, nodeId: string) => {
|
||||
const message: WearMessage = {
|
||||
path: String(path),
|
||||
payload: parsePayload(String(json)),
|
||||
nodeId: String(nodeId),
|
||||
};
|
||||
for (const listener of Array.from(this.messageListeners)) {
|
||||
try {
|
||||
listener(message);
|
||||
} catch (error) {
|
||||
console.warn("wear message listener failed", error);
|
||||
}
|
||||
}
|
||||
},
|
||||
onWatchConnection: (reachable: boolean, watchName: string) => {
|
||||
this.lastConnection = { reachable: Boolean(reachable), watchName: String(watchName ?? "") };
|
||||
for (const listener of Array.from(this.connectionListeners)) {
|
||||
try {
|
||||
listener(this.lastConnection);
|
||||
} catch (error) {
|
||||
console.warn("wear connection listener failed", error);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
java.setListener(this.retainedListenerProxy);
|
||||
}
|
||||
|
||||
/** Subscribe to watch messages. Registers the Java listener on first use. */
|
||||
onMessage(listener: MessageListener): () => void {
|
||||
this.messageListeners.add(listener);
|
||||
this.ensureListener();
|
||||
return () => {
|
||||
this.messageListeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
/** Subscribe to watch reachability; the current value is delivered immediately. */
|
||||
onWatchConnection(listener: ConnectionListener): () => void {
|
||||
this.connectionListeners.add(listener);
|
||||
this.ensureListener();
|
||||
listener(this.getWatchConnection());
|
||||
return () => {
|
||||
this.connectionListeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
getWatchConnection(): WearWatchConnection {
|
||||
const java = this.getJava();
|
||||
if (!java) return { reachable: false, watchName: "" };
|
||||
try {
|
||||
return {
|
||||
reachable: Boolean(java.isWatchReachable()),
|
||||
watchName: String(java.getWatchName() ?? ""),
|
||||
};
|
||||
} catch {
|
||||
return this.lastConnection;
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-query reachable watches (e.g. when the phone UI wants a fresh answer). */
|
||||
refreshWatchNodes(): void {
|
||||
this.getJava()?.refreshWatchNodes();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror the dashboard state to the watch. Unchanged states are dropped on
|
||||
* the Java side unless `force` (a watch explicitly asked for a refresh).
|
||||
*/
|
||||
publishState(state: Record<string, unknown>, force = false): void {
|
||||
this.getJava()?.publishState(JSON.stringify(state), force);
|
||||
}
|
||||
|
||||
/** Send an event to every reachable watch. */
|
||||
sendEvent(payload: Record<string, unknown>): void {
|
||||
this.getJava()?.sendToWatch(WEAR_PATHS.event, JSON.stringify(payload));
|
||||
}
|
||||
|
||||
/** Acknowledge a watch message; `seq` echoes the watch's sequence number. */
|
||||
sendAck(nodeId: string, seq: number, ok: boolean, message = ""): void {
|
||||
this.getJava()?.sendAck(nodeId, seq, ok, true, message);
|
||||
}
|
||||
}
|
||||
|
||||
function parsePayload(json: string): Record<string, unknown> {
|
||||
try {
|
||||
const parsed = JSON.parse(json);
|
||||
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export const wearBridge = new WearBridge();
|
||||
Reference in new issue
Block a user