From 161dbd104e55973ea0c8f09b12c45e703b768921 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 19 Jul 2026 20:45:51 -0700 Subject: [PATCH] Add wakeword support --- .../faceclaw/app/FaceclawVoiceController.java | 128 ++++++++++++++++++ .../app/FaceclawVoiceControllerListener.java | 7 + .../faceclaw/app/g2protocol/BleProtocol.java | 14 ++ .../com/faceclaw/app/g2protocol/G2Event.java | 23 ++++ app/g2/dashboard-controller.ts | 23 +++- app/g2/events.ts | 20 +++ app/native/faceclaw-communicator.ts | 13 ++ app/native/voice-control.ts | 22 +++ app/ui/apps/voice-input.ts | 35 ++++- app/ui/layers.ts | 15 +- app/ui/shell/shell.ts | 28 +++- 11 files changed, 315 insertions(+), 13 deletions(-) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index ab0a6675..2a3cfd8c 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -89,6 +89,8 @@ public class FaceclawVoiceController { private long lastTranscriptDecodeAtMs; private String lastTranscript = ""; private volatile boolean saveRecordings; + private volatile boolean endpointing; + private final EndpointDetector endpointDetector = new EndpointDetector(); private java.io.ByteArrayOutputStream recordingPcm; private long queuedPackets; private long queueDroppedPackets; @@ -116,6 +118,15 @@ public class FaceclawVoiceController { this.saveRecordings = saveRecordings; } + /** + * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the + * speaker stops. Used by hands-free ("Hey Even") capture, which has no + * button release to end the utterance. Must be set before {@link #start}. + */ + public void setEndpointing(boolean endpointing) { + this.endpointing = endpointing; + } + public void start() { start("wakeword"); } @@ -192,6 +203,7 @@ public class FaceclawVoiceController { stream = keywordSpotter.createStream(); } lc3Decoder = new FaceclawLc3Decoder(); + endpointDetector.reset(); recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null; if (!startG2Audio()) { emitStatus("Could not start G2 microphone input."); @@ -408,6 +420,9 @@ public class FaceclawVoiceController { if (recordingPcm != null) { appendRecording(pcm, count); } + if (endpointing && endpointDetector.accept(pcm, count)) { + emitSpeechEnd(); + } if (mode == VoiceInputMode.CLOUD) { emitPcm(pcm, count); } else if (mode == VoiceInputMode.ONBOARD) { @@ -524,6 +539,119 @@ public class FaceclawVoiceController { mainHandler.post(() -> currentListener.onPcm(le)); } + private void emitSpeechEnd() { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(currentListener::onSpeechEnd); + } + + /** + * Decides when a hands-free utterance is over, so "Hey Even" capture can + * stop without a button release. + * + * Runs on the decoded 16 kHz PCM, so it works the same in every input mode + * (the cloud path never sees the samples on this side, and the onboard + * recognizer's own endpointing only covers ONBOARD). + * + * Timing is measured on the sample clock rather than the wall clock: BLE + * delivers mic packets in bursts, so elapsed real time badly overestimates + * how much audio has actually been heard. + * + * The threshold is relative to a noise floor measured over the first + * {@link #CALIBRATE_MS} of the session, which is roughly the interval where + * the user is reacting to the dialog appearing and not yet speaking. + */ + private static final class EndpointDetector { + /** Audio used to estimate the room's noise floor. */ + private static final int CALIBRATE_MS = 300; + /** Speech must exceed this multiple of the noise floor to count as onset. */ + private static final double ONSET_FACTOR = 3.0; + /** Below this multiple of the noise floor counts as silence again. */ + private static final double RELEASE_FACTOR = 1.8; + /** Absolute floor, so a silent room can't make the threshold ~0. */ + private static final double MIN_RMS = 220.0; + /** Trailing silence that ends an utterance. */ + private static final int SILENCE_MS = 900; + /** If the user never speaks, give up rather than record forever. */ + private static final int LEAD_IN_MS = 6000; + /** Hard cap on a single utterance. */ + private static final int MAX_UTTERANCE_MS = 30000; + + private long totalSamples; + private double noiseAccum; + private int noisePackets; + private double threshold; + private boolean speechStarted; + private long silenceSamples; + private boolean fired; + + void reset() { + totalSamples = 0; + noiseAccum = 0; + noisePackets = 0; + threshold = 0; + speechStarted = false; + silenceSamples = 0; + fired = false; + } + + /** Returns true exactly once, on the packet that ends the utterance. */ + boolean accept(short[] pcm, int count) { + if (fired || count <= 0) { + return false; + } + totalSamples += count; + long elapsedMs = totalSamples * 1000L / SAMPLE_RATE; + + double sumSquares = 0; + for (int i = 0; i < count; i++) { + double s = pcm[i]; + sumSquares += s * s; + } + double rms = Math.sqrt(sumSquares / count); + + if (elapsedMs <= CALIBRATE_MS) { + noiseAccum += rms; + noisePackets++; + return false; + } + if (threshold == 0) { + double noiseFloor = noisePackets > 0 ? noiseAccum / noisePackets : 0; + threshold = Math.max(noiseFloor, MIN_RMS); + } + + if (!speechStarted) { + if (rms >= threshold * ONSET_FACTOR) { + speechStarted = true; + silenceSamples = 0; + } else if (elapsedMs >= LEAD_IN_MS) { + // Never heard anything; close the dialog rather than hang. + fired = true; + return true; + } + return false; + } + + if (rms < threshold * RELEASE_FACTOR) { + silenceSamples += count; + if (silenceSamples * 1000L / SAMPLE_RATE >= SILENCE_MS) { + fired = true; + return true; + } + } else { + silenceSamples = 0; + } + + if (elapsedMs >= MAX_UTTERANCE_MS) { + fired = true; + return true; + } + return false; + } + } + private void stopG2Audio() { FaceclawBleCommunicator currentCommunicator = communicator; if (currentCommunicator != null) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index 88f558bf..bce73665 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -17,4 +17,11 @@ public interface FaceclawVoiceControllerListener { * little-endian PCM. Empty/absent in onboard mode. */ void onPcm(byte[] pcm16le); + + /** + * The speaker stopped, in a hands-free session that has no button release + * to end it. Only fires when endpointing was enabled for the session; see + * FaceclawVoiceController.setEndpointing. + */ + void onSpeechEnd(); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java index dc2e53f5..3bf234c8 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java @@ -31,6 +31,20 @@ public class BleProtocol { public static final int SID_APP_LAUNCH = 0x01; public static final int SID_EVENHUB = 0xe0; public static final int SID_UI_SETTING = 0x09; + // UI_FOREGROUND_EVEN_AI_ID: the stock "Even AI" assistant app. It is a + // FOREGROUND app, whereas EvenHub (and therefore faceclaw) is the + // BACKGROUND app -- which is why an Even AI launch draws over us. + // Payload is g2.even_ai.EvenAIDataPackage (see + // g2-kit-unofficial/ble/gen/even_ai_pb.ts). + public static final int SID_EVEN_AI = 0x07; + // eEvenAICommandId + public static final int EVEN_AI_CMD_CTRL = 1; + // eEvenAIStatus, carried in EvenAIDataPackage.ctrl.status (field 3, sub 1). + // WAKE_UP is the on-glasses "Hey Even" wakeword firing; ENTER is a manual + // entry (touch/double-tap), so the two are distinguishable. + public static final int EVEN_AI_STATUS_WAKE_UP = 1; + public static final int EVEN_AI_STATUS_ENTER = 2; + public static final int EVEN_AI_STATUS_EXIT = 3; public static final int FLAG_REQUEST = 0x20; public static final int FLAG_NOTIFY = 0x01; public static final int FLAG_NOTIFY_ALT = 0x06; diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/G2Event.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/G2Event.java index b7ca26f8..7d5c57bb 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/G2Event.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/G2Event.java @@ -32,6 +32,29 @@ public final class G2Event { public static G2Event decode(BleProtocol.ParsedFrame frame) { byte[] pb = BleProtocol.stripTrailingCrc(frame.pb); + + // sid 0x07 is the stock Even AI app, not EvenHub, so it carries an + // EvenAIDataPackage rather than the device-event wrapper below. + // EvenAIDataPackage { command_id = 1, magic_random = 2, ctrl = 3 }, + // EvenAIControl { status = 1 }. The event we care about is + // command_id=CTRL with status=WAKE_UP, which the firmware emits when the + // on-glasses "Hey Even" wakeword fires. + if (frame.sid == BleProtocol.SID_EVEN_AI) { + int commandId = BleProtocol.readVarintFieldValue(pb, 1, -1); + if (commandId != BleProtocol.EVEN_AI_CMD_CTRL) { + return null; + } + byte[] ctrl = BleProtocol.readFieldBytes(pb, 3); + if (ctrl == null) { + return null; + } + int status = BleProtocol.readVarintFieldValue(ctrl, 1, -1); + if (status < 0) { + return null; + } + return new G2Event("even-ai", "", status, 0, 0); + } + byte[] deviceEvent = BleProtocol.readFieldBytes(pb, 13); if (deviceEvent == null) { return null; diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index c10a2148..33a46f97 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1,5 +1,5 @@ import { ImageSource } from "@nativescript/core"; -import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events"; +import { EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events"; import { loadDeviceAddresses } from "./device-addresses"; import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions"; import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator"; @@ -123,6 +123,18 @@ function eventName(eventType: number): string { return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`; } +/** + * Event-type label for logs. "even-ai" frames carry eEvenAIStatus, not an + * OsEventTypeList value, so naming them with the OS table would be wrong + * (status 1 would read as "SCROLL_TOP_EVENT"). + */ +function eventLabel(kind: string, eventType: number): string { + if (kind === "even-ai") { + return EvenAIStatusName[eventType] ?? `EVEN_AI_UNKNOWN_${eventType}`; + } + return eventName(eventType); +} + function sourceName(eventSource: number): string { return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`; } @@ -636,8 +648,8 @@ class DashboardController { * push-to-talk and the Transcribe app. Android mic permission is the consent * gate even though the audio source is the G2 mic over BLE. */ - private startVoiceCapture(): void { - this.beginVoiceCapture("ptt"); + private startVoiceCapture(endpointing = false): void { + this.beginVoiceCapture("ptt", endpointing); } private stopVoiceCapture(): void { @@ -652,7 +664,7 @@ class DashboardController { voiceControlBridge.stopContinuousCapture(); } - private beginVoiceCapture(kind: "ptt" | "continuous"): void { + private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void { if (this.phase !== "connected" || !this.communicator) { return; } @@ -665,6 +677,7 @@ class DashboardController { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), + endpointing, }; if (kind === "ptt") { voiceControlBridge.startPushToTalk(options); @@ -723,7 +736,7 @@ class DashboardController { private async handleInputEvent(event: RawInputEvent): Promise { const frameId = event.frameId > 0 ? event.frameId : frameTimings.startFrame(`input:${event.kind} (untracked source)`); - frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventName(event.eventType)}`); + frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventLabel(event.kind, event.eventType)}`); let frameOwned = false; try { const inputEvent = rawInputEventToInputEvent(event); diff --git a/app/g2/events.ts b/app/g2/events.ts index 3bb05777..65581b01 100644 --- a/app/g2/events.ts +++ b/app/g2/events.ts @@ -12,6 +12,26 @@ export const EventSourceTypeName: Record = { 3: "TOUCH_EVENT_FROM_GLASSES_L", }; +/** + * eEvenAIStatus, from EvenAIDataPackage.ctrl.status on sid 0x07 + * (UI_FOREGROUND_EVEN_AI_ID). WAKE_UP is the on-glasses "Hey Even" wakeword + * firing; ENTER is a manual entry (touch/double-tap), so the two can be told + * apart. EXIT is the assistant tearing down. + */ +export const EvenAIStatus = { + STATUS_UNKNOWN: 0, + EVEN_AI_WAKE_UP: 1, + EVEN_AI_ENTER: 2, + EVEN_AI_EXIT: 3, +} as const; + +export const EvenAIStatusName: Record = { + 0: "STATUS_UNKNOWN", + 1: "EVEN_AI_WAKE_UP", + 2: "EVEN_AI_ENTER", + 3: "EVEN_AI_EXIT", +}; + export const OsEventTypeList = { CLICK_EVENT: 0, SCROLL_TOP_EVENT: 1, diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 3e6f1ee2..4e556c19 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -75,6 +75,19 @@ export type RawInputEvent = eventSource: number; systemExitReasonCode: number; frameId: number; + } + | { + /** + * Stock Even AI app (sid 0x07), not EvenHub. `eventType` carries + * eEvenAIStatus -- see EvenAIStatus in app/g2/events.ts. The only one we + * act on is EVEN_AI_WAKE_UP, i.e. the "Hey Even" wakeword. + */ + kind: "even-ai"; + containerName: string; + eventType: number; + eventSource: number; + systemExitReasonCode: number; + frameId: number; }; function nonNegativeNumber(value: number): number { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 7190a8ef..9dc59009 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -25,6 +25,12 @@ export type PushToTalkOptions = { provider: VoiceProviderKind; elevenLabsApiKey: string; saveRecording: boolean; + /** + * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free + * ("Hey Even") capture, which has no button release to end the utterance. + * Ignored when the mic is already running for another holder. + */ + endpointing?: boolean; }; /** Who currently wants the mic running. */ @@ -34,6 +40,7 @@ export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); private readonly wakeWordListeners = new Set<(keyword: string) => void>(); private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>(); + private readonly speechEndListeners = new Set<() => void>(); private controller: any | null = null; private listenerProxy: any | null = null; private status = "Voice control stopped."; @@ -62,6 +69,15 @@ export class FaceclawVoiceControlBridge { return () => this.transcriptListeners.delete(listener); } + /** + * The speaker stopped, in a hands-free session. Only fires when the capture + * was started with `endpointing: true`. + */ + onSpeechEnd(listener: () => void): () => void { + this.speechEndListeners.add(listener); + return () => this.speechEndListeners.delete(listener); + } + /** Begin push-to-talk capture (momentary; released with stopPushToTalk). */ startPushToTalk(options: PushToTalkOptions): void { this.acquireCapture("ptt", options); @@ -93,6 +109,7 @@ export class FaceclawVoiceControlBridge { this.ensureController(); this.controller?.setCommunicator(options.communicator); this.controller?.setSaveRecordings(options.saveRecording); + this.controller?.setEndpointing(Boolean(options.endpointing)); if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) { this.cloudClient = new ElevenLabsSttClient({ @@ -174,6 +191,11 @@ export class FaceclawVoiceControlBridge { onPcm: (pcm: any) => { this.cloudClient?.acceptPcm(toUint8Array(pcm)); }, + onSpeechEnd: () => { + for (const listener of this.speechEndListeners) { + listener(); + } + }, }); this.controller.setListener(this.listenerProxy); } diff --git a/app/ui/apps/voice-input.ts b/app/ui/apps/voice-input.ts index cd345577..22d695db 100644 --- a/app/ui/apps/voice-input.ts +++ b/app/ui/apps/voice-input.ts @@ -37,6 +37,10 @@ type VoicePhase = "capturing" | "menu" | "continuing" | "refining"; * * When opened from a menu instead of a held button (finishOnClick), there is * no release event to end the capture, so a single click stops the mic. + * + * When opened by the "Hey Even" wakeword (handsFree), there is likewise no + * button involved: the mic starts immediately and stops when the speaker does, + * via endpoint detection on the decoded PCM. A click still ends it early. */ export class VoiceInputLayer implements Layer { private phase: VoicePhase = "capturing"; @@ -56,12 +60,14 @@ export class VoiceInputLayer implements Layer { private menuIndex = 0; private unsubscribeTranscript: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; + private unsubscribeSpeechEnd: (() => void) | null = null; constructor( private readonly actions: LayerActions, private readonly onClosed: () => void, private readonly onSend: (text: string) => void, private readonly finishOnClick = false, + private readonly handsFree = false, ) {} startCapture(): void { @@ -73,12 +79,31 @@ export class VoiceInputLayer implements Layer { this.status = state.status; this.actions.requestRender(); }); + if (this.handsFree) { + // No button is held, so the mic has to stop itself. endCapture() is + // idempotent, and a click still ends the utterance early. + this.unsubscribeSpeechEnd = voiceControlBridge.onSpeechEnd(() => { + if (this.phase === "capturing") { + this.endCapture(); + } + }); + } this.capturing = true; - void this.actions.startVoiceCapture(); + void this.actions.startVoiceCapture(this.handsFree); this.actions.requestRender(); } - /** Button released (or click in finishOnClick mode): stop the mic. */ + /** + * Whether a single click ends the utterance. True when there is no held + * button to release: opened from the overlay menu (finishOnClick), or + * hands-free from the wakeword — where it also lets the user cut the + * silence-detection wait short. + */ + private get clickEndsCapture(): boolean { + return this.finishOnClick || this.handsFree; + } + + /** Button released (or click when clickEndsCapture): stop the mic. */ endCapture(): void { if (!this.capturing) return; if (this.phase === "continuing") { @@ -150,7 +175,7 @@ export class VoiceInputLayer implements Layer { case "capturing": if (event.type === "double-click") { ctx.stack.pop(); - } else if (this.finishOnClick && event.type === "click") { + } else if (this.clickEndsCapture && event.type === "click") { this.endCapture(); } return; @@ -292,6 +317,8 @@ export class VoiceInputLayer implements Layer { this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); this.unsubscribeStatus = null; + this.unsubscribeSpeechEnd?.(); + this.unsubscribeSpeechEnd = null; if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; @@ -327,7 +354,7 @@ export class VoiceInputLayer implements Layer { private hintText(): string { switch (this.phase) { case "capturing": - return this.finishOnClick + return this.clickEndsCapture ? gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "close"]]) : `${GESTURE_DOUBLE_CLICK} close`; case "continuing": diff --git a/app/ui/layers.ts b/app/ui/layers.ts index a80817b9..2ee79b9d 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -10,6 +10,13 @@ export type DashboardInputEvent = | { type: "scroll-down" } | { type: "long-press"; source: "ring" | "left-arm" | "right-arm" } | { type: "long-press-release"; source: "ring" | "left-arm" | "right-arm" } + /** + * The on-glasses "Hey Even" wakeword fired. Delivered on sid 0x07 by the + * stock firmware regardless of CFW; the CFW additionally suppresses the stock + * Even AI app so this is ours to handle. Unlike every other input event this + * one is honoured while the screen is off. + */ + | { type: "wakeword" } | { type: "unknown"; kind: string; eventSource: number; eventType: number }; export type LayerActions = { @@ -18,8 +25,12 @@ export type LayerActions = { disconnect: () => Promise | void; startTextSettingEdit: (setting: ConfigSettingString) => Promise | void; endTextSettingEdit: () => Promise | void; - /** Start push-to-talk voice capture with the provider chosen in settings. */ - startVoiceCapture: () => Promise | void; + /** + * Start push-to-talk voice capture with the provider chosen in settings. + * With `endpointing`, the capture also ends itself when the speaker stops + * (hands-free); otherwise it runs until stopVoiceCapture. + */ + startVoiceCapture: (endpointing?: boolean) => Promise | void; /** Stop push-to-talk; for a cloud provider this also commits for a final result. */ stopVoiceCapture: () => Promise | void; /** Start continuous capture (Transcribe); shares the mic with push-to-talk. */ diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 20db9916..1b4c49d6 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -1,5 +1,5 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; -import { EventSourceType, OsEventTypeList } from "../../g2/events"; +import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events"; import type { RawInputEvent } from "../../native/faceclaw-communicator"; import { DashboardInputEvent, LayerActions, LayerStack } from "../layers"; import { MenuLayer, type MenuItem } from "../menu"; @@ -320,6 +320,20 @@ class Shell { async receiveInput(event: DashboardInputEvent, frameId = 0): Promise { this.lastInputAtMs = Date.now(); + // The wakeword is handled before the screen-off short-circuit: "Hey Even" + // is meant to work from a dark screen, which is the whole point of it. + // With the CFW the stock Even AI app never launches, so the firmware does + // not power the display for us either -- we wake it ourselves. + if (event.type === "wakeword") { + if (!this.screenOn) { + this.wake("sidebar"); + } + if (!this.activeVoiceLayer) { + this.openVoiceDialog(false, true); + } + return { shell: true, window: false }; + } + if (!this.screenOn) { if (event.type === "double-click") { this.wake("sidebar"); @@ -422,7 +436,7 @@ class Shell { next?.requestRender(); } - private openVoiceDialog(finishOnClick = false): void { + private openVoiceDialog(finishOnClick = false, handsFree = false): void { const layer = new VoiceInputLayer( this.config.actions, () => { @@ -432,6 +446,7 @@ class Shell { }, (text) => this.sendTextToForegroundWindow(text), finishOnClick, + handsFree, ); this.activeVoiceLayer = layer; this.stack.push(layer); @@ -517,6 +532,13 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE } else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) { return { type: "long-press-release", source: eventSourceToString(event.eventSource) }; } + } else if (event.kind === "even-ai") { + // sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the + // wakeword interests us -- ENTER means the user manually opened the stock + // assistant, and EXIT is it tearing down. + if (event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP) { + return { type: "wakeword" }; + } } else if (event.kind === "text-click") { if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) { return { type: "scroll-down" }; @@ -557,6 +579,8 @@ export function inputEventToString(event: DashboardInputEvent): string { return `Long press from ${event.source}`; case "long-press-release": return `Long press release from ${event.source}`; + case "wakeword": + return `Wakeword`; default: case "unknown": return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;