Add wakeword support

This commit is contained in:
James Babcock committed 2026-07-19 20:45:51 -07:00
1 parent 4401a4a416
commit 161dbd104e
11 files changed
+315 -13

No files matched your search

+18 -5
View File
@@ -1,5 +1,5 @@
import { ImageSource } from "@nativescript/core";
import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
import { EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
import { loadDeviceAddresses } from "./device-addresses";
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
@@ -123,6 +123,18 @@ function eventName(eventType: number): string {
return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`;
}
/**
* Event-type label for logs. "even-ai" frames carry eEvenAIStatus, not an
* OsEventTypeList value, so naming them with the OS table would be wrong
* (status 1 would read as "SCROLL_TOP_EVENT").
*/
function eventLabel(kind: string, eventType: number): string {
if (kind === "even-ai") {
return EvenAIStatusName[eventType] ?? `EVEN_AI_UNKNOWN_${eventType}`;
}
return eventName(eventType);
}
function sourceName(eventSource: number): string {
return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`;
}
@@ -636,8 +648,8 @@ class DashboardController {
* push-to-talk and the Transcribe app. Android mic permission is the consent
* gate even though the audio source is the G2 mic over BLE.
*/
private startVoiceCapture(): void {
this.beginVoiceCapture("ptt");
private startVoiceCapture(endpointing = false): void {
this.beginVoiceCapture("ptt", endpointing);
}
private stopVoiceCapture(): void {
@@ -652,7 +664,7 @@ class DashboardController {
voiceControlBridge.stopContinuousCapture();
}
private beginVoiceCapture(kind: "ptt" | "continuous"): void {
private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void {
if (this.phase !== "connected" || !this.communicator) {
return;
}
@@ -665,6 +677,7 @@ class DashboardController {
provider: voiceProviderSetting.get(),
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
endpointing,
};
if (kind === "ptt") {
voiceControlBridge.startPushToTalk(options);
@@ -723,7 +736,7 @@ class DashboardController {
private async handleInputEvent(event: RawInputEvent): Promise<void> {
const frameId =
event.frameId > 0 ? event.frameId : frameTimings.startFrame(`input:${event.kind} (untracked source)`);
frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventName(event.eventType)}`);
frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventLabel(event.kind, event.eventType)}`);
let frameOwned = false;
try {
const inputEvent = rawInputEventToInputEvent(event);
+20
View File
@@ -12,6 +12,26 @@ export const EventSourceTypeName: Record<number, string> = {
3: "TOUCH_EVENT_FROM_GLASSES_L",
};
/**
* eEvenAIStatus, from EvenAIDataPackage.ctrl.status on sid 0x07
* (UI_FOREGROUND_EVEN_AI_ID). WAKE_UP is the on-glasses "Hey Even" wakeword
* firing; ENTER is a manual entry (touch/double-tap), so the two can be told
* apart. EXIT is the assistant tearing down.
*/
export const EvenAIStatus = {
STATUS_UNKNOWN: 0,
EVEN_AI_WAKE_UP: 1,
EVEN_AI_ENTER: 2,
EVEN_AI_EXIT: 3,
} as const;
export const EvenAIStatusName: Record<number, string> = {
0: "STATUS_UNKNOWN",
1: "EVEN_AI_WAKE_UP",
2: "EVEN_AI_ENTER",
3: "EVEN_AI_EXIT",
};
export const OsEventTypeList = {
CLICK_EVENT: 0,
SCROLL_TOP_EVENT: 1,
+13
View File
@@ -75,6 +75,19 @@ export type RawInputEvent =
eventSource: number;
systemExitReasonCode: number;
frameId: number;
}
| {
/**
* Stock Even AI app (sid 0x07), not EvenHub. `eventType` carries
* eEvenAIStatus -- see EvenAIStatus in app/g2/events.ts. The only one we
* act on is EVEN_AI_WAKE_UP, i.e. the "Hey Even" wakeword.
*/
kind: "even-ai";
containerName: string;
eventType: number;
eventSource: number;
systemExitReasonCode: number;
frameId: number;
};
function nonNegativeNumber(value: number): number {
+22
View File
@@ -25,6 +25,12 @@ export type PushToTalkOptions = {
provider: VoiceProviderKind;
elevenLabsApiKey: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
* ("Hey Even") capture, which has no button release to end the utterance.
* Ignored when the mic is already running for another holder.
*/
endpointing?: boolean;
};
/** Who currently wants the mic running. */
@@ -34,6 +40,7 @@ export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
private readonly speechEndListeners = new Set<() => void>();
private controller: any | null = null;
private listenerProxy: any | null = null;
private status = "Voice control stopped.";
@@ -62,6 +69,15 @@ export class FaceclawVoiceControlBridge {
return () => this.transcriptListeners.delete(listener);
}
/**
* The speaker stopped, in a hands-free session. Only fires when the capture
* was started with `endpointing: true`.
*/
onSpeechEnd(listener: () => void): () => void {
this.speechEndListeners.add(listener);
return () => this.speechEndListeners.delete(listener);
}
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
startPushToTalk(options: PushToTalkOptions): void {
this.acquireCapture("ptt", options);
@@ -93,6 +109,7 @@ export class FaceclawVoiceControlBridge {
this.ensureController();
this.controller?.setCommunicator(options.communicator);
this.controller?.setSaveRecordings(options.saveRecording);
this.controller?.setEndpointing(Boolean(options.endpointing));
if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) {
this.cloudClient = new ElevenLabsSttClient({
@@ -174,6 +191,11 @@ export class FaceclawVoiceControlBridge {
onPcm: (pcm: any) => {
this.cloudClient?.acceptPcm(toUint8Array(pcm));
},
onSpeechEnd: () => {
for (const listener of this.speechEndListeners) {
listener();
}
},
});
this.controller.setListener(this.listenerProxy);
}
+31 -4
View File
@@ -37,6 +37,10 @@ type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
*
* When opened from a menu instead of a held button (finishOnClick), there is
* no release event to end the capture, so a single click stops the mic.
*
* When opened by the "Hey Even" wakeword (handsFree), there is likewise no
* button involved: the mic starts immediately and stops when the speaker does,
* via endpoint detection on the decoded PCM. A click still ends it early.
*/
export class VoiceInputLayer implements Layer {
private phase: VoicePhase = "capturing";
@@ -56,12 +60,14 @@ export class VoiceInputLayer implements Layer {
private menuIndex = 0;
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private unsubscribeSpeechEnd: (() => void) | null = null;
constructor(
private readonly actions: LayerActions,
private readonly onClosed: () => void,
private readonly onSend: (text: string) => void,
private readonly finishOnClick = false,
private readonly handsFree = false,
) {}
startCapture(): void {
@@ -73,12 +79,31 @@ export class VoiceInputLayer implements Layer {
this.status = state.status;
this.actions.requestRender();
});
if (this.handsFree) {
// No button is held, so the mic has to stop itself. endCapture() is
// idempotent, and a click still ends the utterance early.
this.unsubscribeSpeechEnd = voiceControlBridge.onSpeechEnd(() => {
if (this.phase === "capturing") {
this.endCapture();
}
});
}
this.capturing = true;
void this.actions.startVoiceCapture();
void this.actions.startVoiceCapture(this.handsFree);
this.actions.requestRender();
}
/** Button released (or click in finishOnClick mode): stop the mic. */
/**
* Whether a single click ends the utterance. True when there is no held
* button to release: opened from the overlay menu (finishOnClick), or
* hands-free from the wakeword — where it also lets the user cut the
* silence-detection wait short.
*/
private get clickEndsCapture(): boolean {
return this.finishOnClick || this.handsFree;
}
/** Button released (or click when clickEndsCapture): stop the mic. */
endCapture(): void {
if (!this.capturing) return;
if (this.phase === "continuing") {
@@ -150,7 +175,7 @@ export class VoiceInputLayer implements Layer {
case "capturing":
if (event.type === "double-click") {
ctx.stack.pop();
} else if (this.finishOnClick && event.type === "click") {
} else if (this.clickEndsCapture && event.type === "click") {
this.endCapture();
}
return;
@@ -292,6 +317,8 @@ export class VoiceInputLayer implements Layer {
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
this.unsubscribeSpeechEnd?.();
this.unsubscribeSpeechEnd = null;
if (this.followupFinalizeTimer !== null) {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
@@ -327,7 +354,7 @@ export class VoiceInputLayer implements Layer {
private hintText(): string {
switch (this.phase) {
case "capturing":
return this.finishOnClick
return this.clickEndsCapture
? gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "close"]])
: `${GESTURE_DOUBLE_CLICK} close`;
case "continuing":
+13 -2
View File
@@ -10,6 +10,13 @@ export type DashboardInputEvent =
| { type: "scroll-down" }
| { type: "long-press"; source: "ring" | "left-arm" | "right-arm" }
| { type: "long-press-release"; source: "ring" | "left-arm" | "right-arm" }
/**
* The on-glasses "Hey Even" wakeword fired. Delivered on sid 0x07 by the
* stock firmware regardless of CFW; the CFW additionally suppresses the stock
* Even AI app so this is ours to handle. Unlike every other input event this
* one is honoured while the screen is off.
*/
| { type: "wakeword" }
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
export type LayerActions = {
@@ -18,8 +25,12 @@ export type LayerActions = {
disconnect: () => Promise<void> | void;
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
endTextSettingEdit: () => Promise<void> | void;
/** Start push-to-talk voice capture with the provider chosen in settings. */
startVoiceCapture: () => Promise<void> | void;
/**
* Start push-to-talk voice capture with the provider chosen in settings.
* With `endpointing`, the capture also ends itself when the speaker stops
* (hands-free); otherwise it runs until stopVoiceCapture.
*/
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
stopVoiceCapture: () => Promise<void> | void;
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
+26 -2
View File
@@ -1,5 +1,5 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { EventSourceType, OsEventTypeList } from "../../g2/events";
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import { DashboardInputEvent, LayerActions, LayerStack } from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
@@ -320,6 +320,20 @@ class Shell {
async receiveInput(event: DashboardInputEvent, frameId = 0): Promise<ShellInputOutcome> {
this.lastInputAtMs = Date.now();
// The wakeword is handled before the screen-off short-circuit: "Hey Even"
// is meant to work from a dark screen, which is the whole point of it.
// With the CFW the stock Even AI app never launches, so the firmware does
// not power the display for us either -- we wake it ourselves.
if (event.type === "wakeword") {
if (!this.screenOn) {
this.wake("sidebar");
}
if (!this.activeVoiceLayer) {
this.openVoiceDialog(false, true);
}
return { shell: true, window: false };
}
if (!this.screenOn) {
if (event.type === "double-click") {
this.wake("sidebar");
@@ -422,7 +436,7 @@ class Shell {
next?.requestRender();
}
private openVoiceDialog(finishOnClick = false): void {
private openVoiceDialog(finishOnClick = false, handsFree = false): void {
const layer = new VoiceInputLayer(
this.config.actions,
() => {
@@ -432,6 +446,7 @@ class Shell {
},
(text) => this.sendTextToForegroundWindow(text),
finishOnClick,
handsFree,
);
this.activeVoiceLayer = layer;
this.stack.push(layer);
@@ -517,6 +532,13 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
}
} else if (event.kind === "even-ai") {
// sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the
// wakeword interests us -- ENTER means the user manually opened the stock
// assistant, and EXIT is it tearing down.
if (event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP) {
return { type: "wakeword" };
}
} else if (event.kind === "text-click") {
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
@@ -557,6 +579,8 @@ export function inputEventToString(event: DashboardInputEvent): string {
return `Long press from ${event.source}`;
case "long-press-release":
return `Long press release from ${event.source}`;
case "wakeword":
return `Wakeword`;
default:
case "unknown":
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;