forked from Blendman974/faceclaw
Add wakeword support
This commit is contained in:
1 parent
4401a4a416
commit
161dbd104e
11 files changed
+315
-13
No files matched your search
@@ -37,6 +37,10 @@ type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
|
||||
*
|
||||
* When opened from a menu instead of a held button (finishOnClick), there is
|
||||
* no release event to end the capture, so a single click stops the mic.
|
||||
*
|
||||
* When opened by the "Hey Even" wakeword (handsFree), there is likewise no
|
||||
* button involved: the mic starts immediately and stops when the speaker does,
|
||||
* via endpoint detection on the decoded PCM. A click still ends it early.
|
||||
*/
|
||||
export class VoiceInputLayer implements Layer {
|
||||
private phase: VoicePhase = "capturing";
|
||||
@@ -56,12 +60,14 @@ export class VoiceInputLayer implements Layer {
|
||||
private menuIndex = 0;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private unsubscribeSpeechEnd: (() => void) | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly onClosed: () => void,
|
||||
private readonly onSend: (text: string) => void,
|
||||
private readonly finishOnClick = false,
|
||||
private readonly handsFree = false,
|
||||
) {}
|
||||
|
||||
startCapture(): void {
|
||||
@@ -73,12 +79,31 @@ export class VoiceInputLayer implements Layer {
|
||||
this.status = state.status;
|
||||
this.actions.requestRender();
|
||||
});
|
||||
if (this.handsFree) {
|
||||
// No button is held, so the mic has to stop itself. endCapture() is
|
||||
// idempotent, and a click still ends the utterance early.
|
||||
this.unsubscribeSpeechEnd = voiceControlBridge.onSpeechEnd(() => {
|
||||
if (this.phase === "capturing") {
|
||||
this.endCapture();
|
||||
}
|
||||
});
|
||||
}
|
||||
this.capturing = true;
|
||||
void this.actions.startVoiceCapture();
|
||||
void this.actions.startVoiceCapture(this.handsFree);
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
/** Button released (or click in finishOnClick mode): stop the mic. */
|
||||
/**
|
||||
* Whether a single click ends the utterance. True when there is no held
|
||||
* button to release: opened from the overlay menu (finishOnClick), or
|
||||
* hands-free from the wakeword — where it also lets the user cut the
|
||||
* silence-detection wait short.
|
||||
*/
|
||||
private get clickEndsCapture(): boolean {
|
||||
return this.finishOnClick || this.handsFree;
|
||||
}
|
||||
|
||||
/** Button released (or click when clickEndsCapture): stop the mic. */
|
||||
endCapture(): void {
|
||||
if (!this.capturing) return;
|
||||
if (this.phase === "continuing") {
|
||||
@@ -150,7 +175,7 @@ export class VoiceInputLayer implements Layer {
|
||||
case "capturing":
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
} else if (this.finishOnClick && event.type === "click") {
|
||||
} else if (this.clickEndsCapture && event.type === "click") {
|
||||
this.endCapture();
|
||||
}
|
||||
return;
|
||||
@@ -292,6 +317,8 @@ export class VoiceInputLayer implements Layer {
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
this.unsubscribeSpeechEnd?.();
|
||||
this.unsubscribeSpeechEnd = null;
|
||||
if (this.followupFinalizeTimer !== null) {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
@@ -327,7 +354,7 @@ export class VoiceInputLayer implements Layer {
|
||||
private hintText(): string {
|
||||
switch (this.phase) {
|
||||
case "capturing":
|
||||
return this.finishOnClick
|
||||
return this.clickEndsCapture
|
||||
? gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "close"]])
|
||||
: `${GESTURE_DOUBLE_CLICK} close`;
|
||||
case "continuing":
|
||||
|
||||
+13
-2
@@ -10,6 +10,13 @@ export type DashboardInputEvent =
|
||||
| { type: "scroll-down" }
|
||||
| { type: "long-press"; source: "ring" | "left-arm" | "right-arm" }
|
||||
| { type: "long-press-release"; source: "ring" | "left-arm" | "right-arm" }
|
||||
/**
|
||||
* The on-glasses "Hey Even" wakeword fired. Delivered on sid 0x07 by the
|
||||
* stock firmware regardless of CFW; the CFW additionally suppresses the stock
|
||||
* Even AI app so this is ours to handle. Unlike every other input event this
|
||||
* one is honoured while the screen is off.
|
||||
*/
|
||||
| { type: "wakeword" }
|
||||
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
|
||||
|
||||
export type LayerActions = {
|
||||
@@ -18,8 +25,12 @@ export type LayerActions = {
|
||||
disconnect: () => Promise<void> | void;
|
||||
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
|
||||
endTextSettingEdit: () => Promise<void> | void;
|
||||
/** Start push-to-talk voice capture with the provider chosen in settings. */
|
||||
startVoiceCapture: () => Promise<void> | void;
|
||||
/**
|
||||
* Start push-to-talk voice capture with the provider chosen in settings.
|
||||
* With `endpointing`, the capture also ends itself when the speaker stops
|
||||
* (hands-free); otherwise it runs until stopVoiceCapture.
|
||||
*/
|
||||
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
|
||||
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
||||
|
||||
+26
-2
@@ -1,5 +1,5 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { EventSourceType, OsEventTypeList } from "../../g2/events";
|
||||
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
|
||||
import type { RawInputEvent } from "../../native/faceclaw-communicator";
|
||||
import { DashboardInputEvent, LayerActions, LayerStack } from "../layers";
|
||||
import { MenuLayer, type MenuItem } from "../menu";
|
||||
@@ -320,6 +320,20 @@ class Shell {
|
||||
async receiveInput(event: DashboardInputEvent, frameId = 0): Promise<ShellInputOutcome> {
|
||||
this.lastInputAtMs = Date.now();
|
||||
|
||||
// The wakeword is handled before the screen-off short-circuit: "Hey Even"
|
||||
// is meant to work from a dark screen, which is the whole point of it.
|
||||
// With the CFW the stock Even AI app never launches, so the firmware does
|
||||
// not power the display for us either -- we wake it ourselves.
|
||||
if (event.type === "wakeword") {
|
||||
if (!this.screenOn) {
|
||||
this.wake("sidebar");
|
||||
}
|
||||
if (!this.activeVoiceLayer) {
|
||||
this.openVoiceDialog(false, true);
|
||||
}
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
|
||||
if (!this.screenOn) {
|
||||
if (event.type === "double-click") {
|
||||
this.wake("sidebar");
|
||||
@@ -422,7 +436,7 @@ class Shell {
|
||||
next?.requestRender();
|
||||
}
|
||||
|
||||
private openVoiceDialog(finishOnClick = false): void {
|
||||
private openVoiceDialog(finishOnClick = false, handsFree = false): void {
|
||||
const layer = new VoiceInputLayer(
|
||||
this.config.actions,
|
||||
() => {
|
||||
@@ -432,6 +446,7 @@ class Shell {
|
||||
},
|
||||
(text) => this.sendTextToForegroundWindow(text),
|
||||
finishOnClick,
|
||||
handsFree,
|
||||
);
|
||||
this.activeVoiceLayer = layer;
|
||||
this.stack.push(layer);
|
||||
@@ -517,6 +532,13 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
|
||||
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
|
||||
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
|
||||
}
|
||||
} else if (event.kind === "even-ai") {
|
||||
// sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the
|
||||
// wakeword interests us -- ENTER means the user manually opened the stock
|
||||
// assistant, and EXIT is it tearing down.
|
||||
if (event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP) {
|
||||
return { type: "wakeword" };
|
||||
}
|
||||
} else if (event.kind === "text-click") {
|
||||
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
|
||||
return { type: "scroll-down" };
|
||||
@@ -557,6 +579,8 @@ export function inputEventToString(event: DashboardInputEvent): string {
|
||||
return `Long press from ${event.source}`;
|
||||
case "long-press-release":
|
||||
return `Long press release from ${event.source}`;
|
||||
case "wakeword":
|
||||
return `Wakeword`;
|
||||
default:
|
||||
case "unknown":
|
||||
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;
|
||||
|
||||
Reference in new issue
Block a user