forked from Blendman974/faceclaw
Add wakeword support
This commit is contained in:
1 parent
4401a4a416
commit
161dbd104e
11 files changed
+315
-13
No files matched your search
@@ -89,6 +89,8 @@ public class FaceclawVoiceController {
|
|||||||
private long lastTranscriptDecodeAtMs;
|
private long lastTranscriptDecodeAtMs;
|
||||||
private String lastTranscript = "";
|
private String lastTranscript = "";
|
||||||
private volatile boolean saveRecordings;
|
private volatile boolean saveRecordings;
|
||||||
|
private volatile boolean endpointing;
|
||||||
|
private final EndpointDetector endpointDetector = new EndpointDetector();
|
||||||
private java.io.ByteArrayOutputStream recordingPcm;
|
private java.io.ByteArrayOutputStream recordingPcm;
|
||||||
private long queuedPackets;
|
private long queuedPackets;
|
||||||
private long queueDroppedPackets;
|
private long queueDroppedPackets;
|
||||||
@@ -116,6 +118,15 @@ public class FaceclawVoiceController {
|
|||||||
this.saveRecordings = saveRecordings;
|
this.saveRecordings = saveRecordings;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||||
|
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||||
|
* button release to end the utterance. Must be set before {@link #start}.
|
||||||
|
*/
|
||||||
|
public void setEndpointing(boolean endpointing) {
|
||||||
|
this.endpointing = endpointing;
|
||||||
|
}
|
||||||
|
|
||||||
public void start() {
|
public void start() {
|
||||||
start("wakeword");
|
start("wakeword");
|
||||||
}
|
}
|
||||||
@@ -192,6 +203,7 @@ public class FaceclawVoiceController {
|
|||||||
stream = keywordSpotter.createStream();
|
stream = keywordSpotter.createStream();
|
||||||
}
|
}
|
||||||
lc3Decoder = new FaceclawLc3Decoder();
|
lc3Decoder = new FaceclawLc3Decoder();
|
||||||
|
endpointDetector.reset();
|
||||||
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
|
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
|
||||||
if (!startG2Audio()) {
|
if (!startG2Audio()) {
|
||||||
emitStatus("Could not start G2 microphone input.");
|
emitStatus("Could not start G2 microphone input.");
|
||||||
@@ -408,6 +420,9 @@ public class FaceclawVoiceController {
|
|||||||
if (recordingPcm != null) {
|
if (recordingPcm != null) {
|
||||||
appendRecording(pcm, count);
|
appendRecording(pcm, count);
|
||||||
}
|
}
|
||||||
|
if (endpointing && endpointDetector.accept(pcm, count)) {
|
||||||
|
emitSpeechEnd();
|
||||||
|
}
|
||||||
if (mode == VoiceInputMode.CLOUD) {
|
if (mode == VoiceInputMode.CLOUD) {
|
||||||
emitPcm(pcm, count);
|
emitPcm(pcm, count);
|
||||||
} else if (mode == VoiceInputMode.ONBOARD) {
|
} else if (mode == VoiceInputMode.ONBOARD) {
|
||||||
@@ -524,6 +539,119 @@ public class FaceclawVoiceController {
|
|||||||
mainHandler.post(() -> currentListener.onPcm(le));
|
mainHandler.post(() -> currentListener.onPcm(le));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void emitSpeechEnd() {
|
||||||
|
FaceclawVoiceControllerListener currentListener = listener;
|
||||||
|
if (currentListener == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
mainHandler.post(currentListener::onSpeechEnd);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decides when a hands-free utterance is over, so "Hey Even" capture can
|
||||||
|
* stop without a button release.
|
||||||
|
*
|
||||||
|
* Runs on the decoded 16 kHz PCM, so it works the same in every input mode
|
||||||
|
* (the cloud path never sees the samples on this side, and the onboard
|
||||||
|
* recognizer's own endpointing only covers ONBOARD).
|
||||||
|
*
|
||||||
|
* Timing is measured on the sample clock rather than the wall clock: BLE
|
||||||
|
* delivers mic packets in bursts, so elapsed real time badly overestimates
|
||||||
|
* how much audio has actually been heard.
|
||||||
|
*
|
||||||
|
* The threshold is relative to a noise floor measured over the first
|
||||||
|
* {@link #CALIBRATE_MS} of the session, which is roughly the interval where
|
||||||
|
* the user is reacting to the dialog appearing and not yet speaking.
|
||||||
|
*/
|
||||||
|
private static final class EndpointDetector {
|
||||||
|
/** Audio used to estimate the room's noise floor. */
|
||||||
|
private static final int CALIBRATE_MS = 300;
|
||||||
|
/** Speech must exceed this multiple of the noise floor to count as onset. */
|
||||||
|
private static final double ONSET_FACTOR = 3.0;
|
||||||
|
/** Below this multiple of the noise floor counts as silence again. */
|
||||||
|
private static final double RELEASE_FACTOR = 1.8;
|
||||||
|
/** Absolute floor, so a silent room can't make the threshold ~0. */
|
||||||
|
private static final double MIN_RMS = 220.0;
|
||||||
|
/** Trailing silence that ends an utterance. */
|
||||||
|
private static final int SILENCE_MS = 900;
|
||||||
|
/** If the user never speaks, give up rather than record forever. */
|
||||||
|
private static final int LEAD_IN_MS = 6000;
|
||||||
|
/** Hard cap on a single utterance. */
|
||||||
|
private static final int MAX_UTTERANCE_MS = 30000;
|
||||||
|
|
||||||
|
private long totalSamples;
|
||||||
|
private double noiseAccum;
|
||||||
|
private int noisePackets;
|
||||||
|
private double threshold;
|
||||||
|
private boolean speechStarted;
|
||||||
|
private long silenceSamples;
|
||||||
|
private boolean fired;
|
||||||
|
|
||||||
|
void reset() {
|
||||||
|
totalSamples = 0;
|
||||||
|
noiseAccum = 0;
|
||||||
|
noisePackets = 0;
|
||||||
|
threshold = 0;
|
||||||
|
speechStarted = false;
|
||||||
|
silenceSamples = 0;
|
||||||
|
fired = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Returns true exactly once, on the packet that ends the utterance. */
|
||||||
|
boolean accept(short[] pcm, int count) {
|
||||||
|
if (fired || count <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
totalSamples += count;
|
||||||
|
long elapsedMs = totalSamples * 1000L / SAMPLE_RATE;
|
||||||
|
|
||||||
|
double sumSquares = 0;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
double s = pcm[i];
|
||||||
|
sumSquares += s * s;
|
||||||
|
}
|
||||||
|
double rms = Math.sqrt(sumSquares / count);
|
||||||
|
|
||||||
|
if (elapsedMs <= CALIBRATE_MS) {
|
||||||
|
noiseAccum += rms;
|
||||||
|
noisePackets++;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (threshold == 0) {
|
||||||
|
double noiseFloor = noisePackets > 0 ? noiseAccum / noisePackets : 0;
|
||||||
|
threshold = Math.max(noiseFloor, MIN_RMS);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!speechStarted) {
|
||||||
|
if (rms >= threshold * ONSET_FACTOR) {
|
||||||
|
speechStarted = true;
|
||||||
|
silenceSamples = 0;
|
||||||
|
} else if (elapsedMs >= LEAD_IN_MS) {
|
||||||
|
// Never heard anything; close the dialog rather than hang.
|
||||||
|
fired = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rms < threshold * RELEASE_FACTOR) {
|
||||||
|
silenceSamples += count;
|
||||||
|
if (silenceSamples * 1000L / SAMPLE_RATE >= SILENCE_MS) {
|
||||||
|
fired = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
silenceSamples = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (elapsedMs >= MAX_UTTERANCE_MS) {
|
||||||
|
fired = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void stopG2Audio() {
|
private void stopG2Audio() {
|
||||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||||
if (currentCommunicator != null) {
|
if (currentCommunicator != null) {
|
||||||
|
|||||||
+7
@@ -17,4 +17,11 @@ public interface FaceclawVoiceControllerListener {
|
|||||||
* little-endian PCM. Empty/absent in onboard mode.
|
* little-endian PCM. Empty/absent in onboard mode.
|
||||||
*/
|
*/
|
||||||
void onPcm(byte[] pcm16le);
|
void onPcm(byte[] pcm16le);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The speaker stopped, in a hands-free session that has no button release
|
||||||
|
* to end it. Only fires when endpointing was enabled for the session; see
|
||||||
|
* FaceclawVoiceController.setEndpointing.
|
||||||
|
*/
|
||||||
|
void onSpeechEnd();
|
||||||
}
|
}
|
||||||
@@ -31,6 +31,20 @@ public class BleProtocol {
|
|||||||
public static final int SID_APP_LAUNCH = 0x01;
|
public static final int SID_APP_LAUNCH = 0x01;
|
||||||
public static final int SID_EVENHUB = 0xe0;
|
public static final int SID_EVENHUB = 0xe0;
|
||||||
public static final int SID_UI_SETTING = 0x09;
|
public static final int SID_UI_SETTING = 0x09;
|
||||||
|
// UI_FOREGROUND_EVEN_AI_ID: the stock "Even AI" assistant app. It is a
|
||||||
|
// FOREGROUND app, whereas EvenHub (and therefore faceclaw) is the
|
||||||
|
// BACKGROUND app -- which is why an Even AI launch draws over us.
|
||||||
|
// Payload is g2.even_ai.EvenAIDataPackage (see
|
||||||
|
// g2-kit-unofficial/ble/gen/even_ai_pb.ts).
|
||||||
|
public static final int SID_EVEN_AI = 0x07;
|
||||||
|
// eEvenAICommandId
|
||||||
|
public static final int EVEN_AI_CMD_CTRL = 1;
|
||||||
|
// eEvenAIStatus, carried in EvenAIDataPackage.ctrl.status (field 3, sub 1).
|
||||||
|
// WAKE_UP is the on-glasses "Hey Even" wakeword firing; ENTER is a manual
|
||||||
|
// entry (touch/double-tap), so the two are distinguishable.
|
||||||
|
public static final int EVEN_AI_STATUS_WAKE_UP = 1;
|
||||||
|
public static final int EVEN_AI_STATUS_ENTER = 2;
|
||||||
|
public static final int EVEN_AI_STATUS_EXIT = 3;
|
||||||
public static final int FLAG_REQUEST = 0x20;
|
public static final int FLAG_REQUEST = 0x20;
|
||||||
public static final int FLAG_NOTIFY = 0x01;
|
public static final int FLAG_NOTIFY = 0x01;
|
||||||
public static final int FLAG_NOTIFY_ALT = 0x06;
|
public static final int FLAG_NOTIFY_ALT = 0x06;
|
||||||
|
|||||||
@@ -32,6 +32,29 @@ public final class G2Event {
|
|||||||
|
|
||||||
public static G2Event decode(BleProtocol.ParsedFrame frame) {
|
public static G2Event decode(BleProtocol.ParsedFrame frame) {
|
||||||
byte[] pb = BleProtocol.stripTrailingCrc(frame.pb);
|
byte[] pb = BleProtocol.stripTrailingCrc(frame.pb);
|
||||||
|
|
||||||
|
// sid 0x07 is the stock Even AI app, not EvenHub, so it carries an
|
||||||
|
// EvenAIDataPackage rather than the device-event wrapper below.
|
||||||
|
// EvenAIDataPackage { command_id = 1, magic_random = 2, ctrl = 3 },
|
||||||
|
// EvenAIControl { status = 1 }. The event we care about is
|
||||||
|
// command_id=CTRL with status=WAKE_UP, which the firmware emits when the
|
||||||
|
// on-glasses "Hey Even" wakeword fires.
|
||||||
|
if (frame.sid == BleProtocol.SID_EVEN_AI) {
|
||||||
|
int commandId = BleProtocol.readVarintFieldValue(pb, 1, -1);
|
||||||
|
if (commandId != BleProtocol.EVEN_AI_CMD_CTRL) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
byte[] ctrl = BleProtocol.readFieldBytes(pb, 3);
|
||||||
|
if (ctrl == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
int status = BleProtocol.readVarintFieldValue(ctrl, 1, -1);
|
||||||
|
if (status < 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new G2Event("even-ai", "", status, 0, 0);
|
||||||
|
}
|
||||||
|
|
||||||
byte[] deviceEvent = BleProtocol.readFieldBytes(pb, 13);
|
byte[] deviceEvent = BleProtocol.readFieldBytes(pb, 13);
|
||||||
if (deviceEvent == null) {
|
if (deviceEvent == null) {
|
||||||
return null;
|
return null;
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { ImageSource } from "@nativescript/core";
|
import { ImageSource } from "@nativescript/core";
|
||||||
import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
|
import { EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
|
||||||
import { loadDeviceAddresses } from "./device-addresses";
|
import { loadDeviceAddresses } from "./device-addresses";
|
||||||
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
|
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
|
||||||
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
|
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
|
||||||
@@ -123,6 +123,18 @@ function eventName(eventType: number): string {
|
|||||||
return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`;
|
return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Event-type label for logs. "even-ai" frames carry eEvenAIStatus, not an
|
||||||
|
* OsEventTypeList value, so naming them with the OS table would be wrong
|
||||||
|
* (status 1 would read as "SCROLL_TOP_EVENT").
|
||||||
|
*/
|
||||||
|
function eventLabel(kind: string, eventType: number): string {
|
||||||
|
if (kind === "even-ai") {
|
||||||
|
return EvenAIStatusName[eventType] ?? `EVEN_AI_UNKNOWN_${eventType}`;
|
||||||
|
}
|
||||||
|
return eventName(eventType);
|
||||||
|
}
|
||||||
|
|
||||||
function sourceName(eventSource: number): string {
|
function sourceName(eventSource: number): string {
|
||||||
return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`;
|
return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`;
|
||||||
}
|
}
|
||||||
@@ -636,8 +648,8 @@ class DashboardController {
|
|||||||
* push-to-talk and the Transcribe app. Android mic permission is the consent
|
* push-to-talk and the Transcribe app. Android mic permission is the consent
|
||||||
* gate even though the audio source is the G2 mic over BLE.
|
* gate even though the audio source is the G2 mic over BLE.
|
||||||
*/
|
*/
|
||||||
private startVoiceCapture(): void {
|
private startVoiceCapture(endpointing = false): void {
|
||||||
this.beginVoiceCapture("ptt");
|
this.beginVoiceCapture("ptt", endpointing);
|
||||||
}
|
}
|
||||||
|
|
||||||
private stopVoiceCapture(): void {
|
private stopVoiceCapture(): void {
|
||||||
@@ -652,7 +664,7 @@ class DashboardController {
|
|||||||
voiceControlBridge.stopContinuousCapture();
|
voiceControlBridge.stopContinuousCapture();
|
||||||
}
|
}
|
||||||
|
|
||||||
private beginVoiceCapture(kind: "ptt" | "continuous"): void {
|
private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void {
|
||||||
if (this.phase !== "connected" || !this.communicator) {
|
if (this.phase !== "connected" || !this.communicator) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -665,6 +677,7 @@ class DashboardController {
|
|||||||
provider: voiceProviderSetting.get(),
|
provider: voiceProviderSetting.get(),
|
||||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||||
|
endpointing,
|
||||||
};
|
};
|
||||||
if (kind === "ptt") {
|
if (kind === "ptt") {
|
||||||
voiceControlBridge.startPushToTalk(options);
|
voiceControlBridge.startPushToTalk(options);
|
||||||
@@ -723,7 +736,7 @@ class DashboardController {
|
|||||||
private async handleInputEvent(event: RawInputEvent): Promise<void> {
|
private async handleInputEvent(event: RawInputEvent): Promise<void> {
|
||||||
const frameId =
|
const frameId =
|
||||||
event.frameId > 0 ? event.frameId : frameTimings.startFrame(`input:${event.kind} (untracked source)`);
|
event.frameId > 0 ? event.frameId : frameTimings.startFrame(`input:${event.kind} (untracked source)`);
|
||||||
frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventName(event.eventType)}`);
|
frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventLabel(event.kind, event.eventType)}`);
|
||||||
let frameOwned = false;
|
let frameOwned = false;
|
||||||
try {
|
try {
|
||||||
const inputEvent = rawInputEventToInputEvent(event);
|
const inputEvent = rawInputEventToInputEvent(event);
|
||||||
|
|||||||
@@ -12,6 +12,26 @@ export const EventSourceTypeName: Record<number, string> = {
|
|||||||
3: "TOUCH_EVENT_FROM_GLASSES_L",
|
3: "TOUCH_EVENT_FROM_GLASSES_L",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* eEvenAIStatus, from EvenAIDataPackage.ctrl.status on sid 0x07
|
||||||
|
* (UI_FOREGROUND_EVEN_AI_ID). WAKE_UP is the on-glasses "Hey Even" wakeword
|
||||||
|
* firing; ENTER is a manual entry (touch/double-tap), so the two can be told
|
||||||
|
* apart. EXIT is the assistant tearing down.
|
||||||
|
*/
|
||||||
|
export const EvenAIStatus = {
|
||||||
|
STATUS_UNKNOWN: 0,
|
||||||
|
EVEN_AI_WAKE_UP: 1,
|
||||||
|
EVEN_AI_ENTER: 2,
|
||||||
|
EVEN_AI_EXIT: 3,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export const EvenAIStatusName: Record<number, string> = {
|
||||||
|
0: "STATUS_UNKNOWN",
|
||||||
|
1: "EVEN_AI_WAKE_UP",
|
||||||
|
2: "EVEN_AI_ENTER",
|
||||||
|
3: "EVEN_AI_EXIT",
|
||||||
|
};
|
||||||
|
|
||||||
export const OsEventTypeList = {
|
export const OsEventTypeList = {
|
||||||
CLICK_EVENT: 0,
|
CLICK_EVENT: 0,
|
||||||
SCROLL_TOP_EVENT: 1,
|
SCROLL_TOP_EVENT: 1,
|
||||||
|
|||||||
@@ -75,6 +75,19 @@ export type RawInputEvent =
|
|||||||
eventSource: number;
|
eventSource: number;
|
||||||
systemExitReasonCode: number;
|
systemExitReasonCode: number;
|
||||||
frameId: number;
|
frameId: number;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
/**
|
||||||
|
* Stock Even AI app (sid 0x07), not EvenHub. `eventType` carries
|
||||||
|
* eEvenAIStatus -- see EvenAIStatus in app/g2/events.ts. The only one we
|
||||||
|
* act on is EVEN_AI_WAKE_UP, i.e. the "Hey Even" wakeword.
|
||||||
|
*/
|
||||||
|
kind: "even-ai";
|
||||||
|
containerName: string;
|
||||||
|
eventType: number;
|
||||||
|
eventSource: number;
|
||||||
|
systemExitReasonCode: number;
|
||||||
|
frameId: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
function nonNegativeNumber(value: number): number {
|
function nonNegativeNumber(value: number): number {
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ export type PushToTalkOptions = {
|
|||||||
provider: VoiceProviderKind;
|
provider: VoiceProviderKind;
|
||||||
elevenLabsApiKey: string;
|
elevenLabsApiKey: string;
|
||||||
saveRecording: boolean;
|
saveRecording: boolean;
|
||||||
|
/**
|
||||||
|
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||||
|
* ("Hey Even") capture, which has no button release to end the utterance.
|
||||||
|
* Ignored when the mic is already running for another holder.
|
||||||
|
*/
|
||||||
|
endpointing?: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Who currently wants the mic running. */
|
/** Who currently wants the mic running. */
|
||||||
@@ -34,6 +40,7 @@ export class FaceclawVoiceControlBridge {
|
|||||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||||
|
private readonly speechEndListeners = new Set<() => void>();
|
||||||
private controller: any | null = null;
|
private controller: any | null = null;
|
||||||
private listenerProxy: any | null = null;
|
private listenerProxy: any | null = null;
|
||||||
private status = "Voice control stopped.";
|
private status = "Voice control stopped.";
|
||||||
@@ -62,6 +69,15 @@ export class FaceclawVoiceControlBridge {
|
|||||||
return () => this.transcriptListeners.delete(listener);
|
return () => this.transcriptListeners.delete(listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The speaker stopped, in a hands-free session. Only fires when the capture
|
||||||
|
* was started with `endpointing: true`.
|
||||||
|
*/
|
||||||
|
onSpeechEnd(listener: () => void): () => void {
|
||||||
|
this.speechEndListeners.add(listener);
|
||||||
|
return () => this.speechEndListeners.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
||||||
startPushToTalk(options: PushToTalkOptions): void {
|
startPushToTalk(options: PushToTalkOptions): void {
|
||||||
this.acquireCapture("ptt", options);
|
this.acquireCapture("ptt", options);
|
||||||
@@ -93,6 +109,7 @@ export class FaceclawVoiceControlBridge {
|
|||||||
this.ensureController();
|
this.ensureController();
|
||||||
this.controller?.setCommunicator(options.communicator);
|
this.controller?.setCommunicator(options.communicator);
|
||||||
this.controller?.setSaveRecordings(options.saveRecording);
|
this.controller?.setSaveRecordings(options.saveRecording);
|
||||||
|
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||||
|
|
||||||
if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) {
|
if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) {
|
||||||
this.cloudClient = new ElevenLabsSttClient({
|
this.cloudClient = new ElevenLabsSttClient({
|
||||||
@@ -174,6 +191,11 @@ export class FaceclawVoiceControlBridge {
|
|||||||
onPcm: (pcm: any) => {
|
onPcm: (pcm: any) => {
|
||||||
this.cloudClient?.acceptPcm(toUint8Array(pcm));
|
this.cloudClient?.acceptPcm(toUint8Array(pcm));
|
||||||
},
|
},
|
||||||
|
onSpeechEnd: () => {
|
||||||
|
for (const listener of this.speechEndListeners) {
|
||||||
|
listener();
|
||||||
|
}
|
||||||
|
},
|
||||||
});
|
});
|
||||||
this.controller.setListener(this.listenerProxy);
|
this.controller.setListener(this.listenerProxy);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,6 +37,10 @@ type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
|
|||||||
*
|
*
|
||||||
* When opened from a menu instead of a held button (finishOnClick), there is
|
* When opened from a menu instead of a held button (finishOnClick), there is
|
||||||
* no release event to end the capture, so a single click stops the mic.
|
* no release event to end the capture, so a single click stops the mic.
|
||||||
|
*
|
||||||
|
* When opened by the "Hey Even" wakeword (handsFree), there is likewise no
|
||||||
|
* button involved: the mic starts immediately and stops when the speaker does,
|
||||||
|
* via endpoint detection on the decoded PCM. A click still ends it early.
|
||||||
*/
|
*/
|
||||||
export class VoiceInputLayer implements Layer {
|
export class VoiceInputLayer implements Layer {
|
||||||
private phase: VoicePhase = "capturing";
|
private phase: VoicePhase = "capturing";
|
||||||
@@ -56,12 +60,14 @@ export class VoiceInputLayer implements Layer {
|
|||||||
private menuIndex = 0;
|
private menuIndex = 0;
|
||||||
private unsubscribeTranscript: (() => void) | null = null;
|
private unsubscribeTranscript: (() => void) | null = null;
|
||||||
private unsubscribeStatus: (() => void) | null = null;
|
private unsubscribeStatus: (() => void) | null = null;
|
||||||
|
private unsubscribeSpeechEnd: (() => void) | null = null;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly actions: LayerActions,
|
private readonly actions: LayerActions,
|
||||||
private readonly onClosed: () => void,
|
private readonly onClosed: () => void,
|
||||||
private readonly onSend: (text: string) => void,
|
private readonly onSend: (text: string) => void,
|
||||||
private readonly finishOnClick = false,
|
private readonly finishOnClick = false,
|
||||||
|
private readonly handsFree = false,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
startCapture(): void {
|
startCapture(): void {
|
||||||
@@ -73,12 +79,31 @@ export class VoiceInputLayer implements Layer {
|
|||||||
this.status = state.status;
|
this.status = state.status;
|
||||||
this.actions.requestRender();
|
this.actions.requestRender();
|
||||||
});
|
});
|
||||||
|
if (this.handsFree) {
|
||||||
|
// No button is held, so the mic has to stop itself. endCapture() is
|
||||||
|
// idempotent, and a click still ends the utterance early.
|
||||||
|
this.unsubscribeSpeechEnd = voiceControlBridge.onSpeechEnd(() => {
|
||||||
|
if (this.phase === "capturing") {
|
||||||
|
this.endCapture();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
this.capturing = true;
|
this.capturing = true;
|
||||||
void this.actions.startVoiceCapture();
|
void this.actions.startVoiceCapture(this.handsFree);
|
||||||
this.actions.requestRender();
|
this.actions.requestRender();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Button released (or click in finishOnClick mode): stop the mic. */
|
/**
|
||||||
|
* Whether a single click ends the utterance. True when there is no held
|
||||||
|
* button to release: opened from the overlay menu (finishOnClick), or
|
||||||
|
* hands-free from the wakeword — where it also lets the user cut the
|
||||||
|
* silence-detection wait short.
|
||||||
|
*/
|
||||||
|
private get clickEndsCapture(): boolean {
|
||||||
|
return this.finishOnClick || this.handsFree;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Button released (or click when clickEndsCapture): stop the mic. */
|
||||||
endCapture(): void {
|
endCapture(): void {
|
||||||
if (!this.capturing) return;
|
if (!this.capturing) return;
|
||||||
if (this.phase === "continuing") {
|
if (this.phase === "continuing") {
|
||||||
@@ -150,7 +175,7 @@ export class VoiceInputLayer implements Layer {
|
|||||||
case "capturing":
|
case "capturing":
|
||||||
if (event.type === "double-click") {
|
if (event.type === "double-click") {
|
||||||
ctx.stack.pop();
|
ctx.stack.pop();
|
||||||
} else if (this.finishOnClick && event.type === "click") {
|
} else if (this.clickEndsCapture && event.type === "click") {
|
||||||
this.endCapture();
|
this.endCapture();
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -292,6 +317,8 @@ export class VoiceInputLayer implements Layer {
|
|||||||
this.unsubscribeTranscript = null;
|
this.unsubscribeTranscript = null;
|
||||||
this.unsubscribeStatus?.();
|
this.unsubscribeStatus?.();
|
||||||
this.unsubscribeStatus = null;
|
this.unsubscribeStatus = null;
|
||||||
|
this.unsubscribeSpeechEnd?.();
|
||||||
|
this.unsubscribeSpeechEnd = null;
|
||||||
if (this.followupFinalizeTimer !== null) {
|
if (this.followupFinalizeTimer !== null) {
|
||||||
clearTimeout(this.followupFinalizeTimer);
|
clearTimeout(this.followupFinalizeTimer);
|
||||||
this.followupFinalizeTimer = null;
|
this.followupFinalizeTimer = null;
|
||||||
@@ -327,7 +354,7 @@ export class VoiceInputLayer implements Layer {
|
|||||||
private hintText(): string {
|
private hintText(): string {
|
||||||
switch (this.phase) {
|
switch (this.phase) {
|
||||||
case "capturing":
|
case "capturing":
|
||||||
return this.finishOnClick
|
return this.clickEndsCapture
|
||||||
? gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "close"]])
|
? gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "close"]])
|
||||||
: `${GESTURE_DOUBLE_CLICK} close`;
|
: `${GESTURE_DOUBLE_CLICK} close`;
|
||||||
case "continuing":
|
case "continuing":
|
||||||
|
|||||||
+13
-2
@@ -10,6 +10,13 @@ export type DashboardInputEvent =
|
|||||||
| { type: "scroll-down" }
|
| { type: "scroll-down" }
|
||||||
| { type: "long-press"; source: "ring" | "left-arm" | "right-arm" }
|
| { type: "long-press"; source: "ring" | "left-arm" | "right-arm" }
|
||||||
| { type: "long-press-release"; source: "ring" | "left-arm" | "right-arm" }
|
| { type: "long-press-release"; source: "ring" | "left-arm" | "right-arm" }
|
||||||
|
/**
|
||||||
|
* The on-glasses "Hey Even" wakeword fired. Delivered on sid 0x07 by the
|
||||||
|
* stock firmware regardless of CFW; the CFW additionally suppresses the stock
|
||||||
|
* Even AI app so this is ours to handle. Unlike every other input event this
|
||||||
|
* one is honoured while the screen is off.
|
||||||
|
*/
|
||||||
|
| { type: "wakeword" }
|
||||||
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
|
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
|
||||||
|
|
||||||
export type LayerActions = {
|
export type LayerActions = {
|
||||||
@@ -18,8 +25,12 @@ export type LayerActions = {
|
|||||||
disconnect: () => Promise<void> | void;
|
disconnect: () => Promise<void> | void;
|
||||||
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
|
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
|
||||||
endTextSettingEdit: () => Promise<void> | void;
|
endTextSettingEdit: () => Promise<void> | void;
|
||||||
/** Start push-to-talk voice capture with the provider chosen in settings. */
|
/**
|
||||||
startVoiceCapture: () => Promise<void> | void;
|
* Start push-to-talk voice capture with the provider chosen in settings.
|
||||||
|
* With `endpointing`, the capture also ends itself when the speaker stops
|
||||||
|
* (hands-free); otherwise it runs until stopVoiceCapture.
|
||||||
|
*/
|
||||||
|
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
|
||||||
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
|
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
|
||||||
stopVoiceCapture: () => Promise<void> | void;
|
stopVoiceCapture: () => Promise<void> | void;
|
||||||
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
||||||
|
|||||||
+26
-2
@@ -1,5 +1,5 @@
|
|||||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||||
import { EventSourceType, OsEventTypeList } from "../../g2/events";
|
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
|
||||||
import type { RawInputEvent } from "../../native/faceclaw-communicator";
|
import type { RawInputEvent } from "../../native/faceclaw-communicator";
|
||||||
import { DashboardInputEvent, LayerActions, LayerStack } from "../layers";
|
import { DashboardInputEvent, LayerActions, LayerStack } from "../layers";
|
||||||
import { MenuLayer, type MenuItem } from "../menu";
|
import { MenuLayer, type MenuItem } from "../menu";
|
||||||
@@ -320,6 +320,20 @@ class Shell {
|
|||||||
async receiveInput(event: DashboardInputEvent, frameId = 0): Promise<ShellInputOutcome> {
|
async receiveInput(event: DashboardInputEvent, frameId = 0): Promise<ShellInputOutcome> {
|
||||||
this.lastInputAtMs = Date.now();
|
this.lastInputAtMs = Date.now();
|
||||||
|
|
||||||
|
// The wakeword is handled before the screen-off short-circuit: "Hey Even"
|
||||||
|
// is meant to work from a dark screen, which is the whole point of it.
|
||||||
|
// With the CFW the stock Even AI app never launches, so the firmware does
|
||||||
|
// not power the display for us either -- we wake it ourselves.
|
||||||
|
if (event.type === "wakeword") {
|
||||||
|
if (!this.screenOn) {
|
||||||
|
this.wake("sidebar");
|
||||||
|
}
|
||||||
|
if (!this.activeVoiceLayer) {
|
||||||
|
this.openVoiceDialog(false, true);
|
||||||
|
}
|
||||||
|
return { shell: true, window: false };
|
||||||
|
}
|
||||||
|
|
||||||
if (!this.screenOn) {
|
if (!this.screenOn) {
|
||||||
if (event.type === "double-click") {
|
if (event.type === "double-click") {
|
||||||
this.wake("sidebar");
|
this.wake("sidebar");
|
||||||
@@ -422,7 +436,7 @@ class Shell {
|
|||||||
next?.requestRender();
|
next?.requestRender();
|
||||||
}
|
}
|
||||||
|
|
||||||
private openVoiceDialog(finishOnClick = false): void {
|
private openVoiceDialog(finishOnClick = false, handsFree = false): void {
|
||||||
const layer = new VoiceInputLayer(
|
const layer = new VoiceInputLayer(
|
||||||
this.config.actions,
|
this.config.actions,
|
||||||
() => {
|
() => {
|
||||||
@@ -432,6 +446,7 @@ class Shell {
|
|||||||
},
|
},
|
||||||
(text) => this.sendTextToForegroundWindow(text),
|
(text) => this.sendTextToForegroundWindow(text),
|
||||||
finishOnClick,
|
finishOnClick,
|
||||||
|
handsFree,
|
||||||
);
|
);
|
||||||
this.activeVoiceLayer = layer;
|
this.activeVoiceLayer = layer;
|
||||||
this.stack.push(layer);
|
this.stack.push(layer);
|
||||||
@@ -517,6 +532,13 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
|
|||||||
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
|
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
|
||||||
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
|
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
|
||||||
}
|
}
|
||||||
|
} else if (event.kind === "even-ai") {
|
||||||
|
// sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the
|
||||||
|
// wakeword interests us -- ENTER means the user manually opened the stock
|
||||||
|
// assistant, and EXIT is it tearing down.
|
||||||
|
if (event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP) {
|
||||||
|
return { type: "wakeword" };
|
||||||
|
}
|
||||||
} else if (event.kind === "text-click") {
|
} else if (event.kind === "text-click") {
|
||||||
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
|
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
|
||||||
return { type: "scroll-down" };
|
return { type: "scroll-down" };
|
||||||
@@ -557,6 +579,8 @@ export function inputEventToString(event: DashboardInputEvent): string {
|
|||||||
return `Long press from ${event.source}`;
|
return `Long press from ${event.source}`;
|
||||||
case "long-press-release":
|
case "long-press-release":
|
||||||
return `Long press release from ${event.source}`;
|
return `Long press release from ${event.source}`;
|
||||||
|
case "wakeword":
|
||||||
|
return `Wakeword`;
|
||||||
default:
|
default:
|
||||||
case "unknown":
|
case "unknown":
|
||||||
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;
|
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;
|
||||||
|
|||||||
Reference in new issue
Block a user