forked from Blendman974/faceclaw
Add wakeword support
This commit is contained in:
1 parent
4401a4a416
commit
161dbd104e
11 files changed
+315
-13
No files matched your search
@@ -89,6 +89,8 @@ public class FaceclawVoiceController {
|
||||
private long lastTranscriptDecodeAtMs;
|
||||
private String lastTranscript = "";
|
||||
private volatile boolean saveRecordings;
|
||||
private volatile boolean endpointing;
|
||||
private final EndpointDetector endpointDetector = new EndpointDetector();
|
||||
private java.io.ByteArrayOutputStream recordingPcm;
|
||||
private long queuedPackets;
|
||||
private long queueDroppedPackets;
|
||||
@@ -116,6 +118,15 @@ public class FaceclawVoiceController {
|
||||
this.saveRecordings = saveRecordings;
|
||||
}
|
||||
|
||||
/**
|
||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||
* button release to end the utterance. Must be set before {@link #start}.
|
||||
*/
|
||||
public void setEndpointing(boolean endpointing) {
|
||||
this.endpointing = endpointing;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
start("wakeword");
|
||||
}
|
||||
@@ -192,6 +203,7 @@ public class FaceclawVoiceController {
|
||||
stream = keywordSpotter.createStream();
|
||||
}
|
||||
lc3Decoder = new FaceclawLc3Decoder();
|
||||
endpointDetector.reset();
|
||||
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.");
|
||||
@@ -408,6 +420,9 @@ public class FaceclawVoiceController {
|
||||
if (recordingPcm != null) {
|
||||
appendRecording(pcm, count);
|
||||
}
|
||||
if (endpointing && endpointDetector.accept(pcm, count)) {
|
||||
emitSpeechEnd();
|
||||
}
|
||||
if (mode == VoiceInputMode.CLOUD) {
|
||||
emitPcm(pcm, count);
|
||||
} else if (mode == VoiceInputMode.ONBOARD) {
|
||||
@@ -524,6 +539,119 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(() -> currentListener.onPcm(le));
|
||||
}
|
||||
|
||||
private void emitSpeechEnd() {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(currentListener::onSpeechEnd);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides when a hands-free utterance is over, so "Hey Even" capture can
|
||||
* stop without a button release.
|
||||
*
|
||||
* Runs on the decoded 16 kHz PCM, so it works the same in every input mode
|
||||
* (the cloud path never sees the samples on this side, and the onboard
|
||||
* recognizer's own endpointing only covers ONBOARD).
|
||||
*
|
||||
* Timing is measured on the sample clock rather than the wall clock: BLE
|
||||
* delivers mic packets in bursts, so elapsed real time badly overestimates
|
||||
* how much audio has actually been heard.
|
||||
*
|
||||
* The threshold is relative to a noise floor measured over the first
|
||||
* {@link #CALIBRATE_MS} of the session, which is roughly the interval where
|
||||
* the user is reacting to the dialog appearing and not yet speaking.
|
||||
*/
|
||||
private static final class EndpointDetector {
|
||||
/** Audio used to estimate the room's noise floor. */
|
||||
private static final int CALIBRATE_MS = 300;
|
||||
/** Speech must exceed this multiple of the noise floor to count as onset. */
|
||||
private static final double ONSET_FACTOR = 3.0;
|
||||
/** Below this multiple of the noise floor counts as silence again. */
|
||||
private static final double RELEASE_FACTOR = 1.8;
|
||||
/** Absolute floor, so a silent room can't make the threshold ~0. */
|
||||
private static final double MIN_RMS = 220.0;
|
||||
/** Trailing silence that ends an utterance. */
|
||||
private static final int SILENCE_MS = 900;
|
||||
/** If the user never speaks, give up rather than record forever. */
|
||||
private static final int LEAD_IN_MS = 6000;
|
||||
/** Hard cap on a single utterance. */
|
||||
private static final int MAX_UTTERANCE_MS = 30000;
|
||||
|
||||
private long totalSamples;
|
||||
private double noiseAccum;
|
||||
private int noisePackets;
|
||||
private double threshold;
|
||||
private boolean speechStarted;
|
||||
private long silenceSamples;
|
||||
private boolean fired;
|
||||
|
||||
void reset() {
|
||||
totalSamples = 0;
|
||||
noiseAccum = 0;
|
||||
noisePackets = 0;
|
||||
threshold = 0;
|
||||
speechStarted = false;
|
||||
silenceSamples = 0;
|
||||
fired = false;
|
||||
}
|
||||
|
||||
/** Returns true exactly once, on the packet that ends the utterance. */
|
||||
boolean accept(short[] pcm, int count) {
|
||||
if (fired || count <= 0) {
|
||||
return false;
|
||||
}
|
||||
totalSamples += count;
|
||||
long elapsedMs = totalSamples * 1000L / SAMPLE_RATE;
|
||||
|
||||
double sumSquares = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
double s = pcm[i];
|
||||
sumSquares += s * s;
|
||||
}
|
||||
double rms = Math.sqrt(sumSquares / count);
|
||||
|
||||
if (elapsedMs <= CALIBRATE_MS) {
|
||||
noiseAccum += rms;
|
||||
noisePackets++;
|
||||
return false;
|
||||
}
|
||||
if (threshold == 0) {
|
||||
double noiseFloor = noisePackets > 0 ? noiseAccum / noisePackets : 0;
|
||||
threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
}
|
||||
|
||||
if (!speechStarted) {
|
||||
if (rms >= threshold * ONSET_FACTOR) {
|
||||
speechStarted = true;
|
||||
silenceSamples = 0;
|
||||
} else if (elapsedMs >= LEAD_IN_MS) {
|
||||
// Never heard anything; close the dialog rather than hang.
|
||||
fired = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rms < threshold * RELEASE_FACTOR) {
|
||||
silenceSamples += count;
|
||||
if (silenceSamples * 1000L / SAMPLE_RATE >= SILENCE_MS) {
|
||||
fired = true;
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
silenceSamples = 0;
|
||||
}
|
||||
|
||||
if (elapsedMs >= MAX_UTTERANCE_MS) {
|
||||
fired = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void stopG2Audio() {
|
||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||
if (currentCommunicator != null) {
|
||||
|
||||
+7
@@ -17,4 +17,11 @@ public interface FaceclawVoiceControllerListener {
|
||||
* little-endian PCM. Empty/absent in onboard mode.
|
||||
*/
|
||||
void onPcm(byte[] pcm16le);
|
||||
|
||||
/**
|
||||
* The speaker stopped, in a hands-free session that has no button release
|
||||
* to end it. Only fires when endpointing was enabled for the session; see
|
||||
* FaceclawVoiceController.setEndpointing.
|
||||
*/
|
||||
void onSpeechEnd();
|
||||
}
|
||||
@@ -31,6 +31,20 @@ public class BleProtocol {
|
||||
public static final int SID_APP_LAUNCH = 0x01;
|
||||
public static final int SID_EVENHUB = 0xe0;
|
||||
public static final int SID_UI_SETTING = 0x09;
|
||||
// UI_FOREGROUND_EVEN_AI_ID: the stock "Even AI" assistant app. It is a
|
||||
// FOREGROUND app, whereas EvenHub (and therefore faceclaw) is the
|
||||
// BACKGROUND app -- which is why an Even AI launch draws over us.
|
||||
// Payload is g2.even_ai.EvenAIDataPackage (see
|
||||
// g2-kit-unofficial/ble/gen/even_ai_pb.ts).
|
||||
public static final int SID_EVEN_AI = 0x07;
|
||||
// eEvenAICommandId
|
||||
public static final int EVEN_AI_CMD_CTRL = 1;
|
||||
// eEvenAIStatus, carried in EvenAIDataPackage.ctrl.status (field 3, sub 1).
|
||||
// WAKE_UP is the on-glasses "Hey Even" wakeword firing; ENTER is a manual
|
||||
// entry (touch/double-tap), so the two are distinguishable.
|
||||
public static final int EVEN_AI_STATUS_WAKE_UP = 1;
|
||||
public static final int EVEN_AI_STATUS_ENTER = 2;
|
||||
public static final int EVEN_AI_STATUS_EXIT = 3;
|
||||
public static final int FLAG_REQUEST = 0x20;
|
||||
public static final int FLAG_NOTIFY = 0x01;
|
||||
public static final int FLAG_NOTIFY_ALT = 0x06;
|
||||
|
||||
@@ -32,6 +32,29 @@ public final class G2Event {
|
||||
|
||||
public static G2Event decode(BleProtocol.ParsedFrame frame) {
|
||||
byte[] pb = BleProtocol.stripTrailingCrc(frame.pb);
|
||||
|
||||
// sid 0x07 is the stock Even AI app, not EvenHub, so it carries an
|
||||
// EvenAIDataPackage rather than the device-event wrapper below.
|
||||
// EvenAIDataPackage { command_id = 1, magic_random = 2, ctrl = 3 },
|
||||
// EvenAIControl { status = 1 }. The event we care about is
|
||||
// command_id=CTRL with status=WAKE_UP, which the firmware emits when the
|
||||
// on-glasses "Hey Even" wakeword fires.
|
||||
if (frame.sid == BleProtocol.SID_EVEN_AI) {
|
||||
int commandId = BleProtocol.readVarintFieldValue(pb, 1, -1);
|
||||
if (commandId != BleProtocol.EVEN_AI_CMD_CTRL) {
|
||||
return null;
|
||||
}
|
||||
byte[] ctrl = BleProtocol.readFieldBytes(pb, 3);
|
||||
if (ctrl == null) {
|
||||
return null;
|
||||
}
|
||||
int status = BleProtocol.readVarintFieldValue(ctrl, 1, -1);
|
||||
if (status < 0) {
|
||||
return null;
|
||||
}
|
||||
return new G2Event("even-ai", "", status, 0, 0);
|
||||
}
|
||||
|
||||
byte[] deviceEvent = BleProtocol.readFieldBytes(pb, 13);
|
||||
if (deviceEvent == null) {
|
||||
return null;
|
||||
|
||||
Reference in new issue
Block a user