forked from Blendman974/faceclaw
Refactoring
This commit is contained in:
1 parent
831e927dc8
commit
e853f0ea5d
7 files changed
+211
-123
No files matched your search
@@ -97,6 +97,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
private int consecutiveAckTimeouts;
|
||||
private int lastAudioControlAckMagic = 0;
|
||||
|
||||
private ConnectionOptions connectionOptions = new ConnectionOptions();
|
||||
private final BleMagicPool magicPool = new BleMagicPool();
|
||||
private MessageBuilder messageBuilder = new MessageBuilder(magicPool);
|
||||
private int nextMapSessionId = 0;
|
||||
@@ -156,13 +157,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
running = false;
|
||||
audioCaptureActive = false;
|
||||
audioPacketListener = null;
|
||||
phase = "disconnecting";
|
||||
status = "Disconnecting...";
|
||||
lock.notifyAll();
|
||||
threadToJoin = workerThread;
|
||||
}
|
||||
setStateDisplay("disconnecting", "Disconnecting...");
|
||||
interruptibleSleep.interrupt();
|
||||
emitState();
|
||||
if (threadToJoin != null) {
|
||||
threadToJoin.interrupt();
|
||||
try {
|
||||
@@ -173,15 +171,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
synchronized (lock) {
|
||||
workerThread = null;
|
||||
phase = "disconnected";
|
||||
status = "Disconnected.";
|
||||
resetSessionStateLocked();
|
||||
clearAllMessagesLocked("disconnect");
|
||||
}
|
||||
bleManager.disconnect(rightAddress);
|
||||
bleManager.disconnect(leftAddress);
|
||||
bleManager.close();
|
||||
emitState();
|
||||
setStateDisplay("disconnected", "Disconnected.");
|
||||
}
|
||||
|
||||
public void close() {
|
||||
@@ -203,7 +199,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
magic = message.magic;
|
||||
pendingMessages.addFirst(message);
|
||||
logLine("queue G2 mic enable");
|
||||
lock.notifyAll();
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
return waitForAudioControlAck(magic, "enable");
|
||||
@@ -220,7 +215,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
magic = message.magic;
|
||||
pendingMessages.addFirst(message);
|
||||
logLine("queue G2 mic disable");
|
||||
lock.notifyAll();
|
||||
}
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
@@ -293,7 +287,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
};
|
||||
pendingMessages.addFirst(message);
|
||||
logLine("queue shutdown");
|
||||
lock.notifyAll();
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
|
||||
@@ -435,7 +428,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
}
|
||||
lock.notifyAll();
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
if (event != null) {
|
||||
@@ -483,19 +475,18 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
displayedFingerprint = "";
|
||||
displayedTileBmps = emptyTileSet();
|
||||
reconnectAfterMs = SystemClock.elapsedRealtime() + RECONNECT_DELAY_MS;
|
||||
phase = userDisconnectRequested ? "disconnecting" : "connecting";
|
||||
status = userDisconnectRequested
|
||||
? "Disconnecting..."
|
||||
: "Connection lost. Reconnecting...";
|
||||
}
|
||||
lock.notifyAll();
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
emitState();
|
||||
if (connected) {
|
||||
setStateDisplay("connected", "Connected.");
|
||||
} else {
|
||||
setStateDisplay("connecting", "Connecting to the glasses...");
|
||||
}
|
||||
}
|
||||
|
||||
private void connectLoopOnce() throws InterruptedException { //{{{
|
||||
setState("connecting", "Connecting to the glasses...");
|
||||
setStateDisplay("connecting", "Connecting to the glasses...");
|
||||
try {
|
||||
connectArm(rightAddress, true);
|
||||
connectArm(leftAddress, true);
|
||||
@@ -523,7 +514,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
lastAudioControlAckMagic = 0;
|
||||
audioCaptureActive = false;
|
||||
}
|
||||
setState("connected", "Connected.");
|
||||
setStateDisplay("connected", "Connected.");
|
||||
logLine("session ready");
|
||||
} catch (Throwable t) {
|
||||
logLine("connect failed: " + safeMessage(t));
|
||||
@@ -585,9 +576,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
handleTransportFailure("ack timeout");
|
||||
};
|
||||
prelude.sentAtMs = now;
|
||||
synchronized (lock) {
|
||||
inFlightMessages.add(prelude);
|
||||
}
|
||||
writeMessage(prelude);
|
||||
|
||||
long deadline = SystemClock.elapsedRealtime() + PRELUDE_TIMEOUT_MS;
|
||||
@@ -647,7 +635,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
OutboundMessage pending = pendingMessages.peekFirst();
|
||||
messageToWrite = pendingMessages.removeFirst();
|
||||
Log.i(TAG, "sending pending message: " + messageToWrite.label);
|
||||
prepareMessageWriteLocked(messageToWrite, now);
|
||||
} else if (!shutdownRequested && fixedLayoutCreated && warmedUp && pendingMessages.isEmpty() && inFlightMessages.isEmpty()
|
||||
&& now >= imageRetryAfterMs
|
||||
&& !getDesiredFingerprint().equals(displayedFingerprint)) {
|
||||
@@ -657,7 +644,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
} else if (shouldPollBatteryLocked(now)) {
|
||||
Log.i(TAG, "Writing battery query");
|
||||
messageToWrite = createBatteryQueryMessageLocked();
|
||||
prepareMessageWriteLocked(messageToWrite, now);
|
||||
lastBatteryRefreshAtMs = now;
|
||||
} else if (!pendingMessages.isEmpty() || !inFlightMessages.isEmpty()) {
|
||||
return IDLE_SLEEP_MS;
|
||||
@@ -674,10 +660,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
synchronized (lock) {
|
||||
completeMessageWriteLocked(messageToWrite);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -692,8 +674,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
|
||||
if (heartbeatReady && !heartbeatPending && inFlightMessages.isEmpty()) {
|
||||
Log.i(TAG, "Writing heartbeat");
|
||||
OutboundMessage heartbeatMessage = createHeartbeatMessageLocked();
|
||||
prepareMessageWriteLocked(heartbeatMessage, now);
|
||||
OutboundMessage heartbeatMessage = createHeartbeatMessage();
|
||||
lastHeartbeatSentAtMs = now;
|
||||
writeMessage(heartbeatMessage);
|
||||
return true;
|
||||
@@ -708,15 +689,21 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
return false;
|
||||
}
|
||||
|
||||
private OutboundMessage createHeartbeatMessageLocked() {
|
||||
private OutboundMessage createHeartbeatMessage() {
|
||||
OutboundMessage message = messageBuilder.heartbeat();
|
||||
message.onAck = () -> {
|
||||
lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime();
|
||||
synchronized (lock) {
|
||||
lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime();
|
||||
}
|
||||
};
|
||||
message.onTimeout = () -> {
|
||||
// If a heartbeat fails to ack and we're over the heartbeat deadline, assume the connection is failed and reconnect.
|
||||
// Otherwise ignore it, which will cause a retransmission attempt.
|
||||
if (SystemClock.elapsedRealtime() - lastHeartbeatSentAtMs >= HEARTBEAT_FAILURE_DEADLINE_MS) {
|
||||
boolean isPastDeadline;
|
||||
synchronized (lock) {
|
||||
isPastDeadline = SystemClock.elapsedRealtime() - lastHeartbeatSentAtMs >= HEARTBEAT_FAILURE_DEADLINE_MS;
|
||||
}
|
||||
if (isPastDeadline) {
|
||||
handleTransportFailure("heartbeat ack timeout");
|
||||
}
|
||||
};
|
||||
@@ -724,41 +711,42 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
|
||||
|
||||
private void prepareMessageWriteLocked(OutboundMessage message, long now) {
|
||||
private boolean writeMessage(OutboundMessage message) {
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
message.writeStartedAtMs = now;
|
||||
message.sentAtMs = now;
|
||||
message.ackDeadlineAtMs = now + message.ackTimeoutMs + WRITE_TIMEOUT_MS;
|
||||
if (message.magic != 0) {
|
||||
inFlightMessages.addLast(message);
|
||||
synchronized (lock) {
|
||||
inFlightMessages.addLast(message);
|
||||
}
|
||||
}
|
||||
if (message.imageUpdateId > 0 && message.imageMessageNumber == 1) {
|
||||
BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(message.imageUpdateId);
|
||||
if (stats != null && stats.firstWriteStartedAtMs <= 0) {
|
||||
stats.firstWriteStartedAtMs = now;
|
||||
synchronized(lock) {
|
||||
BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(message.imageUpdateId);
|
||||
if (stats != null && stats.firstWriteStartedAtMs <= 0) {
|
||||
stats.firstWriteStartedAtMs = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean writeMessage(OutboundMessage message) {
|
||||
String writeAddress = isLeftArmMessage(message) ? leftAddress : rightAddress;
|
||||
return bleManager.writeFrames(
|
||||
String writeAddress = message.isLeftArmMessage ? leftAddress : rightAddress;
|
||||
boolean result = bleManager.writeFrames(
|
||||
writeAddress,
|
||||
BleProtocol.WRITE_CHAR_UUID,
|
||||
message.frames,
|
||||
WRITE_TYPE,
|
||||
WRITE_TIMEOUT_MS
|
||||
);
|
||||
}
|
||||
|
||||
private boolean isLeftArmMessage(OutboundMessage message) {
|
||||
return message != null && ("image".equals(message.kind) || "warmup".equals(message.kind));
|
||||
}
|
||||
synchronized (lock) {
|
||||
long sentAtMs = SystemClock.elapsedRealtime();
|
||||
message.sentAtMs = sentAtMs;
|
||||
message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs;
|
||||
logImageUpdateSendLandmarkLocked(message);
|
||||
}
|
||||
|
||||
private void completeMessageWriteLocked(OutboundMessage message) {
|
||||
long sentAtMs = SystemClock.elapsedRealtime();
|
||||
message.sentAtMs = sentAtMs;
|
||||
message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs;
|
||||
logImageUpdateSendLandmarkLocked(message);
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean removePreparedMessageLocked(OutboundMessage message) {
|
||||
@@ -781,20 +769,24 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
while (iterator.hasNext()) {
|
||||
OutboundMessage message = iterator.next();
|
||||
if (message.sid == sid && message.magic == magic) {
|
||||
Log.i(TAG, "Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")");
|
||||
iterator.remove();
|
||||
message.ackPayload = pb == null ? new byte[0] : Arrays.copyOf(pb, pb.length);
|
||||
magicPool.release(message.sid, message.magic, message.label, "ack");
|
||||
if (message.onAck != null) {
|
||||
message.onAck.run();
|
||||
}
|
||||
consecutiveAckTimeouts = 0;
|
||||
resolveAckLocked(message, pb);
|
||||
return;
|
||||
}
|
||||
}
|
||||
recordUnexpectedAckLocked(sid, magic);
|
||||
}
|
||||
|
||||
private void resolveAckLocked(OutboundMessage message, byte[] pb) {
|
||||
Log.i(TAG, "Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")");
|
||||
inFlightMessages.remove(message);
|
||||
message.ackPayload = pb == null ? new byte[0] : Arrays.copyOf(pb, pb.length);
|
||||
magicPool.release(message.sid, message.magic, message.label, "ack");
|
||||
if (message.onAck != null) {
|
||||
message.onAck.run();
|
||||
}
|
||||
consecutiveAckTimeouts = 0;
|
||||
}
|
||||
|
||||
private void logImageUpdateSendLandmarkLocked(OutboundMessage message) {
|
||||
if (message.imageUpdateId <= 0) {
|
||||
return;
|
||||
@@ -891,7 +883,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
int sessionId = nextMapSessionId();
|
||||
List<BleProtocol.ImageFragment> fragments = BleImageOptimizer.planImageFragments(bmp, IMAGE_FRAGMENT_SIZE);
|
||||
for (BleProtocol.ImageFragment fragment : fragments) {
|
||||
OutboundMessage message = messageBuilder.imageWarmupFragment(tile, sessionId, fragment, bmp);
|
||||
OutboundMessage message = messageBuilder.imageWarmupFragment(tile, sessionId, fragment, bmp, connectionOptions.sendImagesToLeft);
|
||||
pendingMessages.addLast(message);
|
||||
message.onAck = () -> {
|
||||
// The first tile warmup primes the image path but does not reliably
|
||||
@@ -992,13 +984,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
boolean requestAck,
|
||||
boolean isLast
|
||||
) {
|
||||
OutboundMessage message = messageBuilder.imageFragment(fragment, plan, requestAck);
|
||||
OutboundMessage message = messageBuilder.imageFragment(fragment, plan, requestAck, connectionOptions.sendImagesToLeft);
|
||||
message.setImageUpdatePosition(updateId, messageNumber, messageCount);
|
||||
message.onAck = () -> {
|
||||
imageRetryAfterMs = 0;
|
||||
logImageUpdateAckLandmarkLocked(message);
|
||||
if (message.tileIndex >= 0 && !hasPendingOrInflightTileLocked(message.tileIndex)) {
|
||||
displayedTileBmps[message.tileIndex] = BmpUtil.copyTileBmp(message.tileBmp);
|
||||
displayedTileBmps[message.tileIndex] = BmpUtil.copyTileBmp(plan.bmp);
|
||||
}
|
||||
boolean imageStillInFlight = false;
|
||||
for (OutboundMessage inFlight : inFlightMessages) {
|
||||
@@ -1203,16 +1195,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
displayedTileBmps = emptyTileSet();
|
||||
clearAllMessagesLocked("transport failure: " + reason);
|
||||
reconnectAfterMs = SystemClock.elapsedRealtime() + RECONNECT_DELAY_MS;
|
||||
if (!userDisconnectRequested) {
|
||||
phase = "retrying";
|
||||
status = reason == null || reason.isEmpty() ? "Reconnecting..." : "Reconnecting after " + reason;
|
||||
}
|
||||
lock.notifyAll();
|
||||
interruptibleSleep.interrupt();
|
||||
bleManager.disconnect(rightAddress);
|
||||
bleManager.disconnect(leftAddress);
|
||||
emitState();
|
||||
}
|
||||
if (!userDisconnectRequested) {
|
||||
setStateDisplay("retrying", reason == null || reason.isEmpty() ? "Reconnecting..." : "Reconnecting after " + reason);
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
}
|
||||
|
||||
private void resetSessionStateLocked() {
|
||||
@@ -1324,7 +1313,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
});
|
||||
}
|
||||
|
||||
private void setState(String nextPhase, String nextStatus) {
|
||||
private void setStateDisplay(String nextPhase, String nextStatus) {
|
||||
synchronized (lock) {
|
||||
phase = nextPhase;
|
||||
status = nextStatus;
|
||||
@@ -1374,7 +1363,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
|
||||
private int nextMapSessionId() {
|
||||
int id = nextMapSessionId;
|
||||
nextMapSessionId = (nextMapSessionId + 1) & 0xff;
|
||||
int increment = connectionOptions.skipSessionIds ? 2 : 1;
|
||||
nextMapSessionId = (nextMapSessionId + increment) & 0xff;
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
+122
-34
@@ -11,10 +11,13 @@ import com.k2fsa.sherpa.onnx.FeatureConfig;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotter;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineStream;
|
||||
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OnlineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OnlineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OnlineStream;
|
||||
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
|
||||
|
||||
@@ -23,6 +26,7 @@ import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Arrays;
|
||||
|
||||
public class FaceclawVoiceController {
|
||||
private static final String TAG = "FaceclawVoice";
|
||||
@@ -32,10 +36,16 @@ public class FaceclawVoiceController {
|
||||
private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
|
||||
private static final int LATE_PACKET_INTERVAL_MS = 90;
|
||||
private static final int STATS_INTERVAL_MS = 5_000;
|
||||
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 900;
|
||||
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 2;
|
||||
private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 8;
|
||||
private static final int TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES = (int) (SAMPLE_RATE * 1.2);
|
||||
private static final float TRANSCRIPT_SILENCE_PEAK = 0.015f;
|
||||
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
|
||||
private static final String ASSET_ROOT = "faceclaw-voice";
|
||||
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
||||
private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr";
|
||||
private static final String ASR_MODEL_DIR = "sherpa-onnx-streaming-zipformer-en-2023-06-26";
|
||||
private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27";
|
||||
private static final String[] MODEL_FILES = {
|
||||
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
@@ -44,9 +54,8 @@ public class FaceclawVoiceController {
|
||||
"screen-on-keywords.txt"
|
||||
};
|
||||
private static final String[] ASR_MODEL_FILES = {
|
||||
"encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
|
||||
"decoder-epoch-99-avg-1-chunk-16-left-128.onnx",
|
||||
"joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
|
||||
"encoder_model.ort",
|
||||
"decoder_model_merged.ort",
|
||||
"tokens.txt"
|
||||
};
|
||||
|
||||
@@ -66,10 +75,16 @@ public class FaceclawVoiceController {
|
||||
private volatile boolean started;
|
||||
private VoiceInputMode mode = VoiceInputMode.WAKEWORD;
|
||||
private KeywordSpotter keywordSpotter;
|
||||
private OnlineRecognizer recognizer;
|
||||
private OfflineRecognizer recognizer;
|
||||
private OnlineStream stream;
|
||||
private FaceclawLc3Decoder lc3Decoder;
|
||||
private float[] transcriptSamples = new float[TRANSCRIPT_MAX_SAMPLES];
|
||||
private int transcriptSampleCount;
|
||||
private int transcriptSilentSamples;
|
||||
private boolean transcriptHadSpeech;
|
||||
private long lastTranscriptDecodeAtMs;
|
||||
private String lastTranscript = "";
|
||||
private float lastTranscriptPeak;
|
||||
private long queuedPackets;
|
||||
private long queueDroppedPackets;
|
||||
private long decodedSamples;
|
||||
@@ -153,8 +168,8 @@ public class FaceclawVoiceController {
|
||||
: "Voice control loading wake-word model...");
|
||||
if (currentMode == VoiceInputMode.FULL) {
|
||||
File modelDir = installAsrModelFiles();
|
||||
recognizer = new OnlineRecognizer(buildRecognizerConfig(modelDir));
|
||||
stream = recognizer.createStream();
|
||||
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
} else {
|
||||
File modelDir = installModelFiles();
|
||||
@@ -208,23 +223,20 @@ public class FaceclawVoiceController {
|
||||
.build();
|
||||
}
|
||||
|
||||
private OnlineRecognizerConfig buildRecognizerConfig(File modelDir) {
|
||||
return OnlineRecognizerConfig.builder()
|
||||
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) {
|
||||
return OfflineRecognizerConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setOnlineModelConfig(OnlineModelConfig.builder()
|
||||
.setTransducer(OnlineTransducerModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath())
|
||||
.setDecoder(new File(modelDir, "decoder-epoch-99-avg-1-chunk-16-left-128.onnx").getAbsolutePath())
|
||||
.setJoiner(new File(modelDir, "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath())
|
||||
.setModelConfig(OfflineModelConfig.builder()
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setModelType("zipformer2")
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setEnableEndpoint(true)
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -291,7 +303,7 @@ public class FaceclawVoiceController {
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
OnlineStream currentStream = stream;
|
||||
FaceclawLc3Decoder currentDecoder = lc3Decoder;
|
||||
if (currentStream == null || currentDecoder == null) {
|
||||
if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -310,11 +322,10 @@ public class FaceclawVoiceController {
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
|
||||
if (mode == VoiceInputMode.FULL) {
|
||||
processRecognizer(currentStream);
|
||||
processRecognizer(samples);
|
||||
} else {
|
||||
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
processKeywordSpotter(currentStream);
|
||||
}
|
||||
maybeEmitAudioStats(false);
|
||||
@@ -337,28 +348,96 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
}
|
||||
|
||||
private void processRecognizer(OnlineStream currentStream) {
|
||||
OnlineRecognizer currentRecognizer = recognizer;
|
||||
if (currentRecognizer == null) {
|
||||
private void processRecognizer(float[] samples) {
|
||||
appendTranscriptSamples(samples);
|
||||
updateTranscriptSilence(samples);
|
||||
|
||||
if (transcriptSampleCount >= TRANSCRIPT_MAX_SAMPLES) {
|
||||
decodeTranscriptSegment(true);
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
return;
|
||||
}
|
||||
while (currentRecognizer.isReady(currentStream)) {
|
||||
currentRecognizer.decode(currentStream);
|
||||
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
|
||||
decodeTranscriptSegment(false);
|
||||
lastTranscriptDecodeAtMs = now;
|
||||
}
|
||||
emitRecognizerResult(currentRecognizer, currentStream, false);
|
||||
if (currentRecognizer.isEndpoint(currentStream)) {
|
||||
emitRecognizerResult(currentRecognizer, currentStream, true);
|
||||
currentRecognizer.reset(currentStream);
|
||||
|
||||
if (transcriptHadSpeech && transcriptSilentSamples >= TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES) {
|
||||
decodeTranscriptSegment(true);
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
}
|
||||
}
|
||||
|
||||
private void emitRecognizerResult(OnlineRecognizer currentRecognizer, OnlineStream currentStream, boolean isFinal) {
|
||||
OnlineRecognizerResult result = currentRecognizer.getResult(currentStream);
|
||||
private void appendTranscriptSamples(float[] samples) {
|
||||
int available = TRANSCRIPT_MAX_SAMPLES - transcriptSampleCount;
|
||||
if (samples.length > available) {
|
||||
int drop = samples.length - available;
|
||||
if (drop >= transcriptSampleCount) {
|
||||
transcriptSampleCount = 0;
|
||||
} else {
|
||||
System.arraycopy(transcriptSamples, drop, transcriptSamples, 0, transcriptSampleCount - drop);
|
||||
transcriptSampleCount -= drop;
|
||||
}
|
||||
}
|
||||
System.arraycopy(samples, 0, transcriptSamples, transcriptSampleCount, samples.length);
|
||||
transcriptSampleCount += samples.length;
|
||||
}
|
||||
|
||||
private void updateTranscriptSilence(float[] samples) {
|
||||
float peak = 0.0f;
|
||||
for (float sample : samples) {
|
||||
float abs = Math.abs(sample);
|
||||
if (abs > peak) {
|
||||
peak = abs;
|
||||
}
|
||||
}
|
||||
if (peak > lastTranscriptPeak) {
|
||||
lastTranscriptPeak = peak;
|
||||
}
|
||||
if (peak < TRANSCRIPT_SILENCE_PEAK) {
|
||||
transcriptSilentSamples += samples.length;
|
||||
} else {
|
||||
transcriptSilentSamples = 0;
|
||||
transcriptHadSpeech = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void decodeTranscriptSegment(boolean isFinal) {
|
||||
OfflineRecognizer currentRecognizer = recognizer;
|
||||
if (currentRecognizer == null || transcriptSampleCount <= 0) {
|
||||
return;
|
||||
}
|
||||
OfflineStream offlineStream = currentRecognizer.createStream();
|
||||
try {
|
||||
offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE);
|
||||
currentRecognizer.decode(offlineStream);
|
||||
emitRecognizerResult(currentRecognizer.getResult(offlineStream), isFinal, transcriptSampleCount, lastTranscriptPeak);
|
||||
} finally {
|
||||
offlineStream.release();
|
||||
}
|
||||
}
|
||||
|
||||
private void emitRecognizerResult(OfflineRecognizerResult result, boolean isFinal, int sampleCount, float peak) {
|
||||
String text = result == null ? "" : result.getText();
|
||||
if (text == null) {
|
||||
text = "";
|
||||
}
|
||||
String trimmed = text.trim();
|
||||
String preview = trimmed.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
|
||||
? trimmed
|
||||
: trimmed.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
|
||||
Log.i(TAG, "Moonshine decode final=" + isFinal
|
||||
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", sampleCount / (double) SAMPLE_RATE)
|
||||
+ " peak=" + String.format(java.util.Locale.US, "%.4f", peak)
|
||||
+ " textLen=" + trimmed.length()
|
||||
+ " text=\"" + preview + "\"");
|
||||
emitStatus("Moonshine decode "
|
||||
+ String.format(java.util.Locale.US, "%.1fs", sampleCount / (double) SAMPLE_RATE)
|
||||
+ ", textLen=" + trimmed.length());
|
||||
if (text.equals(lastTranscript)) {
|
||||
if (isFinal && lastTranscript.length() > 0) {
|
||||
emitTranscript("", true);
|
||||
@@ -379,6 +458,14 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
}
|
||||
|
||||
private void resetTranscriptState() {
|
||||
transcriptSampleCount = 0;
|
||||
transcriptSilentSamples = 0;
|
||||
transcriptHadSpeech = false;
|
||||
lastTranscriptDecodeAtMs = 0;
|
||||
lastTranscriptPeak = 0.0f;
|
||||
}
|
||||
|
||||
private void stopG2Audio() {
|
||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||
if (currentCommunicator != null) {
|
||||
@@ -510,6 +597,7 @@ public class FaceclawVoiceController {
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
Log.i(TAG, "Emit transcript final=" + isFinal + " textLen=" + (text == null ? 0 : text.trim().length()));
|
||||
mainHandler.post(() -> currentListener.onTranscript(text, isFinal));
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public class ConnectionOptions {
|
||||
final boolean sendImagesToLeft = true;
|
||||
final boolean skipSessionIds = true;
|
||||
}
|
||||
+13
-13
@@ -20,8 +20,8 @@ public class MessageBuilder {
|
||||
BleProtocol.PRELUDE_ACK_MAGIC,
|
||||
CollectionUtils.singletonList(Arrays.copyOf(BleProtocol.PRELUDE_F5872, BleProtocol.PRELUDE_F5872.length)),
|
||||
ACK_TIMEOUT_MS,
|
||||
0,
|
||||
null
|
||||
-1,
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
@@ -40,11 +40,11 @@ public class MessageBuilder {
|
||||
),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
null
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
public OutboundMessage imageWarmupFragment(BleProtocol.ImageTileOptions tile, int sessionId, BleProtocol.ImageFragment fragment, byte[] bmp) {
|
||||
public OutboundMessage imageWarmupFragment(BleProtocol.ImageTileOptions tile, int sessionId, BleProtocol.ImageFragment fragment, byte[] bmp, boolean leftArm) {
|
||||
int magic = magicPool.allocate();
|
||||
return new OutboundMessage(
|
||||
"warmup",
|
||||
@@ -57,12 +57,12 @@ public class MessageBuilder {
|
||||
nextTransportSeq++
|
||||
),
|
||||
WARMUP_FRAGMENT_TIMEOUT_MS,
|
||||
0,
|
||||
bmp
|
||||
-1,
|
||||
leftArm
|
||||
);
|
||||
}
|
||||
|
||||
public OutboundMessage imageFragment(BleProtocol.ImageFragment fragment, BleImageOptimizer.TileImagePlan plan, boolean requestAck) {
|
||||
public OutboundMessage imageFragment(BleProtocol.ImageFragment fragment, BleImageOptimizer.TileImagePlan plan, boolean requestAck, boolean leftArm) {
|
||||
int magic = requestAck ? magicPool.allocate() : 0;
|
||||
return new OutboundMessage(
|
||||
"image",
|
||||
@@ -76,7 +76,7 @@ public class MessageBuilder {
|
||||
),
|
||||
ACK_TIMEOUT_MS,
|
||||
plan.tileIndex,
|
||||
plan.bmp
|
||||
leftArm
|
||||
);
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ public class MessageBuilder {
|
||||
),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
null
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
@@ -114,7 +114,7 @@ public class MessageBuilder {
|
||||
),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
null
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ public class MessageBuilder {
|
||||
),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
null
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
@@ -147,7 +147,7 @@ public class MessageBuilder {
|
||||
BleProtocol.framePb(BleProtocol.buildHeartbeat(magic), BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, nextTransportSeq++),
|
||||
HEARTBEAT_TIMEOUT_MS,
|
||||
-1,
|
||||
null
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
@@ -166,7 +166,7 @@ public class MessageBuilder {
|
||||
),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
null
|
||||
false
|
||||
);
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -9,7 +9,7 @@ public final class OutboundMessage {
|
||||
final List<byte[]> frames;
|
||||
final int ackTimeoutMs;
|
||||
final int tileIndex;
|
||||
final byte[] tileBmp;
|
||||
final boolean isLeftArmMessage;
|
||||
|
||||
int imageUpdateId;
|
||||
int imageMessageNumber;
|
||||
@@ -21,7 +21,7 @@ public final class OutboundMessage {
|
||||
Runnable onAck;
|
||||
Runnable onTimeout;
|
||||
|
||||
OutboundMessage(String kind, String label, int sid, int magic, List<byte[]> frames, int ackTimeoutMs, int tileIndex, byte[] tileBmp) {
|
||||
OutboundMessage(String kind, String label, int sid, int magic, List<byte[]> frames, int ackTimeoutMs, int tileIndex, boolean isLeftArmMessage) {
|
||||
this.kind = kind;
|
||||
this.label = label;
|
||||
this.sid = sid;
|
||||
@@ -29,7 +29,7 @@ public final class OutboundMessage {
|
||||
this.frames = frames;
|
||||
this.ackTimeoutMs = ackTimeoutMs;
|
||||
this.tileIndex = tileIndex;
|
||||
this.tileBmp = BmpUtil.copyTileBmp(tileBmp);
|
||||
this.isLeftArmMessage = isLeftArmMessage;
|
||||
}
|
||||
|
||||
void setImageUpdatePosition(int updateId, int messageNumber, int messageCount) {
|
||||
|
||||
@@ -606,6 +606,7 @@ class DashboardController {
|
||||
}
|
||||
|
||||
private async renderDashboard(reason: "initial" | "interval"): Promise<void> {
|
||||
console.log("renderDashboard", reason);
|
||||
const paintStartedAtMs = Date.now();
|
||||
const image = drawDashboard();
|
||||
const paintMs = Date.now() - paintStartedAtMs;
|
||||
@@ -617,6 +618,7 @@ class DashboardController {
|
||||
this.updateDisplayPreviewFromImage(image);
|
||||
}
|
||||
if (this.communicator) {
|
||||
console.log("submitDashboardImage");
|
||||
await this.communicator.submitDashboardImage(tiles, fingerprint, forceTiledCommit, paintMs);
|
||||
await this.communicator.waitForNextFrameMetrics(FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS);
|
||||
if (updatePreviewAfterTransmit) {
|
||||
@@ -628,6 +630,7 @@ class DashboardController {
|
||||
if (reason === "initial") {
|
||||
this.appendLog("initial dashboard image queued");
|
||||
}
|
||||
console.log("renderDashbaord finished");
|
||||
}
|
||||
|
||||
private async handleInputEvent(event: RawInputEvent): Promise<void> {
|
||||
@@ -780,9 +783,9 @@ class DashboardController {
|
||||
|
||||
private appendLog(line: string): void {
|
||||
const stamped = `[${formatTimestamp(new Date())}] ${line}`;
|
||||
this.log = this.log ? `${this.log}\n${stamped}` : stamped;
|
||||
//this.log = this.log ? `${this.log}\n${stamped}` : stamped;
|
||||
console.log(stamped);
|
||||
this.emit();
|
||||
//this.emit();
|
||||
}
|
||||
|
||||
private setDisplayPreview(preview: ImageSource | null): void {
|
||||
|
||||
@@ -241,6 +241,7 @@ export class FaceclawCommunicatorBridge {
|
||||
}
|
||||
const tileSnapshots = tileBmps.map((tile) => new Uint8Array(tile));
|
||||
await this.enqueueJavaCall(() => {
|
||||
console.log("submitDashboardImage calling Java function");
|
||||
this.communicator.submitDashboardImage4(
|
||||
toJavaByteArray(tileSnapshots[0]!),
|
||||
toJavaByteArray(tileSnapshots[1]!),
|
||||
|
||||
Reference in new issue
Block a user