From e853f0ea5dccdcaf98c2d1e2648d54b6b4fa4a36 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Wed, 6 May 2026 16:12:34 -0700 Subject: [PATCH] Refactoring --- .../faceclaw/app/FaceclawBleCommunicator.java | 132 +++++++-------- .../faceclaw/app/FaceclawVoiceController.java | 156 ++++++++++++++---- .../app/g2protocol/ConnectionOptions.java | 6 + .../app/g2protocol/MessageBuilder.java | 26 +-- .../app/g2protocol/OutboundMessage.java | 6 +- app/g2/dashboard-controller.ts | 7 +- app/native/faceclaw-communicator.ts | 1 + 7 files changed, 211 insertions(+), 123 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index 45f6a699..88e10da1 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -97,6 +97,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private int consecutiveAckTimeouts; private int lastAudioControlAckMagic = 0; + private ConnectionOptions connectionOptions = new ConnectionOptions(); private final BleMagicPool magicPool = new BleMagicPool(); private MessageBuilder messageBuilder = new MessageBuilder(magicPool); private int nextMapSessionId = 0; @@ -156,13 +157,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { running = false; audioCaptureActive = false; audioPacketListener = null; - phase = "disconnecting"; - status = "Disconnecting..."; - lock.notifyAll(); threadToJoin = workerThread; } + setStateDisplay("disconnecting", "Disconnecting..."); interruptibleSleep.interrupt(); - emitState(); if (threadToJoin != null) { threadToJoin.interrupt(); try { @@ -173,15 +171,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } synchronized (lock) { workerThread = null; - phase = "disconnected"; - status = "Disconnected."; resetSessionStateLocked(); clearAllMessagesLocked("disconnect"); } bleManager.disconnect(rightAddress); bleManager.disconnect(leftAddress); bleManager.close(); - emitState(); + setStateDisplay("disconnected", "Disconnected."); } public void close() { @@ -203,7 +199,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { magic = message.magic; pendingMessages.addFirst(message); logLine("queue G2 mic enable"); - lock.notifyAll(); } interruptibleSleep.interrupt(); return waitForAudioControlAck(magic, "enable"); @@ -220,7 +215,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { magic = message.magic; pendingMessages.addFirst(message); logLine("queue G2 mic disable"); - lock.notifyAll(); } } interruptibleSleep.interrupt(); @@ -293,7 +287,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { }; pendingMessages.addFirst(message); logLine("queue shutdown"); - lock.notifyAll(); } interruptibleSleep.interrupt(); @@ -435,7 +428,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } } - lock.notifyAll(); } interruptibleSleep.interrupt(); if (event != null) { @@ -483,19 +475,18 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { displayedFingerprint = ""; displayedTileBmps = emptyTileSet(); reconnectAfterMs = SystemClock.elapsedRealtime() + RECONNECT_DELAY_MS; - phase = userDisconnectRequested ? "disconnecting" : "connecting"; - status = userDisconnectRequested - ? "Disconnecting..." - : "Connection lost. Reconnecting..."; } - lock.notifyAll(); } interruptibleSleep.interrupt(); - emitState(); + if (connected) { + setStateDisplay("connected", "Connected."); + } else { + setStateDisplay("connecting", "Connecting to the glasses..."); + } } private void connectLoopOnce() throws InterruptedException { //{{{ - setState("connecting", "Connecting to the glasses..."); + setStateDisplay("connecting", "Connecting to the glasses..."); try { connectArm(rightAddress, true); connectArm(leftAddress, true); @@ -523,7 +514,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { lastAudioControlAckMagic = 0; audioCaptureActive = false; } - setState("connected", "Connected."); + setStateDisplay("connected", "Connected."); logLine("session ready"); } catch (Throwable t) { logLine("connect failed: " + safeMessage(t)); @@ -585,9 +576,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { handleTransportFailure("ack timeout"); }; prelude.sentAtMs = now; - synchronized (lock) { - inFlightMessages.add(prelude); - } writeMessage(prelude); long deadline = SystemClock.elapsedRealtime() + PRELUDE_TIMEOUT_MS; @@ -647,7 +635,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { OutboundMessage pending = pendingMessages.peekFirst(); messageToWrite = pendingMessages.removeFirst(); Log.i(TAG, "sending pending message: " + messageToWrite.label); - prepareMessageWriteLocked(messageToWrite, now); } else if (!shutdownRequested && fixedLayoutCreated && warmedUp && pendingMessages.isEmpty() && inFlightMessages.isEmpty() && now >= imageRetryAfterMs && !getDesiredFingerprint().equals(displayedFingerprint)) { @@ -657,7 +644,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } else if (shouldPollBatteryLocked(now)) { Log.i(TAG, "Writing battery query"); messageToWrite = createBatteryQueryMessageLocked(); - prepareMessageWriteLocked(messageToWrite, now); lastBatteryRefreshAtMs = now; } else if (!pendingMessages.isEmpty() || !inFlightMessages.isEmpty()) { return IDLE_SLEEP_MS; @@ -674,10 +660,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } return 0; } - - synchronized (lock) { - completeMessageWriteLocked(messageToWrite); - } } } @@ -692,8 +674,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { if (heartbeatReady && !heartbeatPending && inFlightMessages.isEmpty()) { Log.i(TAG, "Writing heartbeat"); - OutboundMessage heartbeatMessage = createHeartbeatMessageLocked(); - prepareMessageWriteLocked(heartbeatMessage, now); + OutboundMessage heartbeatMessage = createHeartbeatMessage(); lastHeartbeatSentAtMs = now; writeMessage(heartbeatMessage); return true; @@ -708,15 +689,21 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { return false; } - private OutboundMessage createHeartbeatMessageLocked() { + private OutboundMessage createHeartbeatMessage() { OutboundMessage message = messageBuilder.heartbeat(); message.onAck = () -> { - lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime(); + synchronized (lock) { + lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime(); + } }; message.onTimeout = () -> { // If a heartbeat fails to ack and we're over the heartbeat deadline, assume the connection is failed and reconnect. // Otherwise ignore it, which will cause a retransmission attempt. - if (SystemClock.elapsedRealtime() - lastHeartbeatSentAtMs >= HEARTBEAT_FAILURE_DEADLINE_MS) { + boolean isPastDeadline; + synchronized (lock) { + isPastDeadline = SystemClock.elapsedRealtime() - lastHeartbeatSentAtMs >= HEARTBEAT_FAILURE_DEADLINE_MS; + } + if (isPastDeadline) { handleTransportFailure("heartbeat ack timeout"); } }; @@ -724,41 +711,42 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } - private void prepareMessageWriteLocked(OutboundMessage message, long now) { + private boolean writeMessage(OutboundMessage message) { + long now = SystemClock.elapsedRealtime(); message.writeStartedAtMs = now; message.sentAtMs = now; message.ackDeadlineAtMs = now + message.ackTimeoutMs + WRITE_TIMEOUT_MS; if (message.magic != 0) { - inFlightMessages.addLast(message); + synchronized (lock) { + inFlightMessages.addLast(message); + } } if (message.imageUpdateId > 0 && message.imageMessageNumber == 1) { - BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(message.imageUpdateId); - if (stats != null && stats.firstWriteStartedAtMs <= 0) { - stats.firstWriteStartedAtMs = now; + synchronized(lock) { + BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(message.imageUpdateId); + if (stats != null && stats.firstWriteStartedAtMs <= 0) { + stats.firstWriteStartedAtMs = now; + } } } - } - private boolean writeMessage(OutboundMessage message) { - String writeAddress = isLeftArmMessage(message) ? leftAddress : rightAddress; - return bleManager.writeFrames( + String writeAddress = message.isLeftArmMessage ? leftAddress : rightAddress; + boolean result = bleManager.writeFrames( writeAddress, BleProtocol.WRITE_CHAR_UUID, message.frames, WRITE_TYPE, WRITE_TIMEOUT_MS ); - } - private boolean isLeftArmMessage(OutboundMessage message) { - return message != null && ("image".equals(message.kind) || "warmup".equals(message.kind)); - } + synchronized (lock) { + long sentAtMs = SystemClock.elapsedRealtime(); + message.sentAtMs = sentAtMs; + message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs; + logImageUpdateSendLandmarkLocked(message); + } - private void completeMessageWriteLocked(OutboundMessage message) { - long sentAtMs = SystemClock.elapsedRealtime(); - message.sentAtMs = sentAtMs; - message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs; - logImageUpdateSendLandmarkLocked(message); + return result; } private boolean removePreparedMessageLocked(OutboundMessage message) { @@ -781,20 +769,24 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { while (iterator.hasNext()) { OutboundMessage message = iterator.next(); if (message.sid == sid && message.magic == magic) { - Log.i(TAG, "Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")"); - iterator.remove(); - message.ackPayload = pb == null ? new byte[0] : Arrays.copyOf(pb, pb.length); - magicPool.release(message.sid, message.magic, message.label, "ack"); - if (message.onAck != null) { - message.onAck.run(); - } - consecutiveAckTimeouts = 0; + resolveAckLocked(message, pb); return; } } recordUnexpectedAckLocked(sid, magic); } + private void resolveAckLocked(OutboundMessage message, byte[] pb) { + Log.i(TAG, "Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")"); + inFlightMessages.remove(message); + message.ackPayload = pb == null ? new byte[0] : Arrays.copyOf(pb, pb.length); + magicPool.release(message.sid, message.magic, message.label, "ack"); + if (message.onAck != null) { + message.onAck.run(); + } + consecutiveAckTimeouts = 0; + } + private void logImageUpdateSendLandmarkLocked(OutboundMessage message) { if (message.imageUpdateId <= 0) { return; @@ -891,7 +883,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { int sessionId = nextMapSessionId(); List fragments = BleImageOptimizer.planImageFragments(bmp, IMAGE_FRAGMENT_SIZE); for (BleProtocol.ImageFragment fragment : fragments) { - OutboundMessage message = messageBuilder.imageWarmupFragment(tile, sessionId, fragment, bmp); + OutboundMessage message = messageBuilder.imageWarmupFragment(tile, sessionId, fragment, bmp, connectionOptions.sendImagesToLeft); pendingMessages.addLast(message); message.onAck = () -> { // The first tile warmup primes the image path but does not reliably @@ -992,13 +984,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { boolean requestAck, boolean isLast ) { - OutboundMessage message = messageBuilder.imageFragment(fragment, plan, requestAck); + OutboundMessage message = messageBuilder.imageFragment(fragment, plan, requestAck, connectionOptions.sendImagesToLeft); message.setImageUpdatePosition(updateId, messageNumber, messageCount); message.onAck = () -> { imageRetryAfterMs = 0; logImageUpdateAckLandmarkLocked(message); if (message.tileIndex >= 0 && !hasPendingOrInflightTileLocked(message.tileIndex)) { - displayedTileBmps[message.tileIndex] = BmpUtil.copyTileBmp(message.tileBmp); + displayedTileBmps[message.tileIndex] = BmpUtil.copyTileBmp(plan.bmp); } boolean imageStillInFlight = false; for (OutboundMessage inFlight : inFlightMessages) { @@ -1203,16 +1195,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { displayedTileBmps = emptyTileSet(); clearAllMessagesLocked("transport failure: " + reason); reconnectAfterMs = SystemClock.elapsedRealtime() + RECONNECT_DELAY_MS; - if (!userDisconnectRequested) { - phase = "retrying"; - status = reason == null || reason.isEmpty() ? "Reconnecting..." : "Reconnecting after " + reason; - } - lock.notifyAll(); - interruptibleSleep.interrupt(); bleManager.disconnect(rightAddress); bleManager.disconnect(leftAddress); - emitState(); } + if (!userDisconnectRequested) { + setStateDisplay("retrying", reason == null || reason.isEmpty() ? "Reconnecting..." : "Reconnecting after " + reason); + } + interruptibleSleep.interrupt(); } private void resetSessionStateLocked() { @@ -1324,7 +1313,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { }); } - private void setState(String nextPhase, String nextStatus) { + private void setStateDisplay(String nextPhase, String nextStatus) { synchronized (lock) { phase = nextPhase; status = nextStatus; @@ -1374,7 +1363,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private int nextMapSessionId() { int id = nextMapSessionId; - nextMapSessionId = (nextMapSessionId + 1) & 0xff; + int increment = connectionOptions.skipSessionIds ? 2 : 1; + nextMapSessionId = (nextMapSessionId + increment) & 0xff; return id; } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index f6e86adf..81e6206e 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -11,10 +11,13 @@ import com.k2fsa.sherpa.onnx.FeatureConfig; import com.k2fsa.sherpa.onnx.KeywordSpotter; import com.k2fsa.sherpa.onnx.KeywordSpotterConfig; import com.k2fsa.sherpa.onnx.KeywordSpotterResult; +import com.k2fsa.sherpa.onnx.OfflineModelConfig; +import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; +import com.k2fsa.sherpa.onnx.OfflineRecognizer; +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; +import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; +import com.k2fsa.sherpa.onnx.OfflineStream; import com.k2fsa.sherpa.onnx.OnlineModelConfig; -import com.k2fsa.sherpa.onnx.OnlineRecognizer; -import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig; -import com.k2fsa.sherpa.onnx.OnlineRecognizerResult; import com.k2fsa.sherpa.onnx.OnlineStream; import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig; @@ -23,6 +26,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayDeque; +import java.util.Arrays; public class FaceclawVoiceController { private static final String TAG = "FaceclawVoice"; @@ -32,10 +36,16 @@ public class FaceclawVoiceController { private static final int EXPECTED_PACKET_INTERVAL_MS = 50; private static final int LATE_PACKET_INTERVAL_MS = 90; private static final int STATS_INTERVAL_MS = 5_000; + private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 900; + private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 2; + private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 8; + private static final int TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES = (int) (SAMPLE_RATE * 1.2); + private static final float TRANSCRIPT_SILENCE_PEAK = 0.015f; + private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80; private static final String ASSET_ROOT = "faceclaw-voice"; private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr"; - private static final String ASR_MODEL_DIR = "sherpa-onnx-streaming-zipformer-en-2023-06-26"; + private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27"; private static final String[] MODEL_FILES = { "encoder-epoch-12-avg-2-chunk-16-left-64.onnx", "decoder-epoch-12-avg-2-chunk-16-left-64.onnx", @@ -44,9 +54,8 @@ public class FaceclawVoiceController { "screen-on-keywords.txt" }; private static final String[] ASR_MODEL_FILES = { - "encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx", - "decoder-epoch-99-avg-1-chunk-16-left-128.onnx", - "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx", + "encoder_model.ort", + "decoder_model_merged.ort", "tokens.txt" }; @@ -66,10 +75,16 @@ public class FaceclawVoiceController { private volatile boolean started; private VoiceInputMode mode = VoiceInputMode.WAKEWORD; private KeywordSpotter keywordSpotter; - private OnlineRecognizer recognizer; + private OfflineRecognizer recognizer; private OnlineStream stream; private FaceclawLc3Decoder lc3Decoder; + private float[] transcriptSamples = new float[TRANSCRIPT_MAX_SAMPLES]; + private int transcriptSampleCount; + private int transcriptSilentSamples; + private boolean transcriptHadSpeech; + private long lastTranscriptDecodeAtMs; private String lastTranscript = ""; + private float lastTranscriptPeak; private long queuedPackets; private long queueDroppedPackets; private long decodedSamples; @@ -153,8 +168,8 @@ public class FaceclawVoiceController { : "Voice control loading wake-word model..."); if (currentMode == VoiceInputMode.FULL) { File modelDir = installAsrModelFiles(); - recognizer = new OnlineRecognizer(buildRecognizerConfig(modelDir)); - stream = recognizer.createStream(); + recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir)); + resetTranscriptState(); lastTranscript = ""; } else { File modelDir = installModelFiles(); @@ -208,23 +223,20 @@ public class FaceclawVoiceController { .build(); } - private OnlineRecognizerConfig buildRecognizerConfig(File modelDir) { - return OnlineRecognizerConfig.builder() + private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) { + return OfflineRecognizerConfig.builder() .setFeatureConfig(FeatureConfig.builder() .setSampleRate(SAMPLE_RATE) .setFeatureDim(FEATURE_DIM) .build()) - .setOnlineModelConfig(OnlineModelConfig.builder() - .setTransducer(OnlineTransducerModelConfig.builder() - .setEncoder(new File(modelDir, "encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath()) - .setDecoder(new File(modelDir, "decoder-epoch-99-avg-1-chunk-16-left-128.onnx").getAbsolutePath()) - .setJoiner(new File(modelDir, "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath()) + .setModelConfig(OfflineModelConfig.builder() + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) + .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) .build()) .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) - .setModelType("zipformer2") .setNumThreads(1) .build()) - .setEnableEndpoint(true) .build(); } @@ -291,7 +303,7 @@ public class FaceclawVoiceController { while (started && !Thread.currentThread().isInterrupted()) { OnlineStream currentStream = stream; FaceclawLc3Decoder currentDecoder = lc3Decoder; - if (currentStream == null || currentDecoder == null) { + if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) { return; } @@ -310,11 +322,10 @@ public class FaceclawVoiceController { for (int i = 0; i < count; i++) { samples[i] = pcm[i] / 32768.0f; } - currentStream.acceptWaveform(samples, SAMPLE_RATE); - if (mode == VoiceInputMode.FULL) { - processRecognizer(currentStream); + processRecognizer(samples); } else { + currentStream.acceptWaveform(samples, SAMPLE_RATE); processKeywordSpotter(currentStream); } maybeEmitAudioStats(false); @@ -337,28 +348,96 @@ public class FaceclawVoiceController { } } - private void processRecognizer(OnlineStream currentStream) { - OnlineRecognizer currentRecognizer = recognizer; - if (currentRecognizer == null) { + private void processRecognizer(float[] samples) { + appendTranscriptSamples(samples); + updateTranscriptSilence(samples); + + if (transcriptSampleCount >= TRANSCRIPT_MAX_SAMPLES) { + decodeTranscriptSegment(true); + resetTranscriptState(); + lastTranscript = ""; return; } - while (currentRecognizer.isReady(currentStream)) { - currentRecognizer.decode(currentStream); + + long now = SystemClock.elapsedRealtime(); + if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { + decodeTranscriptSegment(false); + lastTranscriptDecodeAtMs = now; } - emitRecognizerResult(currentRecognizer, currentStream, false); - if (currentRecognizer.isEndpoint(currentStream)) { - emitRecognizerResult(currentRecognizer, currentStream, true); - currentRecognizer.reset(currentStream); + + if (transcriptHadSpeech && transcriptSilentSamples >= TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES) { + decodeTranscriptSegment(true); + resetTranscriptState(); lastTranscript = ""; } } - private void emitRecognizerResult(OnlineRecognizer currentRecognizer, OnlineStream currentStream, boolean isFinal) { - OnlineRecognizerResult result = currentRecognizer.getResult(currentStream); + private void appendTranscriptSamples(float[] samples) { + int available = TRANSCRIPT_MAX_SAMPLES - transcriptSampleCount; + if (samples.length > available) { + int drop = samples.length - available; + if (drop >= transcriptSampleCount) { + transcriptSampleCount = 0; + } else { + System.arraycopy(transcriptSamples, drop, transcriptSamples, 0, transcriptSampleCount - drop); + transcriptSampleCount -= drop; + } + } + System.arraycopy(samples, 0, transcriptSamples, transcriptSampleCount, samples.length); + transcriptSampleCount += samples.length; + } + + private void updateTranscriptSilence(float[] samples) { + float peak = 0.0f; + for (float sample : samples) { + float abs = Math.abs(sample); + if (abs > peak) { + peak = abs; + } + } + if (peak > lastTranscriptPeak) { + lastTranscriptPeak = peak; + } + if (peak < TRANSCRIPT_SILENCE_PEAK) { + transcriptSilentSamples += samples.length; + } else { + transcriptSilentSamples = 0; + transcriptHadSpeech = true; + } + } + + private void decodeTranscriptSegment(boolean isFinal) { + OfflineRecognizer currentRecognizer = recognizer; + if (currentRecognizer == null || transcriptSampleCount <= 0) { + return; + } + OfflineStream offlineStream = currentRecognizer.createStream(); + try { + offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE); + currentRecognizer.decode(offlineStream); + emitRecognizerResult(currentRecognizer.getResult(offlineStream), isFinal, transcriptSampleCount, lastTranscriptPeak); + } finally { + offlineStream.release(); + } + } + + private void emitRecognizerResult(OfflineRecognizerResult result, boolean isFinal, int sampleCount, float peak) { String text = result == null ? "" : result.getText(); if (text == null) { text = ""; } + String trimmed = text.trim(); + String preview = trimmed.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS + ? trimmed + : trimmed.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."; + Log.i(TAG, "Moonshine decode final=" + isFinal + + " audioSec=" + String.format(java.util.Locale.US, "%.2f", sampleCount / (double) SAMPLE_RATE) + + " peak=" + String.format(java.util.Locale.US, "%.4f", peak) + + " textLen=" + trimmed.length() + + " text=\"" + preview + "\""); + emitStatus("Moonshine decode " + + String.format(java.util.Locale.US, "%.1fs", sampleCount / (double) SAMPLE_RATE) + + ", textLen=" + trimmed.length()); if (text.equals(lastTranscript)) { if (isFinal && lastTranscript.length() > 0) { emitTranscript("", true); @@ -379,6 +458,14 @@ public class FaceclawVoiceController { } } + private void resetTranscriptState() { + transcriptSampleCount = 0; + transcriptSilentSamples = 0; + transcriptHadSpeech = false; + lastTranscriptDecodeAtMs = 0; + lastTranscriptPeak = 0.0f; + } + private void stopG2Audio() { FaceclawBleCommunicator currentCommunicator = communicator; if (currentCommunicator != null) { @@ -510,6 +597,7 @@ public class FaceclawVoiceController { if (currentListener == null) { return; } + Log.i(TAG, "Emit transcript final=" + isFinal + " textLen=" + (text == null ? 0 : text.trim().length())); mainHandler.post(() -> currentListener.onTranscript(text, isFinal)); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java new file mode 100644 index 00000000..85e103c8 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java @@ -0,0 +1,6 @@ +package com.faceclaw.app; + +public class ConnectionOptions { + final boolean sendImagesToLeft = true; + final boolean skipSessionIds = true; +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java index 667a3bff..89130ac7 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java @@ -20,8 +20,8 @@ public class MessageBuilder { BleProtocol.PRELUDE_ACK_MAGIC, CollectionUtils.singletonList(Arrays.copyOf(BleProtocol.PRELUDE_F5872, BleProtocol.PRELUDE_F5872.length)), ACK_TIMEOUT_MS, - 0, - null + -1, + false ); } @@ -40,11 +40,11 @@ public class MessageBuilder { ), ACK_TIMEOUT_MS, -1, - null + false ); } - public OutboundMessage imageWarmupFragment(BleProtocol.ImageTileOptions tile, int sessionId, BleProtocol.ImageFragment fragment, byte[] bmp) { + public OutboundMessage imageWarmupFragment(BleProtocol.ImageTileOptions tile, int sessionId, BleProtocol.ImageFragment fragment, byte[] bmp, boolean leftArm) { int magic = magicPool.allocate(); return new OutboundMessage( "warmup", @@ -57,12 +57,12 @@ public class MessageBuilder { nextTransportSeq++ ), WARMUP_FRAGMENT_TIMEOUT_MS, - 0, - bmp + -1, + leftArm ); } - public OutboundMessage imageFragment(BleProtocol.ImageFragment fragment, BleImageOptimizer.TileImagePlan plan, boolean requestAck) { + public OutboundMessage imageFragment(BleProtocol.ImageFragment fragment, BleImageOptimizer.TileImagePlan plan, boolean requestAck, boolean leftArm) { int magic = requestAck ? magicPool.allocate() : 0; return new OutboundMessage( "image", @@ -76,7 +76,7 @@ public class MessageBuilder { ), ACK_TIMEOUT_MS, plan.tileIndex, - plan.bmp + leftArm ); } @@ -95,7 +95,7 @@ public class MessageBuilder { ), ACK_TIMEOUT_MS, -1, - null + false ); } @@ -114,7 +114,7 @@ public class MessageBuilder { ), ACK_TIMEOUT_MS, -1, - null + false ); } @@ -133,7 +133,7 @@ public class MessageBuilder { ), ACK_TIMEOUT_MS, -1, - null + false ); } @@ -147,7 +147,7 @@ public class MessageBuilder { BleProtocol.framePb(BleProtocol.buildHeartbeat(magic), BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, nextTransportSeq++), HEARTBEAT_TIMEOUT_MS, -1, - null + false ); } @@ -166,7 +166,7 @@ public class MessageBuilder { ), ACK_TIMEOUT_MS, -1, - null + false ); } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/OutboundMessage.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/OutboundMessage.java index 06908ac2..51eb89ec 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/OutboundMessage.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/OutboundMessage.java @@ -9,7 +9,7 @@ public final class OutboundMessage { final List frames; final int ackTimeoutMs; final int tileIndex; - final byte[] tileBmp; + final boolean isLeftArmMessage; int imageUpdateId; int imageMessageNumber; @@ -21,7 +21,7 @@ public final class OutboundMessage { Runnable onAck; Runnable onTimeout; - OutboundMessage(String kind, String label, int sid, int magic, List frames, int ackTimeoutMs, int tileIndex, byte[] tileBmp) { + OutboundMessage(String kind, String label, int sid, int magic, List frames, int ackTimeoutMs, int tileIndex, boolean isLeftArmMessage) { this.kind = kind; this.label = label; this.sid = sid; @@ -29,7 +29,7 @@ public final class OutboundMessage { this.frames = frames; this.ackTimeoutMs = ackTimeoutMs; this.tileIndex = tileIndex; - this.tileBmp = BmpUtil.copyTileBmp(tileBmp); + this.isLeftArmMessage = isLeftArmMessage; } void setImageUpdatePosition(int updateId, int messageNumber, int messageCount) { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 49112ad2..f9f3a1b6 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -606,6 +606,7 @@ class DashboardController { } private async renderDashboard(reason: "initial" | "interval"): Promise { + console.log("renderDashboard", reason); const paintStartedAtMs = Date.now(); const image = drawDashboard(); const paintMs = Date.now() - paintStartedAtMs; @@ -617,6 +618,7 @@ class DashboardController { this.updateDisplayPreviewFromImage(image); } if (this.communicator) { + console.log("submitDashboardImage"); await this.communicator.submitDashboardImage(tiles, fingerprint, forceTiledCommit, paintMs); await this.communicator.waitForNextFrameMetrics(FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); if (updatePreviewAfterTransmit) { @@ -628,6 +630,7 @@ class DashboardController { if (reason === "initial") { this.appendLog("initial dashboard image queued"); } + console.log("renderDashbaord finished"); } private async handleInputEvent(event: RawInputEvent): Promise { @@ -780,9 +783,9 @@ class DashboardController { private appendLog(line: string): void { const stamped = `[${formatTimestamp(new Date())}] ${line}`; - this.log = this.log ? `${this.log}\n${stamped}` : stamped; + //this.log = this.log ? `${this.log}\n${stamped}` : stamped; console.log(stamped); - this.emit(); + //this.emit(); } private setDisplayPreview(preview: ImageSource | null): void { diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 6874b287..9f0adf9d 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -241,6 +241,7 @@ export class FaceclawCommunicatorBridge { } const tileSnapshots = tileBmps.map((tile) => new Uint8Array(tile)); await this.enqueueJavaCall(() => { + console.log("submitDashboardImage calling Java function"); this.communicator.submitDashboardImage4( toJavaByteArray(tileSnapshots[0]!), toJavaByteArray(tileSnapshots[1]!),