diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS index 8d852af6..8fb2897b 100644 --- a/ACKNOWLEDGEMENTS +++ b/ACKNOWLEDGEMENTS @@ -5,3 +5,5 @@ to keep my room clean, so to speak. Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/ and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa community. + +G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3 diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/BleProtocol.java b/App_Resources/Android/src/main/java/com/faceclaw/app/BleProtocol.java index 58a65406..f4aa7f6f 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/BleProtocol.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/BleProtocol.java @@ -33,6 +33,7 @@ public class BleProtocol { public static final int FLAG_REQUEST = 0x20; public static final int FLAG_NOTIFY = 0x01; public static final int FLAG_NOTIFY_ALT = 0x06; + public static final int CMD_AUDIO_CONTROL = 15; public static final int EVENT_CLICK = 0; public static final int EVENT_SCROLL_TOP = 1; @@ -112,6 +113,10 @@ public class BleProtocol { return wrapEvenHub(12, magic, 14, encodeVarintField(1, 0)); } + public static byte[] buildAudioControl(int magic, boolean enable) { + return wrapEvenHub(CMD_AUDIO_CONTROL, magic, 18, encodeVarintField(1, enable ? 1 : 0)); + } + public static byte[] buildShutdown(int magic, int exitMode) { return wrapEvenHub(9, magic, 11, encodeVarintField(1, exitMode)); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioPacketListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioPacketListener.java new file mode 100644 index 00000000..16aa0702 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioPacketListener.java @@ -0,0 +1,5 @@ +package com.faceclaw.app; + +public interface FaceclawAudioPacketListener { + void onAudioPacket(byte[] data, String arm, long arrivalMs); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index c17dd70f..cae434a1 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -102,6 +102,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private long lastSessionReadyAtMs; private long lastEvenAppConflictAtMs; private int consecutiveAckTimeouts; + private int lastAudioControlAckMagic = 0; private final BleMagicPool magicPool = new BleMagicPool(); private int nextTransportSeq = 0x40; @@ -111,6 +112,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private long lastShutdownExitAtMs = 0; private int headsetBattery = -1; private int headsetCharging = -1; + private boolean audioCaptureActive; + private volatile FaceclawAudioPacketListener audioPacketListener; private String displayedFingerprint = ""; private byte[][] displayedTileBmps = emptyTileSet(); @@ -158,6 +161,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { synchronized (lock) { userDisconnectRequested = true; running = false; + audioCaptureActive = false; + audioPacketListener = null; phase = "disconnecting"; status = "Disconnecting..."; lock.notifyAll(); @@ -190,6 +195,51 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { disconnect(); } + public boolean startG2AudioCapture(FaceclawAudioPacketListener listener) { + if (listener == null) { + throw new IllegalArgumentException("listener is required"); + } + int magic; + synchronized (lock) { + if (!running || !sessionReady) { + logLine("skip G2 mic enable; session not ready"); + return false; + } + audioPacketListener = listener; + magic = nextMagic(); + pendingMessages.addFirst(createAudioControlMessageLocked(true, magic)); + logLine("queue G2 mic enable"); + lock.notifyAll(); + } + interruptibleSleep.interrupt(); + return waitForAudioControlAck(magic, "enable"); + } + + public void stopG2AudioCapture() { + int magic = 0; + synchronized (lock) { + audioPacketListener = null; + audioCaptureActive = false; + clearMessagesOfKindLocked("audio-control"); + if (running && sessionReady) { + magic = nextMagic(); + pendingMessages.addFirst(createAudioControlMessageLocked(false, magic)); + logLine("queue G2 mic disable"); + lock.notifyAll(); + } + } + interruptibleSleep.interrupt(); + if (magic != 0) { + waitForAudioControlAck(magic, "disable"); + } + } + + public boolean isSessionReady() { + synchronized (lock) { + return running && sessionReady; + } + } + public void submitDashboardImage(List tileBmps, String fingerprint) { submitDashboardImage(tileBmps, fingerprint, false, -1); @@ -365,6 +415,33 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + private boolean waitForAudioControlAck(int magic, String operation) { + long deadline = SystemClock.elapsedRealtime() + ACK_TIMEOUT_MS + WRITE_TIMEOUT_MS + 500; + synchronized (lock) { + while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) { + long remaining = deadline - SystemClock.elapsedRealtime(); + if (remaining <= 0) { + break; + } + try { + lock.wait(Math.min(remaining, 100)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + boolean acked = lastAudioControlAckMagic == magic; + if (!acked) { + if ("enable".equals(operation)) { + audioPacketListener = null; + audioCaptureActive = false; + } + logLine("G2 mic " + operation + " ack timeout"); + } + return acked; + } + } + @Override public void run() { @@ -404,6 +481,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { return; } String uuid = characteristicUuid.toLowerCase(Locale.US); + if (BleProtocol.RENDER_NOTIFY_UUID.equals(uuid)) { + handleRenderNotification(address, data); + return; + } if (!BleProtocol.NOTIFY_CHAR_UUID.equals(uuid)) { return; } @@ -430,6 +511,24 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + private void handleRenderNotification(String address, byte[] data) { + FaceclawAudioPacketListener listenerToCall; + long arrivalMs = SystemClock.elapsedRealtime(); + synchronized (lock) { + lastIncomingAtMs = arrivalMs; + listenerToCall = audioCaptureActive ? audioPacketListener : null; + } + if (listenerToCall == null) { + return; + } + String arm = address.equalsIgnoreCase(leftAddress) ? "L" : address.equalsIgnoreCase(rightAddress) ? "R" : "?"; + try { + listenerToCall.onAudioPacket(Arrays.copyOf(data, data.length), arm, arrivalMs); + } catch (Throwable t) { + logLine("G2 mic packet listener failed: " + safeMessage(t)); + } + } + @Override public void onConnectionStateChange(String address, boolean connected) { synchronized (lock) { @@ -446,6 +545,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { fixedLayoutCreated = false; warmedUp = false; startupProbePending = false; + audioCaptureActive = false; + audioPacketListener = null; clearAllMessagesLocked("connection lost"); displayedFingerprint = ""; displayedTileBmps = emptyTileSet(); @@ -465,7 +566,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { setState("connecting", "Connecting to the glasses..."); try { connectArm(rightAddress, true); - connectArm(leftAddress, false); + connectArm(leftAddress, true); if (!sleepDuringConnectSettling(800)) { return; } @@ -486,6 +587,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { imageRetryAfterMs = 0; lastHeartbeatQueuedAtMs = 0; consecutiveAckTimeouts = 0; + lastAudioControlAckMagic = 0; + audioCaptureActive = false; } setState("connected", "Connected."); logLine("session ready"); @@ -785,6 +888,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { if ("heartbeat".equals(message.kind)) { return; } + if ("audio-control".equals(message.kind)) { + lastAudioControlAckMagic = message.magic; + audioCaptureActive = message.label != null && message.label.contains("enable"); + logLine(audioCaptureActive ? "G2 mic enabled" : "G2 mic disabled"); + lock.notifyAll(); + return; + } if ("battery".equals(message.kind)) { BleProtocol.BatterySnapshot snapshot = BleProtocol.parseSettingsBattery(message.ackPayload); if (snapshot != null) { @@ -1035,6 +1145,25 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { ); } + private OutboundMessage createAudioControlMessageLocked(boolean enable, int magic) { + return new OutboundMessage( + "audio-control", + enable ? "G2 mic enable" : "G2 mic disable", + BleProtocol.SID_EVENHUB, + magic, + BleProtocol.framePb( + BleProtocol.buildAudioControl(magic, enable), + BleProtocol.SID_EVENHUB, + BleProtocol.FLAG_REQUEST, + nextTransportSeq++ + ), + ACK_TIMEOUT_MS, + displayedFingerprint, + -1, + null + ); + } + private boolean shouldPollBatteryLocked(long now) { return !shutdownRequested && sessionReady @@ -1265,6 +1394,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { lastHeartbeatQueuedAtMs = 0; lastSessionReadyAtMs = 0; consecutiveAckTimeouts = 0; + lastAudioControlAckMagic = 0; + audioCaptureActive = false; + audioPacketListener = null; displayedFingerprint = ""; displayedTileBmps = emptyTileSet(); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java index 74884f24..1e362444 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java @@ -107,6 +107,7 @@ public class FaceclawForegroundService extends Service { private int foregroundServiceType() { int type = ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE; + // TODO: Make this depend on which audio path (G2 vs phone) is selected if (hasRecordAudioPermission()) { type |= ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE; } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java new file mode 100644 index 00000000..ce9f0557 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java @@ -0,0 +1,97 @@ +package com.faceclaw.app; + +import android.util.Log; + +public class FaceclawLc3Decoder { + private static final String TAG = "FaceclawLc3"; + public static final int SAMPLE_RATE = 16000; + public static final int FRAME_US = 10000; + public static final int FRAME_BYTES = 40; + public static final int FRAMES_PER_PACKET = 5; + public static final int PACKET_BYTES = 205; + public static final int COUNTER_OFFSET = 204; + public static final int SAMPLES_PER_FRAME = 160; + public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME; + + static { + System.loadLibrary("faceclaw_lc3"); + } + + private long nativeHandle; + private int lastCounter = -1; + private long realPackets; + private long duplicatePackets; + private long missingPackets; + private long decodeErrors; + + public FaceclawLc3Decoder() { + nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE); + if (nativeHandle == 0) { + throw new IllegalStateException("Could not create LC3 decoder"); + } + } + + public synchronized int decodePacket(byte[] packet, short[] pcmOut) { + if (nativeHandle == 0) { + throw new IllegalStateException("LC3 decoder is closed"); + } + if (packet == null || packet.length != PACKET_BYTES) { + decodeErrors++; + Log.w(TAG, "unexpected G2 audio packet length=" + (packet == null ? -1 : packet.length)); + return 0; + } + if (pcmOut == null || pcmOut.length < SAMPLES_PER_PACKET) { + throw new IllegalArgumentException("pcmOut must hold " + SAMPLES_PER_PACKET + " samples"); + } + + int counter = packet[COUNTER_OFFSET] & 0xff; + if (lastCounter >= 0) { + int gap = (counter - lastCounter) & 0xff; + if (gap == 0) { + duplicatePackets++; + return 0; + } + int missing = gap - 1; + if (missing > 0) { + missingPackets += missing; + } + } + + int decoded = nativeDecodePacket(nativeHandle, packet, pcmOut); + if (decoded <= 0) { + decodeErrors++; + return 0; + } + realPackets++; + lastCounter = counter; + return decoded; + } + + public synchronized long getRealPackets() { + return realPackets; + } + + public synchronized long getDuplicatePackets() { + return duplicatePackets; + } + + public synchronized long getMissingPackets() { + return missingPackets; + } + + public synchronized long getDecodeErrors() { + return decodeErrors; + } + + public synchronized void close() { + long handle = nativeHandle; + nativeHandle = 0; + if (handle != 0) { + nativeDestroy(handle); + } + } + + private static native long nativeCreate(int frameUs, int sampleRate); + private static native int nativeDecodePacket(long handle, byte[] packet, short[] pcmOut); + private static native void nativeDestroy(long handle); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 9597818a..a7a0194b 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -1,25 +1,12 @@ package com.faceclaw.app; -import android.Manifest; -import android.bluetooth.BluetoothAdapter; -import android.bluetooth.BluetoothDevice; -import android.bluetooth.BluetoothManager; -import android.bluetooth.BluetoothProfile; import android.content.Context; -import android.content.pm.PackageManager; import android.content.res.AssetManager; -import android.media.AudioDeviceInfo; -import android.media.AudioFormat; -import android.media.AudioManager; -import android.media.AudioRecord; -import android.media.MediaRecorder; -import android.os.Build; import android.os.Handler; import android.os.Looper; +import android.os.SystemClock; import android.util.Log; -import androidx.core.content.ContextCompat; - import com.k2fsa.sherpa.onnx.FeatureConfig; import com.k2fsa.sherpa.onnx.KeywordSpotter; import com.k2fsa.sherpa.onnx.KeywordSpotterConfig; @@ -32,12 +19,16 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Set; +import java.util.ArrayDeque; public class FaceclawVoiceController { private static final String TAG = "FaceclawVoice"; private static final int SAMPLE_RATE = 16000; private static final int FEATURE_DIM = 80; + private static final int MAX_AUDIO_QUEUE_PACKETS = 80; + private static final int EXPECTED_PACKET_INTERVAL_MS = 50; + private static final int LATE_PACKET_INTERVAL_MS = 90; + private static final int STATS_INTERVAL_MS = 5_000; private static final String ASSET_ROOT = "faceclaw-voice"; private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; private static final String[] MODEL_FILES = { @@ -51,12 +42,23 @@ public class FaceclawVoiceController { private final Context appContext; private final Handler mainHandler = new Handler(Looper.getMainLooper()); private final Object lock = new Object(); + private final Object audioQueueLock = new Object(); + private final ArrayDeque audioQueue = new ArrayDeque<>(); private volatile FaceclawVoiceControllerListener listener; + private volatile FaceclawBleCommunicator communicator; private Thread workerThread; private volatile boolean started; - private AudioRecord audioRecord; private KeywordSpotter keywordSpotter; private OnlineStream stream; + private FaceclawLc3Decoder lc3Decoder; + private long queuedPackets; + private long queueDroppedPackets; + private long decodedSamples; + private long latePackets; + private long wrongArmPackets; + private long lastPacketArrivalMs; + private long maxInterPacketMs; + private long lastStatsAtMs; public FaceclawVoiceController(Context context) { this.appContext = context.getApplicationContext(); @@ -66,18 +68,18 @@ public class FaceclawVoiceController { this.listener = listener; } + public void setCommunicator(FaceclawBleCommunicator communicator) { + this.communicator = communicator; + } + public void start() { synchronized (lock) { if (started) { emitStatus("Voice control is already listening."); return; } - if (!hasRecordAudioPermission()) { - emitStatus("Voice control needs microphone permission."); - return; - } - if (!hasBluetoothHeadset()) { - emitStatus("Voice control needs a paired Bluetooth headset."); + if (communicator == null || !communicator.isSessionReady()) { + emitStatus("Voice control needs an active G2 connection."); return; } started = true; @@ -95,7 +97,10 @@ public class FaceclawVoiceController { started = false; threadToJoin = workerThread; } - stopAudio(); + stopG2Audio(); + synchronized (audioQueueLock) { + audioQueueLock.notifyAll(); + } if (threadToJoin != null) { threadToJoin.interrupt(); } @@ -111,19 +116,21 @@ public class FaceclawVoiceController { File modelDir = installModelFiles(); keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); stream = keywordSpotter.createStream(); - if (!startAudio()) { - emitStatus("Voice control could not start microphone input."); + lc3Decoder = new FaceclawLc3Decoder(); + if (!startG2Audio()) { + emitStatus("Voice control could not start G2 microphone input."); return; } - emitStatus("Voice control listening for \"screen on\"."); - processAudio(); + emitStatus("Voice control listening for \"screen on\" from the G2 mic."); + processG2Audio(); } catch (Throwable error) { Log.e(TAG, "Voice control failed", error); emitStatus("Voice control failed: " + error.getMessage()); } finally { + stopG2Audio(); releaseSherpa(); - stopAudio(); + releaseLc3(); synchronized (lock) { started = false; workerThread = null; @@ -183,53 +190,42 @@ public class FaceclawVoiceController { } } - private boolean startAudio() { - routeToBluetoothHeadset(); - int minBufferBytes = AudioRecord.getMinBufferSize( - SAMPLE_RATE, - AudioFormat.CHANNEL_IN_MONO, - AudioFormat.ENCODING_PCM_16BIT - ); - if (minBufferBytes <= 0) { + private boolean startG2Audio() { + FaceclawBleCommunicator currentCommunicator = communicator; + if (currentCommunicator == null) { return false; } - if (!hasRecordAudioPermission()) { - return false; + resetAudioStats(); + synchronized (audioQueueLock) { + audioQueue.clear(); } - audioRecord = new AudioRecord( - MediaRecorder.AudioSource.VOICE_RECOGNITION, - SAMPLE_RATE, - AudioFormat.CHANNEL_IN_MONO, - AudioFormat.ENCODING_PCM_16BIT, - minBufferBytes * 2 - ); - if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { - audioRecord.release(); - audioRecord = null; - return false; - } - audioRecord.startRecording(); - return true; + return currentCommunicator.startG2AudioCapture(this::queueAudioPacket); } - private void processAudio() { - short[] buffer = new short[SAMPLE_RATE / 10]; + private void processG2Audio() { + short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET]; while (started && !Thread.currentThread().isInterrupted()) { - AudioRecord currentAudio = audioRecord; OnlineStream currentStream = stream; KeywordSpotter currentSpotter = keywordSpotter; - if (currentAudio == null || currentStream == null || currentSpotter == null) { + FaceclawLc3Decoder currentDecoder = lc3Decoder; + if (currentStream == null || currentSpotter == null || currentDecoder == null) { return; } - int count = currentAudio.read(buffer, 0, buffer.length); - if (count <= 0) { + AudioPacket packet = takeAudioPacket(); + if (packet == null) { continue; } + int count = currentDecoder.decodePacket(packet.data, pcm); + if (count <= 0) { + maybeEmitAudioStats(false); + continue; + } + decodedSamples += count; float[] samples = new float[count]; for (int i = 0; i < count; i++) { - samples[i] = buffer[i] / 32768.0f; + samples[i] = pcm[i] / 32768.0f; } currentStream.acceptWaveform(samples, SAMPLE_RATE); @@ -242,27 +238,16 @@ public class FaceclawVoiceController { emitWakeWord(keyword); } } + maybeEmitAudioStats(false); } } - private void stopAudio() { - AudioRecord record = audioRecord; - audioRecord = null; - if (record != null) { - try { - record.stop(); - } catch (IllegalStateException ignored) { - } - record.release(); - } - AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE); - if (audioManager != null) { - if (Build.VERSION.SDK_INT >= 31) { - audioManager.clearCommunicationDevice(); - } - audioManager.stopBluetoothSco(); - audioManager.setBluetoothScoOn(false); + private void stopG2Audio() { + FaceclawBleCommunicator currentCommunicator = communicator; + if (currentCommunicator != null) { + currentCommunicator.stopG2AudioCapture(); } + maybeEmitAudioStats(true); } private void releaseSherpa() { @@ -276,51 +261,90 @@ public class FaceclawVoiceController { } } - private void routeToBluetoothHeadset() { - AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE); - if (audioManager == null) { + private void releaseLc3() { + if (lc3Decoder != null) { + lc3Decoder.close(); + lc3Decoder = null; + } + } + + private void queueAudioPacket(byte[] data, String arm, long arrivalMs) { + if (!started || data == null) { return; } - audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION); - audioManager.startBluetoothSco(); - audioManager.setBluetoothScoOn(true); - if (Build.VERSION.SDK_INT >= 31) { - for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) { - if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO - || device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) { - audioManager.setCommunicationDevice(device); - return; + if (!"L".equals(arm)) { + wrongArmPackets++; + } + synchronized (audioQueueLock) { + if (audioQueue.size() >= MAX_AUDIO_QUEUE_PACKETS) { + audioQueue.removeFirst(); + queueDroppedPackets++; + } + audioQueue.addLast(new AudioPacket(data, arm, arrivalMs)); + queuedPackets++; + if (lastPacketArrivalMs > 0) { + long delta = arrivalMs - lastPacketArrivalMs; + if (delta > maxInterPacketMs) { + maxInterPacketMs = delta; + } + if (delta > LATE_PACKET_INTERVAL_MS) { + latePackets++; } } + lastPacketArrivalMs = arrivalMs; + audioQueueLock.notifyAll(); } } - private boolean hasBluetoothHeadset() { - if (Build.VERSION.SDK_INT >= 31 - && ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT) - != PackageManager.PERMISSION_GRANTED) { - return false; - } - BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE); - BluetoothAdapter adapter = manager == null ? null : manager.getAdapter(); - if (adapter == null) { - return false; - } - try { - int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET); - if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) { - return true; + private AudioPacket takeAudioPacket() { + synchronized (audioQueueLock) { + while (started && audioQueue.isEmpty()) { + try { + audioQueueLock.wait(250); + maybeEmitAudioStats(false); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } } - Set bondedDevices = adapter.getBondedDevices(); - return bondedDevices != null && !bondedDevices.isEmpty(); - } catch (SecurityException ignored) { - return false; + return audioQueue.pollFirst(); } } - private boolean hasRecordAudioPermission() { - return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) - == PackageManager.PERMISSION_GRANTED; + private void resetAudioStats() { + queuedPackets = 0; + queueDroppedPackets = 0; + decodedSamples = 0; + latePackets = 0; + wrongArmPackets = 0; + lastPacketArrivalMs = 0; + maxInterPacketMs = 0; + lastStatsAtMs = SystemClock.elapsedRealtime(); + } + + private void maybeEmitAudioStats(boolean force) { + long now = SystemClock.elapsedRealtime(); + if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) { + return; + } + lastStatsAtMs = now; + FaceclawLc3Decoder currentDecoder = lc3Decoder; + long real = currentDecoder == null ? 0 : currentDecoder.getRealPackets(); + long duplicate = currentDecoder == null ? 0 : currentDecoder.getDuplicatePackets(); + long missing = currentDecoder == null ? 0 : currentDecoder.getMissingPackets(); + long decodeErrors = currentDecoder == null ? 0 : currentDecoder.getDecodeErrors(); + String status = "G2 mic packets=" + queuedPackets + + " decoded=" + real + + " missing=" + missing + + " duplicate=" + duplicate + + " late=" + latePackets + + " maxGapMs=" + maxInterPacketMs + + " queueDrop=" + queueDroppedPackets + + " decodeErrors=" + decodeErrors + + " wrongArm=" + wrongArmPackets + + " audioSec=" + String.format(java.util.Locale.US, "%.1f", decodedSamples / (double) SAMPLE_RATE); + Log.i(TAG, status + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS); + emitStatus(status); } private void emitStatus(String status) { @@ -338,4 +362,16 @@ public class FaceclawVoiceController { } mainHandler.post(() -> currentListener.onWakeWord(keyword)); } + + private static final class AudioPacket { + final byte[] data; + final String arm; + final long arrivalMs; + + AudioPacket(byte[] data, String arm, long arrivalMs) { + this.data = data; + this.arm = arm == null ? "?" : arm; + this.arrivalMs = arrivalMs; + } + } } diff --git a/App_Resources/Android/src/main/jniLibs/arm64-v8a/libfaceclaw_lc3.so b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libfaceclaw_lc3.so new file mode 100755 index 00000000..b9eee4f3 Binary files /dev/null and b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libfaceclaw_lc3.so differ diff --git a/App_Resources/Android/src/main/native/faceclaw_lc3_decoder.c b/App_Resources/Android/src/main/native/faceclaw_lc3_decoder.c new file mode 100644 index 00000000..0b4ad337 --- /dev/null +++ b/App_Resources/Android/src/main/native/faceclaw_lc3_decoder.c @@ -0,0 +1,92 @@ +#include +#include +#include + +#include "lc3.h" + +#define G2_FRAME_BYTES 40 +#define G2_FRAMES_PER_PACKET 5 +#define G2_SAMPLES_PER_PACKET 800 + +typedef struct { + void *mem; + lc3_decoder_t decoder; +} faceclaw_lc3_decoder_t; + +JNIEXPORT jlong JNICALL +Java_com_faceclaw_app_FaceclawLc3Decoder_nativeCreate(JNIEnv *env, jclass clazz, jint frame_us, jint sample_rate) { + (void) env; + (void) clazz; + unsigned mem_size = lc3_decoder_size(frame_us, sample_rate); + if (mem_size == 0) { + return 0; + } + + faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) calloc(1, sizeof(faceclaw_lc3_decoder_t)); + if (state == NULL) { + return 0; + } + state->mem = calloc(1, mem_size); + if (state->mem == NULL) { + free(state); + return 0; + } + state->decoder = lc3_setup_decoder(frame_us, sample_rate, 0, state->mem); + if (state->decoder == NULL) { + free(state->mem); + free(state); + return 0; + } + return (jlong) (uintptr_t) state; +} + +JNIEXPORT jint JNICALL +Java_com_faceclaw_app_FaceclawLc3Decoder_nativeDecodePacket(JNIEnv *env, jclass clazz, jlong handle, jbyteArray packet, jshortArray pcm_out) { + (void) clazz; + faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) (uintptr_t) handle; + if (state == NULL || state->decoder == NULL || packet == NULL || pcm_out == NULL) { + return 0; + } + if ((*env)->GetArrayLength(env, packet) < 205 || (*env)->GetArrayLength(env, pcm_out) < G2_SAMPLES_PER_PACKET) { + return 0; + } + + jbyte *packet_bytes = (*env)->GetByteArrayElements(env, packet, NULL); + jshort *pcm = (*env)->GetShortArrayElements(env, pcm_out, NULL); + if (packet_bytes == NULL || pcm == NULL) { + if (packet_bytes != NULL) { + (*env)->ReleaseByteArrayElements(env, packet, packet_bytes, JNI_ABORT); + } + if (pcm != NULL) { + (*env)->ReleaseShortArrayElements(env, pcm_out, pcm, JNI_ABORT); + } + return 0; + } + + int ok = 1; + for (int frame = 0; frame < G2_FRAMES_PER_PACKET; frame++) { + const uint8_t *lc3_frame = (const uint8_t *) packet_bytes + frame * G2_FRAME_BYTES; + int16_t *pcm_frame = (int16_t *) pcm + frame * 160; + int rc = lc3_decode(state->decoder, lc3_frame, G2_FRAME_BYTES, LC3_PCM_FORMAT_S16, pcm_frame, 1); + if (rc < 0) { + ok = 0; + break; + } + } + + (*env)->ReleaseByteArrayElements(env, packet, packet_bytes, JNI_ABORT); + (*env)->ReleaseShortArrayElements(env, pcm_out, pcm, ok ? 0 : JNI_ABORT); + return ok ? G2_SAMPLES_PER_PACKET : 0; +} + +JNIEXPORT void JNICALL +Java_com_faceclaw_app_FaceclawLc3Decoder_nativeDestroy(JNIEnv *env, jclass clazz, jlong handle) { + (void) env; + (void) clazz; + faceclaw_lc3_decoder_t *state = (faceclaw_lc3_decoder_t *) (uintptr_t) handle; + if (state == NULL) { + return; + } + free(state->mem); + free(state); +} diff --git a/app/g2/android-permissions.ts b/app/g2/android-permissions.ts index c58d2247..77ab680f 100644 --- a/app/g2/android-permissions.ts +++ b/app/g2/android-permissions.ts @@ -100,3 +100,4 @@ export async function ensureBlePermissions(): Promise { export async function ensureVoicePermissions(): Promise { await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control"); } + diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 323165ca..7122082a 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -487,6 +487,8 @@ class DashboardController { private async setVoiceControlEnabled(enabled: boolean): Promise { if (enabled) { try { + // Treat Android mic permission as the consent gate for voice control, + // even when the current source is the G2 mic over BLE. await ensureVoicePermissions(); } catch (error) { this.appendLog(`voice control permission failed: ${this.formatError(error)}`); @@ -671,7 +673,7 @@ class DashboardController { if (!this.communicator) return; if (!isDashboardVoiceControlEnabled()) return; this.updateConnectedForegroundNotification(); - voiceControlBridge.start(); + voiceControlBridge.start(this.communicator.getNativeCommunicator()); } private setStopwatchRenderActive(active: boolean): void { diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 2d841bcd..cf283a81 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -216,6 +216,10 @@ export class FaceclawCommunicatorBridge { return () => this.frameMetricsListeners.delete(listener); } + getNativeCommunicator(): any { + return this.communicator; + } + waitForNextFrameMetrics(timeoutMs: number): Promise { const delayMs = Math.max(1, Math.round(nonNegativeNumber(timeoutMs))); return new Promise((resolve) => { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 91b190ae..db5e3a62 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -25,9 +25,12 @@ export class FaceclawVoiceControlBridge { return () => this.wakeWordListeners.delete(listener); } - start(): void { + start(communicator?: any): void { if (this.started || !global.isAndroid) return; this.ensureController(); + if (communicator) { + this.controller?.setCommunicator(communicator); + } this.started = true; this.controller?.start(); }