diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..cffc8d6a --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,40 @@ +name: Deploy website + +# Publishes website/ to GitHub Pages. Requires the repository's +# Settings > Pages > Source to be set to "GitHub Actions". + +on: + push: + branches: [main] + paths: + - 'website/**' + - 'PRIVACY' + - 'scripts/sync-site.mjs' + - '.github/workflows/pages.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v4 + - name: Check generated blocks are up to date + run: node scripts/sync-site.mjs --check + - uses: actions/configure-pages@v5 + - uses: actions/upload-pages-artifact@v3 + with: + path: website + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index dc56b331..5f98a77a 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,5 @@ old faceclaw_settings.xml # Developer configs -build.sh -build_and_run.sh +build_paths.sh + diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index cda991f6..0e24319a 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -12,6 +12,13 @@ dependencies { // code" entry. Scanner UI and camera live in Play Services, so Faceclaw // needs no camera permission and no third-party scanner app. implementation 'com.google.android.gms:play-services-code-scanner:16.1.0' + // Wearable Data Layer (MessageClient / DataClient / CapabilityClient) for + // the Wear OS watch remote; see FaceclawWearBridge and wear/. + implementation 'com.google.android.gms:play-services-wearable:19.0.0' + // On-device language identification and translation for the Microphones + // app's captions (translation models download on demand, per language). + implementation 'com.google.mlkit:language-id:17.0.6' + implementation 'com.google.mlkit:translate:17.0.3' } def faceclawSherpaOnnxVersion = "1.13.0" diff --git a/App_Resources/Android/src/main/AndroidManifest.xml b/App_Resources/Android/src/main/AndroidManifest.xml index 48dda00b..efec7b97 100644 --- a/App_Resources/Android/src/main/AndroidManifest.xml +++ b/App_Resources/Android/src/main/AndroidManifest.xml @@ -77,6 +77,7 @@ android:name="com.tns.NativeScriptActivity" android:label="@string/title_activity_kimera" android:configChanges="keyboard|keyboardHidden|orientation|screenSize|smallestScreenSize|screenLayout|locale|uiMode" + android:windowSoftInputMode="adjustResize" android:theme="@style/LaunchScreenTheme" android:hardwareAccelerated="true" android:launchMode="singleTask" @@ -110,9 +111,34 @@ + + + + + + + + + + + + + diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java new file mode 100644 index 00000000..0aab2ff6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java @@ -0,0 +1,242 @@ +package com.faceclaw.app; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.media.MediaMuxer; +import android.util.Log; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A + * container, cutting speech recordings to roughly a tenth of their WAV size. + * Runs on its own thread; the source WAV is deleted on success when requested. + */ +public class FaceclawAudioTranscoder { + private static final String TAG = "FaceclawTranscode"; + private static final long CODEC_TIMEOUT_US = 10_000; + + public interface Listener { + void onDone(String outputPath, long outputBytes); + + void onError(String message); + } + + private final String wavPath; + private final String outputPath; + private final int bitrateBps; + private final boolean deleteSourceOnSuccess; + private final Listener listener; + + public FaceclawAudioTranscoder( + String wavPath, + String outputPath, + int bitrateBps, + boolean deleteSourceOnSuccess, + Listener listener) { + this.wavPath = wavPath; + this.outputPath = outputPath; + this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000; + this.deleteSourceOnSuccess = deleteSourceOnSuccess; + this.listener = listener; + } + + public void start() { + Thread thread = new Thread(this::run, "FaceclawTranscode"); + thread.setPriority(Thread.MIN_PRIORITY); + thread.start(); + } + + private void run() { + try { + transcode(); + long size = new File(outputPath).length(); + if (deleteSourceOnSuccess) { + new File(wavPath).delete(); + } + if (listener != null) { + listener.onDone(outputPath, size); + } + } catch (Throwable t) { + Log.w(TAG, "transcode failed for " + wavPath, t); + new File(outputPath).delete(); + if (listener != null) { + listener.onError(t.getMessage() == null ? t.toString() : t.getMessage()); + } + } + } + + private void transcode() throws IOException { + WavInfo wav = readWavHeader(wavPath); + MediaFormat format = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels); + format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps); + format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024); + + MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC); + MediaMuxer muxer = null; + FileInputStream input = null; + try { + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + codec.start(); + muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + input = new FileInputStream(wavPath); + long skipped = 0; + while (skipped < wav.dataOffset) { + long step = input.skip(wav.dataOffset - skipped); + if (step <= 0) { + throw new IOException("could not seek past WAV header"); + } + skipped += step; + } + + MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + byte[] chunk = new byte[8192]; + int track = -1; + boolean muxerStarted = false; + boolean inputDone = false; + boolean outputDone = false; + long remaining = wav.dataBytes; + long presentationUs = 0; + long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2; + + while (!outputDone) { + if (!inputDone) { + int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US); + if (inIndex >= 0) { + ByteBuffer inBuf = codec.getInputBuffer(inIndex); + int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity())); + int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1; + if (read <= 0) { + codec.queueInputBuffer(inIndex, 0, 0, presentationUs, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + } else { + inBuf.clear(); + inBuf.put(chunk, 0, read); + codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0); + remaining -= read; + presentationUs += read * 1_000_000L / bytesPerSecond; + } + } + } + int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US); + if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + track = muxer.addTrack(codec.getOutputFormat()); + muxer.start(); + muxerStarted = true; + } else if (outIndex >= 0) { + if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { + if (!muxerStarted) { + throw new IOException("encoder produced data before format"); + } + ByteBuffer outBuf = codec.getOutputBuffer(outIndex); + outBuf.position(info.offset); + outBuf.limit(info.offset + info.size); + muxer.writeSampleData(track, outBuf, info); + } + codec.releaseOutputBuffer(outIndex, false); + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + } + } + } finally { + if (input != null) { + try { + input.close(); + } catch (IOException ignored) { + } + } + try { + codec.stop(); + } catch (Throwable ignored) { + } + codec.release(); + if (muxer != null) { + try { + muxer.stop(); + } catch (Throwable ignored) { + } + muxer.release(); + } + } + } + + private static final class WavInfo { + int sampleRate; + int channels; + long dataOffset; + long dataBytes; + } + + /** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */ + private static WavInfo readWavHeader(String path) throws IOException { + try (FileInputStream in = new FileInputStream(path)) { + byte[] head = new byte[12]; + if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') { + throw new IOException("not a WAV file"); + } + WavInfo info = new WavInfo(); + long offset = 12; + byte[] chunkHead = new byte[8]; + while (in.read(chunkHead) == 8) { + ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN); + int id = b.getInt(); + long size = b.getInt() & 0xffffffffL; + offset += 8; + if (id == 0x20746d66) { // "fmt " + byte[] fmt = new byte[(int) Math.min(size, 16)]; + if (in.read(fmt) != fmt.length) { + throw new IOException("truncated fmt chunk"); + } + ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN); + int audioFormat = f.getShort(); + info.channels = f.getShort(); + info.sampleRate = f.getInt(); + f.getInt(); + f.getShort(); + int bits = f.getShort(); + if (audioFormat != 1 || bits != 16) { + throw new IOException("only 16-bit PCM WAV is supported"); + } + long skip = size - fmt.length; + while (skip > 0) { + long step = in.skip(skip); + if (step <= 0) { + break; + } + skip -= step; + } + offset += size; + } else if (id == 0x61746164) { // "data" + info.dataOffset = offset; + // A zero size means the header was never patched (crash + // mid-recording); recover the length from the file size. + info.dataBytes = size > 0 ? size : new File(path).length() - offset; + break; + } else { + long skip = size; + while (skip > 0) { + long step = in.skip(skip); + if (step <= 0) { + break; + } + skip -= step; + } + offset += size; + } + } + if (info.sampleRate == 0 || info.dataOffset == 0) { + throw new IOException("missing fmt or data chunk"); + } + return info; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index b325dbeb..aaded920 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -60,6 +60,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { new java.util.concurrent.CopyOnWriteArrayList<>(); private final java.util.List compassListeners = new java.util.concurrent.CopyOnWriteArrayList<>(); + private final java.util.List micStatusListeners = + new java.util.concurrent.CopyOnWriteArrayList<>(); + private volatile String lastFirmwareCapabilities = ""; private volatile Thread workerThread; private volatile boolean running; private volatile boolean userDisconnectRequested; @@ -389,6 +392,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + /** + * Whether the glasses mic is enabled right now. The enable lives in the + * current EvenHub session, so it dies with a transport drop, the charging + * case, or a suspend — silently, from the phone's point of view. Callers + * that track a capture across those events must check this rather than + * assume their earlier enable still holds. + */ + public boolean isAudioCaptureActive() { + synchronized (lock) { + return running && sessionReady && !shutdownRequested && audioCaptureActive; + } + } + /** * Acquire/renew or release CFW's volatile wake-takeover lease on both * arms. Delivery (not a protocol ACK) is awaited so a caller can ensure @@ -743,6 +759,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + /** The CFW capability token string from the last firmware-info read ("" before one arrives). */ + public String getFirmwareCapabilities() { + return lastFirmwareCapabilities; + } + + public void addMicStatusListener(FaceclawMicStatusListener listener) { + if (listener != null) { + micStatusListeners.add(listener); + } + } + + public void removeMicStatusListener(FaceclawMicStatusListener listener) { + if (listener != null) { + micStatusListeners.remove(listener); + } + } + + private void emitMicStatus(byte[] body, String address) { + if (micStatusListeners.isEmpty()) { + return; + } + String arm = address.equalsIgnoreCase(leftAddress) ? "L" + : address.equalsIgnoreCase(rightAddress) ? "R" : "?"; + byte[] copy = java.util.Arrays.copyOf(body, body.length); + mainHandler.post(() -> { + for (FaceclawMicStatusListener micListener : micStatusListeners) { + try { + micListener.onMicStatus(copy, arm); + } catch (Throwable t) { + Log.w(TAG, "mic status listener failed", t); + } + } + }); + } + + /** + * Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings + * field-103 write to both temples, or to a single one. Fire-and-forget: + * the firmware answers with a field-104 status notify per temple, which + * arrives through addMicStatusListener. + */ + public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) { + if (record == null || record.length < 4) { + return; + } + synchronized (lock) { + if (!running || !sessionReady) { + logLine("skip mic control (" + label + "); session not ready"); + return; + } + if (rightTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false)); + } + if (leftTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true)); + } + logLine("queue mic control " + label); + } + interruptibleSleep.interrupt(); + } + + /** + * Forward render-characteristic audio packets to the listener WITHOUT + * sending the stock EvenHub audio-control enable. Used for the CFW + * mic_control streaming path, where capture is armed through settings + * field 103 and the temples emit 'SM' frames on the same characteristic + * that stock mono LC3 uses. Returns false when no session is up. + */ + public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) { + if (listener == null) { + throw new IllegalArgumentException("listener is required"); + } + synchronized (lock) { + if (!running || !sessionReady || shutdownRequested) { + logLine("skip G2 audio forwarding; session not ready"); + return false; + } + audioPacketListener = listener; + audioCaptureActive = true; + logLine("G2 audio forwarding enabled"); + } + return true; + } + + public void stopG2AudioForwarding() { + synchronized (lock) { + audioPacketListener = null; + audioCaptureActive = false; + logLine("G2 audio forwarding disabled"); + } + } + public void addCompassListener(FaceclawCompassListener listener) { if (listener != null) { compassListeners.add(listener); @@ -777,12 +885,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { * foreground), including worker-app frames the TS side never sees. */ public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) { + return getCompositePreviewBitmap(brightenGamma, false); + } + + /** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */ + public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) { SurfaceCompositor.Composite composite = compositor.previewComposite(); if (composite == null) { return null; } return PreviewBitmapUtil.fromGray( - java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma); + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green); } /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ @@ -1433,6 +1546,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { 0 ); } + if (!faceclawWakeNotification + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // CFW mic status (field 104) rides both standalone pushes and + // settings read acks, from each temple on its own link. + byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb); + if (micStatus != null) { + emitMicStatus(micStatus, address); + } + } if (!faceclawWakeNotification && decodedWearState < 0 && frame.ok @@ -2716,6 +2839,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload); if (firmwareInfo != null) { + lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities; cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11"); textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12") && hasCapability(firmwareInfo.capabilities, "texstr14"); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java new file mode 100644 index 00000000..488d2bca --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java @@ -0,0 +1,446 @@ +package com.faceclaw.app; + +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import com.k2fsa.sherpa.onnx.FeatureConfig; +import com.k2fsa.sherpa.onnx.OfflineModelConfig; +import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; +import com.k2fsa.sherpa.onnx.OfflineRecognizer; +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; +import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; +import com.k2fsa.sherpa.onnx.OfflineStream; + +import java.io.File; +import java.util.ArrayDeque; +import java.util.Arrays; + +/** + * Continuous captioning over the decoded mic PCM: segments speech into + * utterances with an adaptive energy gate, transcribes each utterance with + * the on-device Moonshine model, and attaches a speaker voice-print per + * utterance. PCM is pushed in from the TS side (which owns mic arbitration + * and any beam-direction gating), so this engine has no BLE dependencies. + * + * Utterance boundaries are measured on the sample clock, not the wall clock: + * BLE delivers mic packets in bursts, so elapsed real time overestimates the + * audio heard. The TS side converts startMs/endMs to wall-clock times using + * the engine start timestamp. + */ +public class FaceclawCaptionEngine { + private static final String TAG = "FaceclawCaptions"; + private static final int SAMPLE_RATE = 16000; + private static final int FEATURE_DIM = 80; + private static final int MAX_QUEUE_PACKETS = 200; + + // Segmentation: RMS thresholds relative to a rolling noise floor, in the + // same spirit as FaceclawVoiceController.EndpointDetector but recurring. + private static final double ONSET_FACTOR = 3.0; + private static final double RELEASE_FACTOR = 1.8; + private static final double MIN_RMS = 220.0; + private static final double NOISE_EMA_ALPHA = 0.05; + private static final int PRE_ROLL_MS = 400; + private static final int DEFAULT_SILENCE_MS = 800; + private static final int MIN_UTTERANCE_MS = 350; + private static final int MAX_UTTERANCE_MS = 15000; + // Moonshine v2 fails past ~9.1 s of input; decode long utterances in + // segments cut at the quietest window (mirrors FaceclawVoiceController). + private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8; + private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2; + private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000; + private static final float NORMALIZE_TARGET_PEAK = 0.9f; + private static final float NORMALIZE_MAX_GAIN = 30f; + // Voice-prints degrade on very long inputs; embed at most the first 10 s. + private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10; + + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + private final Object lock = new Object(); + private final Object queueLock = new Object(); + private final ArrayDeque queue = new ArrayDeque<>(); + + private volatile FaceclawCaptionEngineListener listener; + private volatile String asrModelDir; + private volatile String speakerModelPath; + private volatile int silenceMs = DEFAULT_SILENCE_MS; + private Thread workerThread; + private volatile boolean started; + + private OfflineRecognizer recognizer; + private FaceclawSpeakerId speakerId; + + // Segmentation state (worker thread only). + private long totalSamples; + private double noiseFloor; + private boolean inUtterance; + private long utteranceStartSample; + private long silenceRunSamples; + private double utterancePeakRms; + private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)]; + private int utteranceLength; + private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000]; + private int preRollLength; + + public void setListener(FaceclawCaptionEngineListener listener) { + this.listener = listener; + } + + /** Directory holding the Moonshine model files, or null to disable ASR. */ + public void setAsrModelDir(String dir) { + this.asrModelDir = dir; + } + + /** Speaker-embedding ONNX model path, or null to disable voice-prints. */ + public void setSpeakerModelPath(String path) { + this.speakerModelPath = path; + } + + public void setSilenceMs(int ms) { + this.silenceMs = Math.max(200, Math.min(3000, ms)); + } + + public void start() { + synchronized (lock) { + if (started) { + return; + } + started = true; + workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine"); + workerThread.start(); + } + } + + public void stop() { + Thread threadToJoin; + synchronized (lock) { + if (!started) { + return; + } + started = false; + threadToJoin = workerThread; + workerThread = null; + } + synchronized (queueLock) { + queueLock.notifyAll(); + } + if (threadToJoin != null) { + threadToJoin.interrupt(); + if (Thread.currentThread() != threadToJoin) { + try { + threadToJoin.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } + + /** Push decoded 16 kHz mono S16LE PCM (any chunking). */ + public void acceptPcm(byte[] pcm16le) { + if (!started || pcm16le == null || pcm16le.length < 2) { + return; + } + synchronized (queueLock) { + if (queue.size() >= MAX_QUEUE_PACKETS) { + queue.removeFirst(); + } + queue.addLast(pcm16le); + queueLock.notifyAll(); + } + } + + private void runLoop() { + try { + loadModels(); + resetSegmentation(); + while (started && !Thread.currentThread().isInterrupted()) { + byte[] chunk = takeChunk(); + if (chunk == null) { + continue; + } + processChunk(chunk); + } + // Flush a trailing utterance so its text isn't lost on stop. + if (inUtterance && utteranceLength > 0) { + finalizeUtterance(); + } + } catch (Throwable t) { + Log.e(TAG, "caption engine failed", t); + emitStatus("Captions failed: " + t.getMessage()); + } finally { + releaseModels(); + } + } + + private void loadModels() { + String modelDir = asrModelDir; + if (modelDir != null && new File(modelDir, "tokens.txt").exists()) { + emitStatus("Loading caption model..."); + recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder() + .setFeatureConfig(FeatureConfig.builder() + .setSampleRate(SAMPLE_RATE) + .setFeatureDim(FEATURE_DIM) + .build()) + .setModelConfig(OfflineModelConfig.builder() + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) + .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) + .build()) + .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) + .setNumThreads(1) + .build()) + .build()); + } else { + recognizer = null; + } + String embedModel = speakerModelPath; + if (embedModel != null && new File(embedModel).exists()) { + speakerId = new FaceclawSpeakerId(embedModel); + speakerId.ensureLoaded(); + } else { + speakerId = null; + } + emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)..."); + } + + private void releaseModels() { + if (recognizer != null) { + recognizer.release(); + recognizer = null; + } + if (speakerId != null) { + speakerId.close(); + speakerId = null; + } + } + + private byte[] takeChunk() { + synchronized (queueLock) { + while (started && queue.isEmpty()) { + try { + queueLock.wait(250); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } + return queue.pollFirst(); + } + } + + private void resetSegmentation() { + totalSamples = 0; + noiseFloor = 0; + inUtterance = false; + utteranceLength = 0; + preRollLength = 0; + silenceRunSamples = 0; + utterancePeakRms = 0; + } + + private void processChunk(byte[] chunk) { + int count = chunk.length / 2; + short[] pcm = new short[count]; + double sumSquares = 0; + for (int i = 0; i < count; i++) { + short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8)); + pcm[i] = s; + sumSquares += (double) s * s; + } + double rms = Math.sqrt(sumSquares / Math.max(1, count)); + totalSamples += count; + + if (!inUtterance) { + // Track the noise floor only while idle so speech doesn't raise it. + noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA; + double threshold = Math.max(noiseFloor, MIN_RMS); + if (rms >= threshold * ONSET_FACTOR) { + inUtterance = true; + utteranceLength = 0; + silenceRunSamples = 0; + utterancePeakRms = rms; + appendUtterance(preRoll, preRollLength); + utteranceStartSample = Math.max(0, totalSamples - count - preRollLength); + appendUtterance(pcm, count); + emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE); + } else { + appendPreRoll(pcm, count); + } + return; + } + + appendUtterance(pcm, count); + utterancePeakRms = Math.max(utterancePeakRms, rms); + double threshold = Math.max(noiseFloor, MIN_RMS); + if (rms < threshold * RELEASE_FACTOR) { + silenceRunSamples += count; + } else { + silenceRunSamples = 0; + } + long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE; + boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs; + if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) { + finalizeUtterance(); + inUtterance = false; + preRollLength = 0; + } + } + + private void appendPreRoll(short[] pcm, int count) { + // Keep the last PRE_ROLL_MS of idle audio so onset consonants survive. + if (count >= preRoll.length) { + System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length); + preRollLength = preRoll.length; + return; + } + int keep = Math.min(preRollLength, preRoll.length - count); + System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep); + System.arraycopy(pcm, 0, preRoll, keep, count); + preRollLength = keep + count; + } + + private void appendUtterance(short[] pcm, int count) { + int room = utterance.length - utteranceLength; + int copied = Math.min(room, count); + if (copied > 0) { + System.arraycopy(pcm, 0, utterance, utteranceLength, copied); + utteranceLength += copied; + } + } + + private void finalizeUtterance() { + int length = utteranceLength; + long startMs = utteranceStartSample * 1000 / SAMPLE_RATE; + long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE; + if (endMs - startMs < MIN_UTTERANCE_MS) { + return; + } + String text = recognizeUtterance(length); + float[] embedding = embedUtterance(length); + emitUtterance(text, embedding, startMs, endMs, utterancePeakRms); + } + + private String recognizeUtterance(int length) { + if (recognizer == null || length <= 0) { + return ""; + } + StringBuilder joined = new StringBuilder(); + int offset = 0; + while (offset < length) { + int remaining = length - offset; + int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES); + if (remaining > DECODE_SEGMENT_MAX_SAMPLES) { + segment = findQuietCut(offset, segment); + } + String part = recognizeRange(offset, segment); + if (part.length() > 0) { + if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) { + joined.append(' '); + } + joined.append(part); + } + offset += segment; + } + return joined.toString().trim(); + } + + /** + * End a decode segment at the center of the quietest window near its end + * so the cut lands between words (same approach as the PTT controller). + */ + private int findQuietCut(int offset, int segment) { + int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES); + int win = CUT_WINDOW_SAMPLES; + if (segment - searchStart <= win) { + return segment; + } + double sum = 0; + for (int i = searchStart; i < searchStart + win; i++) { + double s = utterance[offset + i]; + sum += s * s; + } + double best = sum; + int bestStart = searchStart; + for (int start = searchStart + 1; start + win <= segment; start++) { + double dropped = utterance[offset + start - 1]; + double added = utterance[offset + start + win - 1]; + sum += added * added - dropped * dropped; + if (sum < best) { + best = sum; + bestStart = start; + } + } + return bestStart + win / 2; + } + + private String recognizeRange(int offset, int count) { + float[] samples = new float[count]; + float peak = 0f; + for (int i = 0; i < count; i++) { + float v = utterance[offset + i] / 32768.0f; + samples[i] = v; + float a = Math.abs(v); + if (a > peak) { + peak = a; + } + } + if (peak > 0f) { + float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN); + if (gain > 1f) { + for (int i = 0; i < count; i++) { + samples[i] *= gain; + } + } + } + OfflineStream stream = recognizer.createStream(); + try { + stream.acceptWaveform(samples, SAMPLE_RATE); + recognizer.decode(stream); + OfflineRecognizerResult result = recognizer.getResult(stream); + String raw = result == null ? "" : result.getText(); + return raw == null ? "" : raw.trim(); + } finally { + stream.release(); + } + } + + private float[] embedUtterance(int length) { + FaceclawSpeakerId currentSpeakerId = speakerId; + if (currentSpeakerId == null || length <= 0) { + return null; + } + int count = Math.min(length, EMBED_MAX_SAMPLES); + byte[] le = new byte[count * 2]; + for (int i = 0; i < count; i++) { + short s = utterance[i]; + le[i * 2] = (byte) (s & 0xff); + le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); + } + return currentSpeakerId.embed(le, SAMPLE_RATE); + } + + private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) { + FaceclawCaptionEngineListener currentListener = listener; + if (currentListener == null) { + return; + } + float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length); + mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms)); + } + + private void emitSpeechStart(long startMs) { + FaceclawCaptionEngineListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onSpeechStart(startMs)); + } + + private void emitStatus(String status) { + FaceclawCaptionEngineListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onStatus(status)); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngineListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngineListener.java new file mode 100644 index 00000000..24b58a85 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngineListener.java @@ -0,0 +1,17 @@ +package com.faceclaw.app; + +public interface FaceclawCaptionEngineListener { + /** + * One finished utterance. text may be empty when no ASR model is loaded + * or recognition produced nothing; embedding is the L2-normalized + * voice-print or null when the speaker model is unavailable or the + * utterance was too short. startMs/endMs are on the engine's sample + * clock (milliseconds of audio since start()). + */ + void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms); + + /** A speech onset was detected; the utterance is now being collected. */ + void onSpeechStart(long startMs); + + void onStatus(String status); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java new file mode 100644 index 00000000..38f723fe --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java @@ -0,0 +1,778 @@ +package com.faceclaw.app; + +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteOpenHelper; +import android.util.Base64; +import android.util.Log; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; + +/** + * Persistent store for caption sessions, transcript segments, and speaker + * voice profiles, backing the Microphones app and its phone-side review UI. + * All complex values cross the JS bridge as JSON strings; voice-print + * embeddings are float32 LE blobs, exposed as base64. + */ +public class FaceclawConversationStore extends SQLiteOpenHelper { + private static final String TAG = "FaceclawConvStore"; + private static final String DB_NAME = "faceclaw-conversations.db"; + private static final int DB_VERSION = 2; + + private static FaceclawConversationStore instance; + + public static synchronized FaceclawConversationStore getInstance(Context context) { + if (instance == null) { + instance = new FaceclawConversationStore(context.getApplicationContext()); + } + return instance; + } + + private FaceclawConversationStore(Context context) { + super(context, DB_NAME, null, DB_VERSION); + } + + @Override + public void onCreate(SQLiteDatabase db) { + db.execSQL("CREATE TABLE speakers (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "name TEXT NOT NULL," + + "color TEXT NOT NULL DEFAULT '#4FC3F7'," + + "is_wearer INTEGER NOT NULL DEFAULT 0," + + "embedding BLOB," + + "embedding_count INTEGER NOT NULL DEFAULT 0," + + "created_at INTEGER NOT NULL," + + "last_heard_at INTEGER," + + "tag TEXT," + + "last_recap TEXT," + + "action_items TEXT," + + "facts TEXT," + + "insights_updated_at INTEGER," + + "insights_session_id INTEGER)"); + db.execSQL("CREATE TABLE sessions (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "started_at INTEGER NOT NULL," + + "ended_at INTEGER," + + "title TEXT," + + "audio_path TEXT," + + "audio_codec TEXT," + + "avg_sentiment REAL," + + "segment_count INTEGER NOT NULL DEFAULT 0)"); + db.execSQL("CREATE TABLE segments (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "session_id INTEGER NOT NULL," + + "speaker_id INTEGER," + + "started_at INTEGER NOT NULL," + + "ended_at INTEGER NOT NULL," + + "audio_offset_ms INTEGER," + + "text TEXT NOT NULL," + + "lang TEXT," + + "translation TEXT," + + "translation_lang TEXT," + + "sentiment REAL," + + "emotion TEXT," + + "search_meta TEXT," + + "angle INTEGER," + + "embedding BLOB)"); + db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)"); + db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)"); + db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)"); + db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)"); + } + + @Override + public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { + if (oldVersion < 2) { + // v2: per-speaker conversation insights (recap of the most recent + // conversation, its action items, and accumulated inferred facts). + db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT"); + db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT"); + db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT"); + db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER"); + db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER"); + } + } + + // ---- sessions ---- + + public long startSession(long startedAtMs, String title) { + ContentValues values = new ContentValues(); + values.put("started_at", startedAtMs); + values.put("title", title == null || title.isEmpty() ? null : title); + return getWritableDatabase().insert("sessions", null, values); + } + + public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) { + ContentValues values = new ContentValues(); + values.put("ended_at", endedAtMs); + if (audioPath != null && !audioPath.isEmpty()) { + values.put("audio_path", audioPath); + values.put("audio_codec", audioCodec); + } + values.put("avg_sentiment", avgSentiment); + getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)}); + } + + public void setSessionAudio(long sessionId, String audioPath, String audioCodec) { + ContentValues values = new ContentValues(); + values.put("audio_path", audioPath); + values.put("audio_codec", audioCodec); + getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)}); + } + + public String getSessionAudioPath(long sessionId) { + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) { + if (c.moveToFirst()) { + return c.isNull(0) ? "" : c.getString(0); + } + } + return ""; + } + + /** + * Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs, + * speakerId, emotion (matches any segment emotion in the session), query + * (LIKE over title and segment text/search metadata), limit. + */ + public String querySessions(String filterJson) { + try { + JSONObject filter = filterJson == null || filterJson.isEmpty() + ? new JSONObject() : new JSONObject(filterJson); + StringBuilder sql = new StringBuilder( + "SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec," + + " s.avg_sentiment, s.segment_count FROM sessions s"); + List args = new ArrayList<>(); + List where = new ArrayList<>(); + boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query"); + if (joinSegments) { + sql.append(" LEFT JOIN segments g ON g.session_id = s.id"); + } + if (filter.has("sinceMs")) { + where.add("s.started_at >= ?"); + args.add(String.valueOf(filter.getLong("sinceMs"))); + } + if (filter.has("untilMs")) { + where.add("s.started_at <= ?"); + args.add(String.valueOf(filter.getLong("untilMs"))); + } + if (filter.has("speakerId")) { + where.add("g.speaker_id = ?"); + args.add(String.valueOf(filter.getLong("speakerId"))); + } + if (filter.has("emotion")) { + where.add("g.emotion = ?"); + args.add(filter.getString("emotion")); + } + if (filter.has("query")) { + String like = "%" + filter.getString("query") + "%"; + where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)"); + args.add(like); + args.add(like); + args.add(like); + } + if (!where.isEmpty()) { + sql.append(" WHERE ").append(String.join(" AND ", where)); + } + sql.append(" ORDER BY s.started_at DESC"); + int limit = filter.optInt("limit", 200); + sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000))); + + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + long id = c.getLong(0); + row.put("id", id); + row.put("startedAt", c.getLong(1)); + row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2)); + row.put("title", c.isNull(3) ? "" : c.getString(3)); + row.put("audioPath", c.isNull(4) ? "" : c.getString(4)); + row.put("audioCodec", c.isNull(5) ? "" : c.getString(5)); + row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6)); + row.put("segmentCount", c.getLong(7)); + row.put("speakers", sessionSpeakersJson(id)); + out.put(row); + } + } + return out.toString(); + } catch (JSONException e) { + Log.w(TAG, "querySessions failed", e); + return "[]"; + } + } + + private JSONArray sessionSpeakersJson(long sessionId) throws JSONException { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT DISTINCT p.id, p.name, p.color FROM segments g" + + " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?", + new String[]{String.valueOf(sessionId)})) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("name", c.getString(1)); + row.put("color", c.getString(2)); + out.put(row); + } + } + return out; + } + + // ---- segments ---- + + /** + * Insert one transcript segment. json fields: sessionId, speakerId?, + * startedAt, endedAt, audioOffsetMs?, text, lang?, translation?, + * translationLang?, sentiment?, emotion?, searchMeta?, angle?, + * embeddingBase64?. Returns the new row id, or -1. + */ + public long insertSegment(String json) { + try { + JSONObject s = new JSONObject(json); + ContentValues values = new ContentValues(); + long sessionId = s.getLong("sessionId"); + values.put("session_id", sessionId); + if (s.has("speakerId")) { + values.put("speaker_id", s.getLong("speakerId")); + } + values.put("started_at", s.getLong("startedAt")); + values.put("ended_at", s.getLong("endedAt")); + if (s.has("audioOffsetMs")) { + values.put("audio_offset_ms", s.getLong("audioOffsetMs")); + } + values.put("text", s.getString("text")); + putOptString(values, s, "lang", "lang"); + putOptString(values, s, "translation", "translation"); + putOptString(values, s, "translationLang", "translation_lang"); + if (s.has("sentiment")) { + values.put("sentiment", s.getDouble("sentiment")); + } + putOptString(values, s, "emotion", "emotion"); + putOptString(values, s, "searchMeta", "search_meta"); + if (s.has("angle")) { + values.put("angle", s.getInt("angle")); + } + if (s.has("embeddingBase64")) { + values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP)); + } + SQLiteDatabase db = getWritableDatabase(); + long id = db.insert("segments", null, values); + db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?", + new Object[]{sessionId}); + return id; + } catch (Throwable t) { + Log.w(TAG, "insertSegment failed", t); + return -1; + } + } + + private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column) + throws JSONException { + if (s.has(jsonKey) && !s.isNull(jsonKey)) { + values.put(column, s.getString(jsonKey)); + } + } + + /** Update mutable fields of a segment (same keys as insertSegment). */ + public void updateSegment(long segmentId, String json) { + try { + JSONObject s = new JSONObject(json); + ContentValues values = new ContentValues(); + if (s.has("speakerId")) { + if (s.isNull("speakerId")) { + values.putNull("speaker_id"); + } else { + values.put("speaker_id", s.getLong("speakerId")); + } + } + if (s.has("text")) { + values.put("text", s.getString("text")); + } + putOptString(values, s, "lang", "lang"); + putOptString(values, s, "translation", "translation"); + putOptString(values, s, "translationLang", "translation_lang"); + if (s.has("sentiment")) { + values.put("sentiment", s.getDouble("sentiment")); + } + putOptString(values, s, "emotion", "emotion"); + putOptString(values, s, "searchMeta", "search_meta"); + if (values.size() > 0) { + getWritableDatabase().update("segments", values, "id=?", + new String[]{String.valueOf(segmentId)}); + } + } catch (Throwable t) { + Log.w(TAG, "updateSegment failed", t); + } + } + + public String querySegments(long sessionId) { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," + + " translation, translation_lang, sentiment, emotion, angle" + + " FROM segments WHERE session_id=? ORDER BY started_at ASC", + new String[]{String.valueOf(sessionId)})) { + while (c.moveToNext()) { + out.put(segmentRowJson(c)); + } + } catch (JSONException e) { + Log.w(TAG, "querySegments failed", e); + } + return out.toString(); + } + + private JSONObject segmentRowJson(Cursor c) throws JSONException { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1)); + row.put("startedAt", c.getLong(2)); + row.put("endedAt", c.getLong(3)); + row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4)); + row.put("text", c.getString(5)); + row.put("lang", c.isNull(6) ? "" : c.getString(6)); + row.put("translation", c.isNull(7) ? "" : c.getString(7)); + row.put("translationLang", c.isNull(8) ? "" : c.getString(8)); + row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9)); + row.put("emotion", c.isNull(10) ? "" : c.getString(10)); + row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11)); + return row; + } + + /** + * Search segments across sessions. filterJson: query?, emotion?, + * speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and + * the sentiment search metadata. + */ + public String searchSegments(String filterJson) { + try { + JSONObject filter = filterJson == null || filterJson.isEmpty() + ? new JSONObject() : new JSONObject(filterJson); + StringBuilder sql = new StringBuilder( + "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," + + " translation, translation_lang, sentiment, emotion, angle, session_id" + + " FROM segments"); + List args = new ArrayList<>(); + List where = new ArrayList<>(); + if (filter.has("query") && !filter.getString("query").isEmpty()) { + String like = "%" + filter.getString("query") + "%"; + where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)"); + args.add(like); + args.add(like); + args.add(like); + } + if (filter.has("emotion")) { + where.add("emotion = ?"); + args.add(filter.getString("emotion")); + } + if (filter.has("speakerId")) { + where.add("speaker_id = ?"); + args.add(String.valueOf(filter.getLong("speakerId"))); + } + if (filter.has("sinceMs")) { + where.add("started_at >= ?"); + args.add(String.valueOf(filter.getLong("sinceMs"))); + } + if (filter.has("untilMs")) { + where.add("started_at <= ?"); + args.add(String.valueOf(filter.getLong("untilMs"))); + } + if (!where.isEmpty()) { + sql.append(" WHERE ").append(String.join(" AND ", where)); + } + sql.append(" ORDER BY started_at DESC LIMIT ") + .append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000))); + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) { + while (c.moveToNext()) { + JSONObject row = segmentRowJson(c); + row.put("sessionId", c.getLong(12)); + out.put(row); + } + } + return out.toString(); + } catch (JSONException e) { + Log.w(TAG, "searchSegments failed", e); + return "[]"; + } + } + + /** Per-segment voice-print embeddings of a session (for re-diarization). */ + public String querySegmentEmbeddings(long sessionId) { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL", + new String[]{String.valueOf(sessionId)})) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP)); + out.put(row); + } + } catch (JSONException e) { + Log.w(TAG, "querySegmentEmbeddings failed", e); + } + return out.toString(); + } + + // ---- speakers ---- + + public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) { + ContentValues values = new ContentValues(); + values.put("name", name); + values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color); + values.put("is_wearer", isWearer ? 1 : 0); + values.put("created_at", System.currentTimeMillis()); + values.put("last_heard_at", System.currentTimeMillis()); + if (embeddingBase64 != null && !embeddingBase64.isEmpty()) { + values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP)); + values.put("embedding_count", 1); + } + return getWritableDatabase().insert("speakers", null, values); + } + + public String querySpeakers() { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count," + + " p.last_heard_at, p.tag," + + " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id," + + " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments" + + " FROM speakers p ORDER BY p.last_heard_at DESC", null)) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("name", c.getString(1)); + row.put("color", c.getString(2)); + row.put("isWearer", c.getInt(3) != 0); + byte[] blob = c.isNull(4) ? null : c.getBlob(4); + row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP)); + row.put("embeddingCount", c.getInt(5)); + row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6)); + row.put("tag", c.isNull(7) ? "" : c.getString(7)); + row.put("lastRecap", c.isNull(8) ? "" : c.getString(8)); + row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9)); + row.put("factsJson", c.isNull(10) ? "" : c.getString(10)); + row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11)); + row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12)); + row.put("segmentCount", c.getLong(13)); + out.put(row); + } + } catch (JSONException e) { + Log.w(TAG, "querySpeakers failed", e); + } + return out.toString(); + } + + public void renameSpeaker(long speakerId, String name) { + ContentValues values = new ContentValues(); + values.put("name", name); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + public void setSpeakerColor(long speakerId, String color) { + ContentValues values = new ContentValues(); + values.put("color", color); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + public void setSpeakerTag(long speakerId, String tag) { + ContentValues values = new ContentValues(); + values.put("tag", tag); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + /** + * Store LLM-derived conversation insights for a speaker: a recap of their + * most recent conversation, its open action items (JSON string array), and + * the accumulated inferred facts about them (JSON string array). + */ + public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson, + String factsJson, long sessionId) { + ContentValues values = new ContentValues(); + values.put("last_recap", recap == null ? "" : recap); + values.put("action_items", actionItemsJson == null ? "" : actionItemsJson); + values.put("facts", factsJson == null ? "" : factsJson); + values.put("insights_updated_at", System.currentTimeMillis()); + values.put("insights_session_id", sessionId); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + public void setSpeakerWearer(long speakerId, boolean isWearer) { + SQLiteDatabase db = getWritableDatabase(); + if (isWearer) { + // Exactly one wearer profile at a time. + db.execSQL("UPDATE speakers SET is_wearer = 0"); + } + ContentValues values = new ContentValues(); + values.put("is_wearer", isWearer ? 1 : 0); + db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + /** + * Running-mean centroid update for a speaker's voice-print, capped so a + * long session cannot drown the enrolled voice. + */ + public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) { + try { + byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP); + float[] update = blobToFloats(incoming); + if (update == null) { + return; + } + float[] current = null; + int count = 0; + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT embedding, embedding_count FROM speakers WHERE id=?", + new String[]{String.valueOf(speakerId)})) { + if (c.moveToFirst()) { + current = c.isNull(0) ? null : blobToFloats(c.getBlob(0)); + count = c.getInt(1); + } + } + float[] merged; + if (current == null || current.length != update.length) { + merged = update; + count = 1; + } else { + int effective = Math.min(count, Math.max(1, maxCount)); + merged = new float[current.length]; + for (int i = 0; i < merged.length; i++) { + merged[i] = (current[i] * effective + update[i]) / (effective + 1); + } + l2NormalizeInPlace(merged); + count = count + 1; + } + ContentValues values = new ContentValues(); + values.put("embedding", floatsToBlob(merged)); + values.put("embedding_count", count); + values.put("last_heard_at", System.currentTimeMillis()); + getWritableDatabase().update("speakers", values, "id=?", + new String[]{String.valueOf(speakerId)}); + } catch (Throwable t) { + Log.w(TAG, "updateSpeakerEmbedding failed", t); + } + } + + /** Move every segment from one speaker onto another and drop the source. */ + public void mergeSpeakers(long fromId, long intoId) { + if (fromId == intoId) { + return; + } + SQLiteDatabase db = getWritableDatabase(); + db.beginTransaction(); + try { + db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?", + new Object[]{intoId, fromId}); + // Blend the two centroids weighted by their sample counts. + float[] from = null; + float[] into = null; + int fromCount = 0; + int intoCount = 0; + boolean fromWearer = false; + try (Cursor c = db.rawQuery( + "SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)", + new String[]{String.valueOf(fromId), String.valueOf(intoId)})) { + while (c.moveToNext()) { + long id = c.getLong(0); + float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1)); + int count = c.getInt(2); + if (id == fromId) { + from = embedding; + fromCount = count; + fromWearer = c.getInt(3) != 0; + } else { + into = embedding; + intoCount = count; + } + } + } + float[] merged = into; + if (from != null && into != null && from.length == into.length) { + merged = new float[into.length]; + int total = Math.max(1, fromCount + intoCount); + for (int i = 0; i < merged.length; i++) { + merged[i] = (into[i] * intoCount + from[i] * fromCount) / total; + } + l2NormalizeInPlace(merged); + } else if (into == null) { + merged = from; + } + ContentValues values = new ContentValues(); + if (merged != null) { + values.put("embedding", floatsToBlob(merged)); + values.put("embedding_count", fromCount + intoCount); + } + if (fromWearer) { + values.put("is_wearer", 1); + } + values.put("last_heard_at", System.currentTimeMillis()); + // Keep the target's conversation insights; adopt the source's only + // where the target has none (e.g. merging a rich profile into a + // freshly auto-created one). + try (Cursor c = db.rawQuery( + "SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id" + + " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END", + new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) { + boolean targetHasInsights = false; + while (c.moveToNext()) { + boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty(); + if (c.getPosition() == 0) { + targetHasInsights = hasInsights; + } else if (!targetHasInsights && hasInsights) { + values.put("last_recap", c.getString(0)); + values.put("action_items", c.isNull(1) ? "" : c.getString(1)); + values.put("facts", c.isNull(2) ? "" : c.getString(2)); + values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3)); + values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4)); + } + } + } + db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)}); + db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)}); + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + } + } + + public void reassignSegmentSpeaker(long segmentId, long speakerId) { + ContentValues values = new ContentValues(); + values.put("speaker_id", speakerId); + getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)}); + } + + /** Reassign every segment of a speaker within one session (split support). */ + public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) { + getWritableDatabase().execSQL( + "UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?", + new Object[]{toSpeakerId, sessionId, fromSpeakerId}); + } + + public void deleteSpeaker(long speakerId) { + SQLiteDatabase db = getWritableDatabase(); + db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId}); + db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)}); + } + + // ---- retention ---- + + /** + * Delete caption sessions older than the cutoff (0 disables), and strip + * recordings older than their own cutoff while keeping the transcript. + * Returns a JSON summary of what was removed. + */ + public String applyRetention(long captionCutoffMs, long recordingCutoffMs) { + SQLiteDatabase db = getWritableDatabase(); + int sessionsDeleted = 0; + int recordingsDeleted = 0; + if (recordingCutoffMs > 0) { + List paths = new ArrayList<>(); + try (Cursor c = db.rawQuery( + "SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", + new String[]{String.valueOf(recordingCutoffMs)})) { + while (c.moveToNext()) { + if (!c.isNull(1)) { + paths.add(c.getString(1)); + } + } + } + for (String path : paths) { + if (new File(path).delete()) { + recordingsDeleted++; + } + } + db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?", + new Object[]{recordingCutoffMs}); + } + if (captionCutoffMs > 0) { + List paths = new ArrayList<>(); + try (Cursor c = db.rawQuery( + "SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", + new String[]{String.valueOf(captionCutoffMs)})) { + while (c.moveToNext()) { + paths.add(c.getString(0)); + } + } + for (String path : paths) { + if (path != null && new File(path).delete()) { + recordingsDeleted++; + } + } + db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)", + new Object[]{captionCutoffMs}); + Cursor count = db.rawQuery("SELECT changes()", null); + count.close(); + db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs}); + try (Cursor c = db.rawQuery("SELECT changes()", null)) { + if (c.moveToFirst()) { + sessionsDeleted = c.getInt(0); + } + } + } + try { + JSONObject summary = new JSONObject(); + summary.put("sessionsDeleted", sessionsDeleted); + summary.put("recordingsDeleted", recordingsDeleted); + return summary.toString(); + } catch (JSONException e) { + return "{}"; + } + } + + public void deleteSession(long sessionId) { + SQLiteDatabase db = getWritableDatabase(); + String path = getSessionAudioPath(sessionId); + if (!path.isEmpty()) { + new File(path).delete(); + } + db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)}); + db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)}); + } + + // ---- blob helpers ---- + + private static float[] blobToFloats(byte[] blob) { + if (blob == null || blob.length < 4 || blob.length % 4 != 0) { + return null; + } + ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN); + float[] out = new float[blob.length / 4]; + for (int i = 0; i < out.length; i++) { + out[i] = buffer.getFloat(); + } + return out; + } + + private static byte[] floatsToBlob(float[] values) { + ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (float v : values) { + buffer.putFloat(v); + } + return buffer.array(); + } + + private static void l2NormalizeInPlace(float[] v) { + double sum = 0; + for (float x : v) { + sum += (double) x * x; + } + double norm = Math.sqrt(sum); + if (norm <= 0) { + return; + } + for (int i = 0; i < v.length; i++) { + v[i] = (float) (v[i] / norm); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java index 39223a64..53645f27 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java @@ -37,9 +37,6 @@ import java.util.Set; * matching are decided in TypeScript (app/g2/even-advertisement.ts, * app/g2/pairing-candidates.ts) where they can be unit-tested; this filter * must stay a superset of the TS classifier's admission. - * - * Ported from the discovery path of SybilSight's Bluetooth SDK (G2.didDiscover - * and R1's advertisement snapshot), reduced to what Android exposes. */ @SuppressLint("MissingPermission") public class FaceclawDeviceDiscovery { @@ -296,7 +293,7 @@ public class FaceclawDeviceDiscovery { /** * Android strips the 2-byte company identifier from each manufacturer * record and keys the SparseArray by it. Put the bytes back (little-endian) - * so TypeScript sees the same "ER"+… layout the iOS SDK and HCI captures + * so TypeScript sees the same "ER"+… layout the HCI captures * document. Prefers Even's own id; otherwise returns the first record so an * unexpected company id still reaches the diagnostics log. */ diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java new file mode 100644 index 00000000..ea9352aa --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java @@ -0,0 +1,262 @@ +package com.faceclaw.app; + +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import com.k2fsa.sherpa.onnx.FastClusteringConfig; +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization; +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig; +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment; +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig; +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig; +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Offline re-diarization of a stored conversation recording (16-bit mono + * WAV): pyannote segmentation plus the shared speaker-embedding model with + * clustering, via sherpa-onnx. Runs on its own thread; results are handed to + * TS as JSON turns for reconciliation against the transcript segments. + */ +public class FaceclawDiarizer { + private static final String TAG = "FaceclawDiarizer"; + + public interface Listener { + /** progress in [0, 1]. */ + void onProgress(double progress); + + /** turns: [{startMs, endMs, cluster}], sorted by start time. */ + void onDone(String turnsJson); + + void onError(String message); + } + + private final String segmentationModelPath; + private final String embeddingModelPath; + private final String wavPath; + private final int numClusters; + private final float threshold; + private final Listener listener; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + private volatile boolean cancelled; + + public FaceclawDiarizer( + String segmentationModelPath, + String embeddingModelPath, + String wavPath, + int numClusters, + float threshold, + Listener listener) { + this.segmentationModelPath = segmentationModelPath; + this.embeddingModelPath = embeddingModelPath; + this.wavPath = wavPath; + this.numClusters = numClusters; + this.threshold = threshold <= 0 ? 0.5f : threshold; + this.listener = listener; + } + + public void start() { + Thread thread = new Thread(this::run, "FaceclawDiarizer"); + thread.setPriority(Thread.MIN_PRIORITY); + thread.start(); + } + + public void cancel() { + cancelled = true; + } + + private void run() { + OfflineSpeakerDiarization diarization = null; + try { + float[] samples = wavPath.endsWith(".wav") + ? readWavMono16(wavPath) + : decodeCompressedMono(wavPath); + OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder() + .setSegmentation(OfflineSpeakerSegmentationModelConfig.builder() + .setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder() + .setModel(segmentationModelPath) + .build()) + .setNumThreads(1) + .build()) + .setEmbedding(SpeakerEmbeddingExtractorConfig.builder() + .setModel(embeddingModelPath) + .setNumThreads(1) + .build()) + .setClustering(FastClusteringConfig.builder() + .setNumClusters(numClusters > 0 ? numClusters : -1) + .setThreshold(threshold) + .build()) + .setMinDurationOn(0.3f) + .setMinDurationOff(0.5f) + .build(); + diarization = new OfflineSpeakerDiarization(config); + OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback( + samples, + (numProcessedChunks, numTotalChunks, arg) -> { + emitProgress(numTotalChunks <= 0 ? 0 + : (double) numProcessedChunks / numTotalChunks); + return cancelled ? 1 : 0; + }); + JSONArray turns = new JSONArray(); + if (segments != null) { + for (OfflineSpeakerDiarizationSegment segment : segments) { + JSONObject turn = new JSONObject(); + turn.put("startMs", Math.round(segment.getStart() * 1000)); + turn.put("endMs", Math.round(segment.getEnd() * 1000)); + turn.put("cluster", segment.getSpeaker()); + turns.put(turn); + } + } + String result = turns.toString(); + mainHandler.post(() -> listener.onDone(result)); + } catch (Throwable t) { + Log.w(TAG, "diarization failed for " + wavPath, t); + String message = t.getMessage() == null ? t.toString() : t.getMessage(); + mainHandler.post(() -> listener.onError(message)); + } finally { + if (diarization != null) { + diarization.release(); + } + } + } + + private void emitProgress(double progress) { + mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress)))); + } + + /** + * Decode a compressed recording (M4A/AAC from the transcode sweep) into + * [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written + * as 16 kHz mono, so no resampling is needed. + */ + private static float[] decodeCompressedMono(String path) throws IOException { + android.media.MediaExtractor extractor = new android.media.MediaExtractor(); + android.media.MediaCodec codec = null; + java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream(); + try { + extractor.setDataSource(path); + int trackIndex = -1; + android.media.MediaFormat format = null; + for (int i = 0; i < extractor.getTrackCount(); i++) { + android.media.MediaFormat candidate = extractor.getTrackFormat(i); + String mime = candidate.getString(android.media.MediaFormat.KEY_MIME); + if (mime != null && mime.startsWith("audio/")) { + trackIndex = i; + format = candidate; + break; + } + } + if (trackIndex < 0 || format == null) { + throw new IOException("no audio track in " + path); + } + extractor.selectTrack(trackIndex); + codec = android.media.MediaCodec.createDecoderByType( + format.getString(android.media.MediaFormat.KEY_MIME)); + codec.configure(format, null, null, 0); + codec.start(); + android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo(); + boolean inputDone = false; + boolean outputDone = false; + while (!outputDone) { + if (!inputDone) { + int inIndex = codec.dequeueInputBuffer(10_000); + if (inIndex >= 0) { + ByteBuffer inBuf = codec.getInputBuffer(inIndex); + int size = extractor.readSampleData(inBuf, 0); + if (size < 0) { + codec.queueInputBuffer(inIndex, 0, 0, 0, + android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + } else { + codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0); + extractor.advance(); + } + } + } + int outIndex = codec.dequeueOutputBuffer(info, 10_000); + if (outIndex >= 0) { + ByteBuffer outBuf = codec.getOutputBuffer(outIndex); + byte[] chunk = new byte[info.size]; + outBuf.position(info.offset); + outBuf.get(chunk); + pcm.write(chunk); + codec.releaseOutputBuffer(outIndex, false); + if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + } + } + } finally { + extractor.release(); + if (codec != null) { + try { + codec.stop(); + } catch (Throwable ignored) { + } + codec.release(); + } + } + byte[] bytes = pcm.toByteArray(); + int count = bytes.length / 2; + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f; + } + return samples; + } + + /** Load a 16-bit mono PCM WAV into [-1, 1] floats. */ + private static float[] readWavMono16(String path) throws IOException { + File file = new File(path); + long fileLength = file.length(); + try (DataInputStream in = new DataInputStream(new FileInputStream(file))) { + byte[] header = new byte[44]; + in.readFully(header); + if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') { + throw new IOException("not a WAV file"); + } + int channels = (header[22] & 0xff) | (header[23] << 8); + int bits = (header[34] & 0xff) | (header[35] << 8); + if (channels != 1 || bits != 16) { + throw new IOException("only 16-bit mono WAV is supported"); + } + long dataBytes = fileLength - 44; + int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE); + float[] samples = new float[count]; + byte[] chunk = new byte[65536]; + int sampleIndex = 0; + int carried = 0; + byte carriedByte = 0; + int read; + while (sampleIndex < count && (read = in.read(chunk)) > 0) { + int offset = 0; + if (carried == 1) { + samples[sampleIndex++] = + ((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f; + offset = 1; + carried = 0; + } + int pairs = (read - offset) / 2; + for (int i = 0; i < pairs && sampleIndex < count; i++) { + int base = offset + i * 2; + samples[sampleIndex++] = + ((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f; + } + if (((read - offset) & 1) == 1) { + carried = 1; + carriedByte = chunk[read - 1]; + } + } + return samples; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java index 6aba3b3d..e7bce6ca 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java @@ -26,6 +26,10 @@ import java.util.concurrent.TimeUnit; * protobuf helpers (BleProtocol), but owns its own GATT connection and runs its * whole lifecycle on a single worker thread. It expects the main app to be * disconnected while it runs (onboarding, before flashing). + * + * Only the right arm is used: the lenses relay messages to each other, so + * anything sent to the right arm reaches both, and acks/events only ever come + * from the right arm anyway. */ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private static final String TAG = "FaceclawFlashPrompt"; @@ -43,7 +47,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private final Context context; private final String rightAddress; - private final String leftAddress; private final String warningText; private final FaceclawBleManager bleManager; private final Handler mainHandler = new Handler(Looper.getMainLooper()); @@ -58,7 +61,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private volatile boolean cancelled = false; private volatile boolean finished = false; private volatile boolean rightConnected = false; - private volatile boolean leftConnected = false; private volatile boolean rightLost = false; private volatile Boolean approved = null; private volatile ScheduledExecutorService heartbeatExecutor; @@ -66,10 +68,9 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private int nextMagic = 100; private int nextSeq = 0x40; - public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String leftAddress, String warningText) { + public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String warningText) { this.context = context.getApplicationContext(); this.rightAddress = rightAddress == null ? "" : rightAddress; - this.leftAddress = leftAddress == null ? "" : leftAddress; this.warningText = warningText == null ? "" : warningText; this.bleManager = new FaceclawBleManager(this.context); this.bleManager.setListener(this); @@ -116,18 +117,8 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } emitState("connecting", ""); - connectArm(rightAddress, "right"); + connectArm(rightAddress); rightConnected = true; - - boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress); - if (haveLeft) { - try { - connectArm(leftAddress, "left"); - leftConnected = true; - } catch (Exception e) { - emitLog("left arm connect failed (continuing on right only): " + e.getMessage()); - } - } if (cancelled) { teardown(); return; @@ -135,14 +126,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { emitState("connected", ""); bringUpArm(rightAddress); - if (leftConnected) { - try { - bringUpArm(leftAddress); - } catch (Exception e) { - emitLog("left arm prompt failed (continuing on right only): " + e.getMessage()); - leftConnected = false; - } - } startHeartbeat(); emitState("prompting", ""); @@ -158,7 +141,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { if (approved != null) { boolean result = Boolean.TRUE.equals(approved); try { - sendShutdownBoth(); + sendShutdown(); } catch (Exception ignored) { } emitResult(result); @@ -182,7 +165,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } } - private void connectArm(String address, String labelForLog) { + private void connectArm(String address) { if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { throw new IllegalStateException("connect failed: " + address); } @@ -194,7 +177,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { throw new IllegalStateException("enableNotifications failed: " + address); } - emitLog("connected " + labelForLog + " arm"); + emitLog("connected right arm"); } private void bringUpArm(String address) throws InterruptedException { @@ -301,9 +284,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { if (rightConnected) { writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat); } - if (leftConnected) { - writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat); - } } catch (Exception e) { Log.w(TAG, "heartbeat failed: " + e.getMessage()); } @@ -317,14 +297,11 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } } - private void sendShutdownBoth() { + private void sendShutdown() { byte[] shutdown = BleProtocol.buildShutdown(allocMagic(), 0); if (rightConnected) { writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown); } - if (leftConnected) { - writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown); - } } private void teardown() { @@ -335,7 +312,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } catch (Exception ignored) { } rightConnected = false; - leftConnected = false; } @Override @@ -355,10 +331,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { return; } if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { - // Async events (taps/selections) only ever come from the right arm. - if (address.equalsIgnoreCase(rightAddress)) { - handleEvent(frame); - } + handleEvent(frame); return; } if (frame.msgSeq >= 0) { @@ -407,8 +380,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { selectionLatch.countDown(); } } - } else if (address.equalsIgnoreCase(leftAddress)) { - leftConnected = false; } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java index ce9f0557..1ed6b950 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java @@ -12,6 +12,12 @@ public class FaceclawLc3Decoder { public static final int COUNTER_OFFSET = 204; public static final int SAMPLES_PER_FRAME = 160; public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME; + // Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the + // glasses DSP appends two signed 16-bit LE values computed on the raw + // stereo capture before the mono downmix — a signal-strength ratio and the + // asin-derived direction-of-arrival in degrees — then the packet counter. + public static final int SSR_OFFSET = 200; + public static final int ANGLE_OFFSET = 202; static { System.loadLibrary("faceclaw_lc3"); @@ -23,6 +29,8 @@ public class FaceclawLc3Decoder { private long duplicatePackets; private long missingPackets; private long decodeErrors; + private int lastSsr; + private int lastAngleDegrees; public FaceclawLc3Decoder() { nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE); @@ -64,9 +72,21 @@ public class FaceclawLc3Decoder { } realPackets++; lastCounter = counter; + lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8)); + lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8)); return decoded; } + /** Firmware signal-strength ratio from the most recent decoded packet. */ + public synchronized int getLastSsr() { + return lastSsr; + } + + /** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */ + public synchronized int getLastAngleDegrees() { + return lastAngleDegrees; + } + public synchronized long getRealPackets() { return realPackets; } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMicStatusListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMicStatusListener.java new file mode 100644 index 00000000..e739ea34 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMicStatusListener.java @@ -0,0 +1,10 @@ +package com.faceclaw.app; + +public interface FaceclawMicStatusListener { + /** + * A CFW mic_control field-104 status record from one temple. body is the + * raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is + * "L" or "R" for the temple's own BLE link, "?" if unmatched. + */ + void onMicStatus(byte[] body, String arm); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawNoiseSuppressor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawNoiseSuppressor.java new file mode 100644 index 00000000..5d9514dc --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawNoiseSuppressor.java @@ -0,0 +1,266 @@ +package com.faceclaw.app; + +import android.media.audiofx.AcousticEchoCanceler; +import android.media.audiofx.AudioEffect; +import android.media.audiofx.AutomaticGainControl; +import android.media.audiofx.NoiseSuppressor; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Phone-side noise suppression for the microphone stream. + * + * Android's built-in noise cancellation (the {@link NoiseSuppressor} capture + * effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio + * arrives as raw PCM over BLE from the glasses, which that effect chain never + * sees, so this class picks the best engine that can actually reach the + * samples: + * + * - Audio captured by the phone itself: {@link #attachPlatformEffects(int)} + * enables the device's built-in NS/AEC/AGC chain (hardware or OEM software, + * whatever the Android build ships) on the AudioRecord session. + * - Audio from the glasses (the normal case): {@link #process(byte[])} runs a + * WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann + * frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an + * over-subtracted Wiener gain with a residual floor so speech never pumps + * in and out of silence. + */ +public final class FaceclawNoiseSuppressor { + + private static final int FFT_SIZE = 256; + private static final int HOP = FFT_SIZE / 2; + private static final float OVERSUBTRACTION = 1.6f; + /** About -16 dB of residual: keeps the noise bed natural instead of gated. */ + private static final float GAIN_FLOOR = 0.15f; + private static final float GAIN_SMOOTHING = 0.6f; + /** Frames of pure noise-floor learning after start/reset (~100 ms). */ + private static final int NOISE_WARMUP_FRAMES = 12; + + private final int sampleRate; + private final float[] window = new float[FFT_SIZE]; + private final float[] history = new float[FFT_SIZE]; + private final float[] overlap = new float[HOP]; + private final float[] noise = new float[FFT_SIZE / 2 + 1]; + private final float[] gain = new float[FFT_SIZE / 2 + 1]; + private final float[] re = new float[FFT_SIZE]; + private final float[] im = new float[FFT_SIZE]; + private final short[] pending = new short[HOP]; + private int pendingCount = 0; + private int warmupFrames = 0; + + public FaceclawNoiseSuppressor(int sampleRate) { + this.sampleRate = sampleRate; + // sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap. + for (int i = 0; i < FFT_SIZE; i++) { + window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE); + } + reset(); + } + + public int getSampleRate() { + return sampleRate; + } + + /** Forget the learned noise floor (e.g. after the suppressor was toggled off). */ + public void reset() { + Arrays.fill(history, 0f); + Arrays.fill(overlap, 0f); + Arrays.fill(noise, 0f); + Arrays.fill(gain, 1f); + pendingCount = 0; + warmupFrames = 0; + } + + /** + * Suppress noise in 16-bit little-endian mono PCM. Streaming: input not + * filling a whole 8 ms hop is carried into the next call, so the output + * may be up to one hop shorter than the input (and longer on a later + * call); total latency is one hop. + */ + public byte[] process(byte[] pcm16le) { + if (pcm16le == null || pcm16le.length < 2) { + return pcm16le; + } + int sampleCount = pcm16le.length / 2; + int hops = (pendingCount + sampleCount) / HOP; + byte[] out = new byte[hops * HOP * 2]; + int outPos = 0; + int inPos = 0; + while (inPos < sampleCount) { + int take = Math.min(HOP - pendingCount, sampleCount - inPos); + for (int i = 0; i < take; i++) { + int lo = pcm16le[(inPos + i) * 2] & 0xff; + int hi = pcm16le[(inPos + i) * 2 + 1]; + pending[pendingCount + i] = (short) ((hi << 8) | lo); + } + pendingCount += take; + inPos += take; + if (pendingCount == HOP) { + outPos = processHop(out, outPos); + pendingCount = 0; + } + } + return out; + } + + private int processHop(byte[] out, int outPos) { + System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP); + for (int i = 0; i < HOP; i++) { + history[FFT_SIZE - HOP + i] = pending[i] / 32768f; + } + for (int i = 0; i < FFT_SIZE; i++) { + re[i] = history[i] * window[i]; + im[i] = 0f; + } + fft(re, im, false); + + boolean warming = warmupFrames < NOISE_WARMUP_FRAMES; + if (warming) { + warmupFrames++; + } + for (int b = 0; b <= FFT_SIZE / 2; b++) { + float power = re[b] * re[b] + im[b] * im[b]; + if (warming) { + noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames; + } else if (power < noise[b]) { + // Track the floor down quickly; drift up slowly so speech + // energy never gets learned as noise. + noise[b] += (power - noise[b]) * 0.2f; + } else { + noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f); + } + float clean = power - OVERSUBTRACTION * noise[b]; + float g = clean > 0f ? clean / power : 0f; + if (g < GAIN_FLOOR) { + g = GAIN_FLOOR; + } + g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g; + gain[b] = g; + re[b] *= g; + im[b] *= g; + if (b > 0 && b < FFT_SIZE / 2) { + re[FFT_SIZE - b] *= g; + im[FFT_SIZE - b] *= g; + } + } + + fft(re, im, true); + for (int i = 0; i < HOP; i++) { + float value = (overlap[i] + re[i] * window[i]) * 32767f; + int sample = Math.round(value); + if (sample > 32767) { + sample = 32767; + } + if (sample < -32768) { + sample = -32768; + } + out[outPos + i * 2] = (byte) (sample & 0xff); + out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff); + } + for (int i = 0; i < HOP; i++) { + overlap[i] = re[HOP + i] * window[HOP + i]; + } + return outPos + HOP * 2; + } + + /** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */ + private static void fft(float[] re, float[] im, boolean inverse) { + int n = re.length; + for (int i = 1, j = 0; i < n; i++) { + int bit = n >> 1; + for (; (j & bit) != 0; bit >>= 1) { + j ^= bit; + } + j ^= bit; + if (i < j) { + float t = re[i]; + re[i] = re[j]; + re[j] = t; + t = im[i]; + im[i] = im[j]; + im[j] = t; + } + } + for (int len = 2; len <= n; len <<= 1) { + double ang = 2 * Math.PI / len * (inverse ? 1 : -1); + float wRe = (float) Math.cos(ang); + float wIm = (float) Math.sin(ang); + for (int i = 0; i < n; i += len) { + float curRe = 1f; + float curIm = 0f; + for (int k = 0; k < len / 2; k++) { + int a = i + k; + int b = a + len / 2; + float xr = re[b] * curRe - im[b] * curIm; + float xi = re[b] * curIm + im[b] * curRe; + re[b] = re[a] - xr; + im[b] = im[a] - xi; + re[a] += xr; + im[a] += xi; + float nextRe = curRe * wRe - curIm * wIm; + curIm = curRe * wIm + curIm * wRe; + curRe = nextRe; + } + } + } + if (inverse) { + for (int i = 0; i < n; i++) { + re[i] /= n; + im[i] /= n; + } + } + } + + /** Whether this Android build ships the built-in capture noise suppressor. */ + public static boolean platformSuppressorAvailable() { + try { + return NoiseSuppressor.isAvailable(); + } catch (Throwable t) { + return false; + } + } + + /** Short engine name for the UI. */ + public static String engineDescription() { + return "spectral (phone DSP)"; + } + + /** + * Attach the platform's built-in capture-effect chain (NoiseSuppressor, + * AcousticEchoCanceler, AutomaticGainControl — whichever this Android + * build provides) to a phone AudioRecord session. Only applicable when the + * phone itself captures the audio; the BLE glasses stream must go through + * {@link #process(byte[])} instead. The caller keeps the returned effects + * for the life of the recording and release()s them with it. + */ + public static AudioEffect[] attachPlatformEffects(int audioSessionId) { + ArrayList effects = new ArrayList<>(); + try { + if (NoiseSuppressor.isAvailable()) { + NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId); + if (ns != null) { + ns.setEnabled(true); + effects.add(ns); + } + } + if (AcousticEchoCanceler.isAvailable()) { + AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId); + if (aec != null) { + aec.setEnabled(true); + effects.add(aec); + } + } + if (AutomaticGainControl.isAvailable()) { + AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId); + if (agc != null) { + agc.setEnabled(true); + effects.add(agc); + } + } + } catch (Throwable t) { + // A missing effect just means this tier is unavailable. + } + return effects.toArray(new AudioEffect[0]); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java new file mode 100644 index 00000000..06d803c4 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java @@ -0,0 +1,185 @@ +package com.faceclaw.app; + +import android.content.Context; + +/** + * Headless display target for preview-only mode (no glasses paired). Hosts + * the same SurfaceCompositor the BLE communicator uses, so the TS side can + * run its full surface/render pipeline unchanged; composited frames simply + * have nowhere to go, and the phone-mirror preview, screenshots, and GIF + * recordings read the retained composite back exactly as they do on a live + * connection. + * + * The method signatures mirror the display subset of FaceclawBleCommunicator + * (the TS DisplayTarget interface is typed against that subset). + */ +public final class FaceclawPreviewCompositor { + private final Context appContext; + private final SurfaceCompositor compositor = new SurfaceCompositor(); + private final android.os.Handler mainHandler = new android.os.Handler(android.os.Looper.getMainLooper()); + + // Notified (on the main looper, where the phone UI's JS runs) after each + // applied surface frame, standing in for the BLE path's per-frame metrics + // callback so the phone mirror refreshes promptly instead of waiting for + // its safety-net poll. + private volatile Runnable frameListener; + + // Active animated-GIF screen recording, or null when idle. Frames are + // pushed by recordScreenFrame(), which the TS side calls at each + // phone-preview flush. + private volatile GifScreenRecorder screenRecorder; + + // Worker-app isolates locate their frame target through a static, like + // FaceclawBleCommunicator.getActive(); Java statics are shared across + // isolates, unlike anything on the JS side. + private static volatile FaceclawPreviewCompositor activeInstance; + + public static FaceclawPreviewCompositor getActive() { + return activeInstance; + } + + public FaceclawPreviewCompositor(Context context) { + this.appContext = context.getApplicationContext(); + } + + /** Publish this compositor as the workers' frame target. */ + public void makeActive() { + activeInstance = this; + } + + /** Withdraw from workers (a real connection is taking over). */ + public void release() { + if (activeInstance == this) { + activeInstance = null; + } + frameListener = null; + } + + public void setFrameListener(Runnable listener) { + this.frameListener = listener; + } + + /** Set the compositor's output frame size. Call before configuring surfaces. */ + public void configureCompositorScreen(int width, int height) { + compositor.configureScreen(width, height); + } + + public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) { + compositor.configureSurface(id, x, y, width, height, zOrder, transparency); + } + + public void removeSurface(String id) { + compositor.removeSurface(id); + } + + public void setSurfaceVisible(String id, boolean visible) { + compositor.setSurfaceVisible(id, visible); + } + + /** Blank (screen off) or unblank the output; retained surface state survives. */ + public void setScreenBlanked(boolean blanked) { + compositor.setBlanked(blanked); + } + + /** + * Apply an update to one compositor surface. Unlike the connected path + * there is no desired-frame store or transmit pipeline downstream; the + * preview/screenshot calls recomposite the retained state on demand, so + * applying the update is the whole job and the frame finishes here. + */ + public void submitSurfaceFrame( + java.nio.ByteBuffer pixels8bpp, + String surfaceId, + int rectX, + int rectY, + int rectWidth, + int rectHeight, + String contentFingerprint, + int paintMs, + int frameId, + java.nio.ByteBuffer glyphs + ) { + compositor.applyAndComposite( + surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs); + FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)"); + Runnable listener = frameListener; + if (listener != null) { + mainHandler.post(listener); + } + } + + /** + * The current composited screen as a phone-UI preview bitmap, or null + * before any surface has been configured. + */ + public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return null; + } + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green); + } + + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ + public String saveCompositePngScreenshot() throws java.io.IOException { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return ""; + } + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); + } + + /** + * Save the current composite cropped to the given screen rect (the region + * the shell says is actually occupied). The rect is clamped to the screen; + * a degenerate rect falls back to the full screen. + */ + public String saveCompositePngScreenshot(int cropX, int cropY, int cropWidth, int cropHeight) + throws java.io.IOException { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return ""; + } + int x = Math.max(0, cropX); + int y = Math.max(0, cropY); + int width = Math.min(composite.width - x, cropWidth - (x - cropX)); + int height = Math.min(composite.height - y, cropHeight - (y - cropY)); + if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) { + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); + } + byte[] cropped = new byte[width * height]; + for (int row = 0; row < height; row++) { + System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width); + } + return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height); + } + + /** Begin collecting composite frames for an animated-GIF screen recording. */ + public void startScreenRecording() { + screenRecorder = new GifScreenRecorder(); + } + + /** Capture the current composite into the active recording; no-op when idle. */ + public void recordScreenFrame() { + GifScreenRecorder recorder = screenRecorder; + if (recorder == null) { + return; + } + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return; + } + recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis()); + } + + /** Finish the recording and save it as an animated GIF; returns the path or "". */ + public String stopScreenRecording() throws java.io.IOException { + GifScreenRecorder recorder = screenRecorder; + screenRecorder = null; + if (recorder == null) { + return ""; + } + return recorder.save(appContext); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.java new file mode 100644 index 00000000..90becbef --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsPortReceiver.java @@ -0,0 +1,141 @@ +package com.faceclaw.app; + +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; +import android.util.Xml; + +import org.xmlpull.v1.XmlPullParser; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * adb-triggered export/import of the faceclaw_settings SharedPreferences + * file, so configuration (API keys, device addresses, ...) can be moved off a + * release build, where run-as is unavailable. See scripts/pull_config.sh and + * scripts/push_config.sh, which use run-as on debuggable builds and fall back + * to this receiver otherwise. + * + * adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \ + * -a com.faceclaw.app.SETTINGS_EXPORT + * adb pull /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-export.xml + * + * adb push settings.xml /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-import.xml + * adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \ + * -a com.faceclaw.app.SETTINGS_IMPORT + * + * Security: the settings file contains API tokens, so the manifest guards + * this receiver with android.permission.DUMP — a development permission the + * adb shell holds but ordinary apps cannot obtain without adb's help, keeping + * the trigger reachable from adb only. The transfer file lives in + * getExternalFilesDir(), which other apps cannot read on Android 11+ (and + * which the scripts delete as soon as the transfer completes); on the + * Android 7-10 devices minSdk still admits, apps holding READ_EXTERNAL_STORAGE + * could read it during that window. + */ +public class FaceclawSettingsPortReceiver extends BroadcastReceiver { + private static final String TAG = "FaceclawSettingsPort"; + public static final String ACTION_EXPORT = "com.faceclaw.app.SETTINGS_EXPORT"; + public static final String ACTION_IMPORT = "com.faceclaw.app.SETTINGS_IMPORT"; + private static final String PREFS_FILE = "faceclaw_settings.xml"; + private static final String EXPORT_NAME = "faceclaw-settings-export.xml"; + private static final String IMPORT_NAME = "faceclaw-settings-import.xml"; + + @Override + public void onReceive(Context context, Intent intent) { + String action = intent == null ? null : intent.getAction(); + try { + if (ACTION_EXPORT.equals(action)) { + setResultData(exportSettings(context)); + } else if (ACTION_IMPORT.equals(action)) { + setResultData(importSettings(context)); + // Exit once the broadcast result has been delivered: this + // process may hold stale in-memory SharedPreferences that a + // later apply() would flush over the imported file, and the + // next launch must re-read from disk. + new Handler(Looper.getMainLooper()).postDelayed(() -> { + Log.i(TAG, "Exiting after settings import"); + System.exit(0); + }, 500); + } + } catch (Exception e) { + Log.e(TAG, "Settings " + action + " failed", e); + setResultData("error: " + e); + } + } + + private File prefsFile(Context context) { + return new File(context.getApplicationInfo().dataDir, "shared_prefs/" + PREFS_FILE); + } + + private File externalFile(Context context, String name) throws IOException { + File dir = context.getExternalFilesDir(null); + if (dir == null) { + throw new IOException("external files dir unavailable"); + } + return new File(dir, name); + } + + private String exportSettings(Context context) throws IOException { + File prefs = prefsFile(context); + if (!prefs.exists()) { + return "error: no settings file yet (" + prefs + ")"; + } + File out = externalFile(context, EXPORT_NAME); + copy(prefs, out); + return "exported: " + out; + } + + private String importSettings(Context context) throws Exception { + File staged = externalFile(context, IMPORT_NAME); + if (!staged.exists()) { + return "error: nothing staged at " + staged; + } + validatePrefsXml(staged); + File prefs = prefsFile(context); + prefs.getParentFile().mkdirs(); + if (prefs.exists()) { + copy(prefs, new File(prefs.getPath() + ".bak")); + } + copy(staged, prefs); + staged.delete(); + return "imported: " + prefs + " (previous settings in " + PREFS_FILE + ".bak)"; + } + + /** Reject files that are not a SharedPreferences document. */ + private void validatePrefsXml(File file) throws Exception { + try (InputStream in = new FileInputStream(file)) { + XmlPullParser parser = Xml.newPullParser(); + parser.setInput(in, null); + int event = parser.next(); + while (event != XmlPullParser.START_TAG && event != XmlPullParser.END_DOCUMENT) { + event = parser.next(); + } + if (event != XmlPullParser.START_TAG || !"map".equals(parser.getName())) { + throw new IOException("not a SharedPreferences document"); + } + // Walk the rest so malformed XML is caught before it replaces + // the live settings file. + while (parser.next() != XmlPullParser.END_DOCUMENT) { } + } + } + + private void copy(File from, File to) throws IOException { + try (InputStream in = new FileInputStream(from); + OutputStream out = new FileOutputStream(to)) { + byte[] buffer = new byte[8192]; + int n; + while ((n = in.read(buffer)) > 0) { + out.write(buffer, 0, n); + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java new file mode 100644 index 00000000..c258ad91 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java @@ -0,0 +1,111 @@ +package com.faceclaw.app; + +import android.util.Log; + +import com.k2fsa.sherpa.onnx.OnlineStream; +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor; +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig; + +/** + * Speaker voice-print extraction over sherpa-onnx. One embedding per + * utterance; matching/enrollment happens on the TS side (cosine similarity + * against the stored speaker profiles). + */ +public class FaceclawSpeakerId { + private static final String TAG = "FaceclawSpeakerId"; + + private SpeakerEmbeddingExtractor extractor; + private final String modelPath; + + public FaceclawSpeakerId(String modelPath) { + this.modelPath = modelPath; + } + + /** Lazily loads the model; returns false when it cannot be loaded. */ + public synchronized boolean ensureLoaded() { + if (extractor != null) { + return true; + } + try { + SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder() + .setModel(modelPath) + .setNumThreads(1) + .setDebug(false) + .setProvider("cpu") + .build(); + extractor = new SpeakerEmbeddingExtractor(config); + return true; + } catch (Throwable t) { + Log.w(TAG, "could not load speaker model at " + modelPath, t); + extractor = null; + return false; + } + } + + public synchronized int embeddingDim() { + return extractor == null ? 0 : extractor.getDim(); + } + + /** + * Compute an L2-normalized voice-print for one utterance of 16 kHz mono + * S16LE PCM. Returns null when the model is unavailable or the utterance + * is too short to embed (under ~0.5 s). + */ + public synchronized float[] embed(byte[] pcm16le, int sampleRate) { + if (pcm16le == null || pcm16le.length < sampleRate) { + return null; + } + if (!ensureLoaded()) { + return null; + } + int count = pcm16le.length / 2; + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8)); + samples[i] = s / 32768.0f; + } + OnlineStream stream = null; + try { + stream = extractor.createStream(); + stream.acceptWaveform(samples, sampleRate); + stream.inputFinished(); + if (!extractor.isReady(stream)) { + return null; + } + float[] embedding = extractor.compute(stream); + return l2Normalize(embedding); + } catch (Throwable t) { + Log.w(TAG, "embedding failed", t); + return null; + } finally { + if (stream != null) { + stream.release(); + } + } + } + + private static float[] l2Normalize(float[] v) { + if (v == null) { + return null; + } + double sum = 0; + for (float x : v) { + sum += (double) x * x; + } + double norm = Math.sqrt(sum); + if (norm <= 0) { + return v; + } + for (int i = 0; i < v.length; i++) { + v[i] = (float) (v[i] / norm); + } + return v; + } + + public synchronized void close() { + if (extractor != null) { + extractor.release(); + extractor = null; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index e0a75ab3..12680324 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -76,6 +76,15 @@ public class FaceclawVoiceController { private volatile FaceclawBleCommunicator communicator; private Thread workerThread; private volatile boolean started; + // Set once the worker has the glasses mic enabled for this session. + // Read and written under `lock`, so it flips with `started` atomically. + private boolean audioStarted; + // Capture from the phone's own microphone instead of the G2 over BLE + // (preview-only mode, where no glasses are connected). Latched into + // activePhoneMic at start() (under `lock`) so a mid-session setter call + // can't switch pipelines underneath the worker. + private volatile boolean usePhoneMic; + private boolean activePhoneMic; private VoiceInputMode mode = VoiceInputMode.CLOUD; private OfflineRecognizer recognizer; private FaceclawLc3Decoder lc3Decoder; @@ -90,6 +99,26 @@ public class FaceclawVoiceController { private volatile boolean endpointing; private final EndpointDetector endpointDetector = new EndpointDetector(); private java.io.ByteArrayOutputStream recordingPcm; + // Speaker verification against the enrolled wearer voice-print ("my voice + // only" command gating). Configured before start(); the utterance PCM is + // buffered (capped) and verified once at session end. + private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10; + private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE; + private volatile String verifySpeakerModelPath; + private volatile float[] verifyWearerEmbedding; + private volatile float verifyThreshold = 0.8f; + private short[] verifyBuffer; + private int verifyCount; + // Global mic processing (Microphones app config): spectral noise + // suppression and firmware-DoA beam gating, applied to every capture + // session that opts in (assistant push-to-talk, Transcribe, hands-free). + // The raw tap opts out — raw means raw, and the Microphones session does + // its own beam-compensated processing on that path. + private volatile boolean suppressionEnabled; + private volatile boolean beamFilterEnabled; + private volatile int beamCenterDeg; + private volatile int beamHalfWidthDeg = 180; + private FaceclawNoiseSuppressor suppressor; private long queuedPackets; private long queueDroppedPackets; private long decodedSamples; @@ -111,6 +140,11 @@ public class FaceclawVoiceController { this.communicator = communicator; } + /** Source the next capture from the phone microphone (no glasses paired). */ + public void setUsePhoneMic(boolean usePhoneMic) { + this.usePhoneMic = usePhoneMic; + } + /** When true, the decoded mic PCM for each session is saved as a WAV. */ public void setSaveRecordings(boolean saveRecordings) { this.saveRecordings = saveRecordings; @@ -125,23 +159,117 @@ public class FaceclawVoiceController { this.endpointing = endpointing; } + /** + * Verify this session's speaker against the enrolled wearer voice-print + * and report the result via onSpeakerVerified just before the final + * transcript. Must be set before {@link #start}; pass a null model path + * to disable. + */ + public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) { + this.verifySpeakerModelPath = speakerModelPath; + this.verifyWearerEmbedding = wearerEmbedding; + this.verifyThreshold = threshold > 0 ? threshold : 0.8f; + } + + public void clearSpeakerVerification() { + this.verifySpeakerModelPath = null; + this.verifyWearerEmbedding = null; + } + + /** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */ + public void setNoiseSuppression(boolean enabled) { + this.suppressionEnabled = enabled; + } + + /** + * Direction gating from the Sonic Radar beam: packets whose firmware + * direction-of-arrival falls outside centerDeg ± halfWidthDeg (device + * frame, 0 = straight ahead, positive right) are dropped before any + * consumer sees them. Safe to update mid-run. + */ + public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) { + this.beamFilterEnabled = enabled; + this.beamCenterDeg = centerDeg; + this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg)); + } + + private boolean withinBeam(int angleDegrees) { + int delta = angleDegrees - beamCenterDeg; + while (delta > 180) delta -= 360; + while (delta < -180) delta += 360; + return Math.abs(delta) <= beamHalfWidthDeg; + } + + private void applySuppression(short[] pcm, int count) { + try { + if (suppressor == null) { + suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE); + } + byte[] le = new byte[count * 2]; + for (int i = 0; i < count; i++) { + le[i * 2] = (byte) (pcm[i] & 0xff); + le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff); + } + byte[] cleaned = suppressor.process(le); + int cleanedCount = Math.min(count, cleaned.length / 2); + for (int i = 0; i < cleanedCount; i++) { + pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8)); + } + } catch (Throwable t) { + Log.w(TAG, "noise suppression failed; passing audio through", t); + suppressionEnabled = false; + } + } + public void start(String requestedMode) { synchronized (lock) { if (started) { emitStatus("Voice control is already listening."); return; } - if (communicator == null || !communicator.isSessionReady()) { + if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) { emitStatus("Voice control needs an active G2 connection."); return; } mode = parseMode(requestedMode); + activePhoneMic = usePhoneMic; started = true; + audioStarted = false; workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); workerThread.start(); } } + /** + * Whether mic audio is actually flowing. {@link #start} only records + * intent: the enable lives in the glasses' EvenHub session, so a transport + * drop or a session suspend can leave this controller started with a + * worker that will never see another packet. Anything deciding whether to + * (re)start capture must ask this rather than assume its own bookkeeping. + */ + public boolean isCapturing() { + boolean audioUp; + boolean phoneMic; + synchronized (lock) { + if (!started) { + return false; + } + audioUp = audioStarted; + phoneMic = activePhoneMic; + } + if (!audioUp) { + // The worker is still bringing the mic up; report it as running so + // a concurrent request shares it instead of restarting it. + return true; + } + if (phoneMic) { + // AudioRecord has no session to lose the enable to; it runs until stop(). + return true; + } + FaceclawBleCommunicator currentCommunicator = communicator; + return currentCommunicator != null && currentCommunicator.isAudioCaptureActive(); + } + public void stop() { Thread threadToJoin; synchronized (lock) { @@ -193,18 +321,54 @@ public class FaceclawVoiceController { resetTranscriptState(); lastTranscript = ""; } - lc3Decoder = new FaceclawLc3Decoder(); endpointDetector.reset(); - recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null; - if (!startG2Audio()) { - emitStatus("Could not start G2 microphone input."); - return; + if (suppressor != null) { + suppressor.reset(); } - - emitStatus(currentMode == VoiceInputMode.CLOUD - ? "Listening (cloud)..." - : "Listening..."); - processG2Audio(); + recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null; + boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null; + verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null; + verifyCount = 0; + if (activePhoneMic) { + android.media.AudioRecord record = openPhoneMic(); + if (record == null) { + emitStatus("Could not start the phone microphone."); + return; + } + synchronized (lock) { + audioStarted = true; + } + emitStatus(currentMode == VoiceInputMode.CLOUD + ? "Listening (cloud)..." + : "Listening..."); + try { + processPhoneAudio(record); + } finally { + try { + record.stop(); + } catch (Throwable ignored) { + // Already stopped or never recording; release below either way. + } + record.release(); + } + } else { + lc3Decoder = new FaceclawLc3Decoder(); + if (!startG2Audio()) { + emitStatus("Could not start G2 microphone input."); + return; + } + synchronized (lock) { + audioStarted = true; + } + emitStatus(currentMode == VoiceInputMode.CLOUD + ? "Listening (cloud)..." + : "Listening..."); + processG2Audio(); + } + // Verification result must precede the final transcript so the + // TS bridge can suppress a non-wearer command before it is acted + // on (the callbacks are posted in order to the main handler). + runSpeakerVerification(); // Button released / stop requested: emit one final full-utterance // transcript so the UI can freeze it. if (currentMode == VoiceInputMode.ONBOARD) { @@ -220,6 +384,7 @@ public class FaceclawVoiceController { releaseLc3(); synchronized (lock) { started = false; + audioStarted = false; workerThread = null; } } @@ -374,25 +539,123 @@ public class FaceclawVoiceController { continue; } decodedSamples += count; - if (recordingPcm != null) { - appendRecording(pcm, count); - } - if (endpointing && endpointDetector.accept(pcm, count)) { - emitSpeechEnd(); - } - if (mode == VoiceInputMode.CLOUD) { - emitPcm(pcm, count); - } else { - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = pcm[i] / 32768.0f; - } - processRecognizer(samples); + // Global mic processing from the Microphones app config: the + // beam filter drops packets whose firmware direction-of-arrival + // falls outside the listening wedge (isolating the aimed talker + // for every consumer, recognition included), and the spectral + // noise suppressor cleans what remains before it reaches the + // recognizer, cloud PCM, endpointing, or speaker verification. + int angleDegrees = currentDecoder.getLastAngleDegrees(); + int ssr = currentDecoder.getLastSsr(); + if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) { + emitFrameMeta(angleDegrees, ssr); + maybeEmitAudioStats(false); + continue; } + processPcmChunk(pcm, count, angleDegrees, ssr, true); maybeEmitAudioStats(false); } } + /** + * Per-chunk processing shared by the G2 and phone-mic paths, downstream of + * decode and the beam filter. hasFrameMeta is false for the phone mic, + * which has no firmware DSP metadata to report. + */ + private void processPcmChunk(short[] pcm, int count, int angleDegrees, int ssr, boolean hasFrameMeta) { + if (suppressionEnabled) { + applySuppression(pcm, count); + } + if (recordingPcm != null) { + appendRecording(pcm, count); + } + if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { + int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount); + System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied); + verifyCount += copied; + } + if (endpointing && endpointDetector.accept(pcm, count)) { + emitSpeechEnd(); + } + // PCM and frame metadata flow in every mode so levels, recording, + // and the Microphones radar keep working alongside onboard ASR. + emitPcm(pcm, count); + if (hasFrameMeta) { + emitFrameMeta(angleDegrees, ssr); + } + if (mode != VoiceInputMode.CLOUD) { + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + samples[i] = pcm[i] / 32768.0f; + } + processRecognizer(samples); + } + } + + // 50 ms chunks match the G2 packet cadence the rest of the pipeline + // (endpointing, transcript pacing) is tuned for. + private static final int PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20; + + /** + * Open the phone's own microphone at the pipeline's native format + * (16 kHz mono PCM16), or null when it cannot start — the permission is + * missing (SecurityException) or the device refuses the configuration. + */ + private android.media.AudioRecord openPhoneMic() { + android.media.AudioRecord record = null; + try { + int minBytes = android.media.AudioRecord.getMinBufferSize( + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT); + int bufferBytes = Math.max(minBytes, PHONE_MIC_CHUNK_SAMPLES * 2 * 4); + record = new android.media.AudioRecord( + android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION, + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT, + bufferBytes); + if (record.getState() != android.media.AudioRecord.STATE_INITIALIZED) { + record.release(); + return null; + } + record.startRecording(); + if (record.getRecordingState() != android.media.AudioRecord.RECORDSTATE_RECORDING) { + record.release(); + return null; + } + return record; + } catch (Throwable t) { + Log.w(TAG, "phone mic open failed", t); + if (record != null) { + record.release(); + } + return null; + } + } + + /** + * Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame + * metadata — AudioRecord already delivers the pipeline's PCM format. The + * blocking read returns every chunk (50 ms), which bounds how long a + * stop() waits for the loop to notice `started` dropped. + */ + private void processPhoneAudio(android.media.AudioRecord record) { + short[] pcm = new short[PHONE_MIC_CHUNK_SAMPLES]; + while (started && !Thread.currentThread().isInterrupted()) { + int read = record.read(pcm, 0, pcm.length); + if (read < 0) { + Log.w(TAG, "phone mic read failed: " + read); + return; + } + if (read == 0) { + continue; + } + decodedSamples += read; + processPcmChunk(pcm, read, 0, 0, false); + } + } + private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); long now = SystemClock.elapsedRealtime(); @@ -582,6 +845,83 @@ public class FaceclawVoiceController { mainHandler.post(() -> currentListener.onPcm(le)); } + /** + * Embed the session's buffered utterance and compare it to the enrolled + * wearer voice-print. Fails open: a session too short to verify, or a + * model that will not load, counts as the wearer rather than silencing + * every command. + */ + private void runSpeakerVerification() { + short[] buffer = verifyBuffer; + float[] wearer = verifyWearerEmbedding; + String modelPath = verifySpeakerModelPath; + verifyBuffer = null; + if (buffer == null || wearer == null || modelPath == null) { + return; + } + if (verifyCount < VERIFY_MIN_SAMPLES) { + emitSpeakerVerified(true, 0f); + return; + } + FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath); + try { + byte[] le = new byte[verifyCount * 2]; + for (int i = 0; i < verifyCount; i++) { + short s = buffer[i]; + le[i * 2] = (byte) (s & 0xff); + le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); + } + float[] embedding = speakerId.embed(le, SAMPLE_RATE); + if (embedding == null || embedding.length != wearer.length) { + emitSpeakerVerified(true, 0f); + return; + } + double dot = 0; + for (int i = 0; i < embedding.length; i++) { + dot += (double) embedding[i] * wearer[i]; + } + boolean isWearer = dot >= verifyThreshold; + Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot) + + " threshold=" + verifyThreshold + " isWearer=" + isWearer); + emitSpeakerVerified(isWearer, (float) dot); + } catch (Throwable t) { + Log.w(TAG, "speaker verification failed", t); + emitSpeakerVerified(true, 0f); + } + } + + // The 28 MB embedding model takes seconds to load; keep one instance + // across capture sessions so verification adds only the embed time. + private static FaceclawSpeakerId sharedSpeakerId; + private static String sharedSpeakerIdPath; + + private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) { + if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) { + if (sharedSpeakerId != null) { + sharedSpeakerId.close(); + } + sharedSpeakerId = new FaceclawSpeakerId(modelPath); + sharedSpeakerIdPath = modelPath; + } + return sharedSpeakerId; + } + + private void emitSpeakerVerified(boolean isWearer, float similarity) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity)); + } + + private void emitFrameMeta(int angleDegrees, int ssr) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr)); + } + private void emitSpeechEnd() { FaceclawVoiceControllerListener currentListener = listener; if (currentListener == null) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index 504a5df3..c60ec655 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -11,15 +11,34 @@ public interface FaceclawVoiceControllerListener { void onTranscript(String text, boolean isFinal); /** - * Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit - * little-endian PCM. Empty/absent in onboard mode. + * Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM. + * Emitted in every mode so raw-PCM consumers (levels, recording, captions) + * keep working while the onboard recognizer owns the transcript. */ void onPcm(byte[] pcm16le); + /** + * Per-packet DSP metadata the glasses firmware appends to each 50 ms audio + * packet, computed on the raw stereo capture before its mono downmix: the + * direction-of-arrival in signed degrees (0 = straight ahead, positive to + * the wearer's right) and a signal-strength ratio that serves as a + * confidence/energy proxy for the angle. + */ + void onFrameMeta(int angleDegrees, int ssr); + /** * The speaker stopped, in a hands-free session that has no button release * to end it. Only fires when endpointing was enabled for the session; see * FaceclawVoiceController.setEndpointing. */ void onSpeechEnd(); + + /** + * Result of speaker verification against the enrolled wearer voice-print, + * emitted once per capture session just before the final transcript. + * Only fires when verification was configured for the session; fails + * open (isWearer=true) when the utterance was too short to verify or the + * speaker model could not be loaded. + */ + void onSpeakerVerified(boolean isWearer, float similarity); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWavRecorder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWavRecorder.java new file mode 100644 index 00000000..ed08d388 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWavRecorder.java @@ -0,0 +1,100 @@ +package com.faceclaw.app; + +import android.util.Log; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +/** + * Streaming WAV writer for conversation recordings: the header is written up + * front with zero sizes and patched on finish, so a crash mid-session leaves + * a recoverable file (data length can be derived from the file size). + */ +public class FaceclawWavRecorder { + private static final String TAG = "FaceclawWavRecorder"; + + private final String path; + private final int sampleRate; + private final int channels; + private RandomAccessFile file; + private long dataBytes; + + public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException { + this.path = path; + this.sampleRate = sampleRate; + this.channels = channels; + File parent = new File(path).getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IOException("could not create " + parent); + } + file = new RandomAccessFile(path, "rw"); + file.setLength(0); + file.write(buildHeader(0)); + } + + public synchronized void append(byte[] pcm16le) { + if (file == null || pcm16le == null || pcm16le.length == 0) { + return; + } + try { + file.write(pcm16le); + dataBytes += pcm16le.length; + } catch (IOException e) { + Log.w(TAG, "append failed", e); + } + } + + public synchronized long getDurationMs() { + return dataBytes * 1000L / (sampleRate * channels * 2L); + } + + public String getPath() { + return path; + } + + /** Patch the header and close. Returns the PCM byte count (0 = empty). */ + public synchronized long finish() { + RandomAccessFile current = file; + file = null; + if (current == null) { + return dataBytes; + } + try { + current.seek(0); + current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE))); + current.close(); + } catch (IOException e) { + Log.w(TAG, "finish failed", e); + } + return dataBytes; + } + + /** Delete the (finished or abandoned) recording. */ + public synchronized void discard() { + finish(); + new File(path).delete(); + } + + private byte[] buildHeader(int pcmBytes) { + int byteRate = sampleRate * channels * 2; + ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN); + b.put("RIFF".getBytes(StandardCharsets.US_ASCII)); + b.putInt(36 + pcmBytes); + b.put("WAVE".getBytes(StandardCharsets.US_ASCII)); + b.put("fmt ".getBytes(StandardCharsets.US_ASCII)); + b.putInt(16); + b.putShort((short) 1); + b.putShort((short) channels); + b.putInt(sampleRate); + b.putInt(byteRate); + b.putShort((short) (channels * 2)); + b.putShort((short) 16); + b.put("data".getBytes(StandardCharsets.US_ASCII)); + b.putInt(pcmBytes); + return b.array(); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java new file mode 100644 index 00000000..8e8f96ef --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java @@ -0,0 +1,383 @@ +package com.faceclaw.app; + +import android.content.Context; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import com.google.android.gms.common.ConnectionResult; +import com.google.android.gms.common.GoogleApiAvailability; +import com.google.android.gms.wearable.CapabilityClient; +import com.google.android.gms.wearable.CapabilityInfo; +import com.google.android.gms.wearable.DataMap; +import com.google.android.gms.wearable.MessageEvent; +import com.google.android.gms.wearable.Node; +import com.google.android.gms.wearable.PutDataMapRequest; +import com.google.android.gms.wearable.PutDataRequest; +import com.google.android.gms.wearable.Wearable; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Phone side of the Wear OS integration: the single place that talks to the + * Wearable Data Layer (Google Play services), so the JS side only ever sees + * JSON strings and never touches Play services types. + * + * Inbound: FaceclawWearListenerService hands every watch message to + * {@link #handleMessage}; it is forwarded to the JS listener on the thread + * that registered it (the main isolate), mirroring FaceclawSettings. Messages + * that arrive before a listener exists (the watch poked a phone whose JS + * dashboard is not up yet) are acked with jsReady=false so the watch can say + * "open Faceclaw on your phone"; only idempotent state requests are kept for + * replay on registration — the watch already reported gestures and commands + * as failed, and executing them tens of seconds later would act on whatever + * the glasses happen to show by then. + * + * Outbound: {@link #publishState} mirrors the dashboard state into a Data + * Layer item (delivered even if the watch is out of range right now, and + * available to the watch app the moment it opens), while {@link #sendToWatch} + * sends fire-and-forget messages (acks, assistant text) to every reachable + * watch that advertises the Faceclaw watch-app capability. + * + * Everything is a no-op on phones without Google Play services. + */ +public final class FaceclawWearBridge { + private static final String TAG = "FaceclawWear"; + + /** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */ + public static final String CAPABILITY_WATCH = "faceclaw_watch"; + /** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */ + public static final String CAPABILITY_PHONE = "faceclaw_phone"; + + public static final String PATH_PREFIX = "/faceclaw"; + public static final String PATH_STATE = "/faceclaw/state"; + public static final String PATH_STATE_REQUEST = "/faceclaw/state/request"; + public static final String PATH_ACK = "/faceclaw/ack"; + public static final String PATH_EVENT = "/faceclaw/event"; + + private static final String STATE_KEY_JSON = "json"; + private static final String STATE_KEY_UPDATED_AT = "updatedAt"; + + /** Messages kept for a listener that is not registered yet. */ + private static final int PENDING_LIMIT = 16; + private static final long PENDING_MAX_AGE_MS = 30_000; + + private static volatile FaceclawWearBridge instance; + + private final Context context; + private final boolean available; + private final Object lock = new Object(); + private FaceclawWearListener listener; + private Handler listenerHandler; + private final ArrayDeque pending = new ArrayDeque<>(); + private final List watchNodes = new ArrayList<>(); + private String lastPublishedState = null; + + private static final class PendingMessage { + final String path; + final String json; + final String nodeId; + final long receivedAt; + + PendingMessage(String path, String json, String nodeId, long receivedAt) { + this.path = path; + this.json = json; + this.nodeId = nodeId; + this.receivedAt = receivedAt; + } + } + + private FaceclawWearBridge(Context context) { + this.context = context.getApplicationContext(); + boolean playServices; + try { + playServices = GoogleApiAvailability.getInstance() + .isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS; + } catch (Throwable error) { + Log.w(TAG, "Play services availability check failed", error); + playServices = false; + } + this.available = playServices; + if (!available) { + Log.i(TAG, "Google Play services unavailable; watch integration disabled"); + } + } + + /** Initialize (idempotent) and return the singleton. */ + public static FaceclawWearBridge getInstance(Context context) { + if (instance == null) { + synchronized (FaceclawWearBridge.class) { + if (instance == null) { + instance = new FaceclawWearBridge(context); + } + } + } + return instance; + } + + /** Whether the Wearable Data Layer can be used on this phone at all. */ + public boolean isAvailable() { + return available; + } + + /** + * Register the JS listener. Must be called from the thread whose isolate + * owns it; that thread's Looper is captured for dispatch. Replays any + * recently queued watch messages and refreshes the reachable-watch set. + */ + public void setListener(FaceclawWearListener newListener) { + Looper looper = Looper.myLooper(); + if (looper == null) { + Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified"); + } + List replay; + synchronized (lock) { + listener = newListener; + listenerHandler = looper != null ? new Handler(looper) : null; + replay = new ArrayList<>(pending); + pending.clear(); + } + long now = System.currentTimeMillis(); + for (PendingMessage message : replay) { + if (now - message.receivedAt <= PENDING_MAX_AGE_MS) { + dispatchMessage(message.path, message.json, message.nodeId); + } + } + refreshWatchNodes(); + } + + public void clearListener() { + synchronized (lock) { + listener = null; + listenerHandler = null; + } + } + + public boolean isWatchReachable() { + synchronized (lock) { + return !watchNodes.isEmpty(); + } + } + + /** Display name of a reachable watch ("" if none). */ + public String getWatchName() { + synchronized (lock) { + return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); + } + } + + /** Re-query which watches advertise the Faceclaw watch-app capability. */ + public void refreshWatchNodes() { + if (!available) return; + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener(this::updateWatchNodes) + .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); + } catch (Throwable error) { + Log.w(TAG, "capability query failed", error); + } + } + + /** + * Mirror the dashboard state to the watch. Identical consecutive states + * are not re-sent (the Data Layer would drop them anyway). + */ + public void publishState(String json) { + publishState(json, false); + } + + /** As {@link #publishState(String)}; `force` re-sends an unchanged state. */ + public void publishState(String json, boolean force) { + if (!available || json == null) return; + synchronized (lock) { + if (!force && json.equals(lastPublishedState)) return; + lastPublishedState = json; + } + try { + PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE); + DataMap map = request.getDataMap(); + map.putString(STATE_KEY_JSON, json); + map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis()); + PutDataRequest put = request.asPutDataRequest().setUrgent(); + Wearable.getDataClient(context) + .putDataItem(put) + .addOnFailureListener(error -> Log.w(TAG, "state publish failed", error)); + } catch (Throwable error) { + Log.w(TAG, "state publish failed", error); + } + } + + /** Send a message to every reachable watch running the Faceclaw watch app. */ + public void sendToWatch(String path, String json) { + if (!available) return; + List targets; + synchronized (lock) { + targets = new ArrayList<>(watchNodes); + } + if (targets.isEmpty()) { + // Nothing known yet (first message after boot): look the watch up, then send. + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener(info -> { + updateWatchNodes(info); + for (Node node : info.getNodes()) { + sendToNode(node.getId(), path, json); + } + }) + .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); + } catch (Throwable error) { + Log.w(TAG, "capability query failed", error); + } + return; + } + for (Node node : targets) { + sendToNode(node.getId(), path, json); + } + } + + /** Send a message to one specific node (e.g. an ack back to the sender). */ + public void sendToNode(String nodeId, String path, String json) { + if (!available || nodeId == null || nodeId.isEmpty()) return; + try { + byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8); + Wearable.getMessageClient(context) + .sendMessage(nodeId, path, payload) + .addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error)); + } catch (Throwable error) { + Log.w(TAG, "send " + path + " failed", error); + } + } + + /** Reply to a watch message; `seq` echoes the watch's sequence number. */ + public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) { + JSONObject ack = new JSONObject(); + try { + ack.put("seq", seq); + ack.put("ok", ok); + ack.put("jsReady", jsReady); + ack.put("message", message != null ? message : ""); + } catch (JSONException ignored) { + // The keys are constants; this cannot happen. + } + sendToNode(nodeId, PATH_ACK, ack.toString()); + } + + /** Entry point for FaceclawWearListenerService; may run on any thread. */ + void handleMessage(MessageEvent event) { + if (event == null) return; + String path = event.getPath(); + if (path == null || !path.startsWith(PATH_PREFIX)) return; + byte[] data = event.getData(); + String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}"; + String nodeId = event.getSourceNodeId(); + Log.d(TAG, "recv " + path + " seq=" + readSeq(json)); + + boolean delivered = dispatchMessage(path, json, nodeId); + if (delivered) return; + + // No JS listener yet: tell the watch so it can explain why nothing is + // happening. Only state requests are kept for replay (see class doc); + // everything else is dropped after the failure ack. + if (PATH_STATE_REQUEST.equals(path)) { + synchronized (lock) { + long now = System.currentTimeMillis(); + while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) { + pending.pollFirst(); + } + if (pending.size() >= PENDING_LIMIT) { + pending.pollFirst(); + } + pending.addLast(new PendingMessage(path, json, nodeId, now)); + } + } + sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses."); + } + + /** Entry point for FaceclawWearListenerService's capability callback. */ + void handleCapabilityChanged(CapabilityInfo info) { + if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return; + updateWatchNodes(info); + } + + private void updateWatchNodes(CapabilityInfo info) { + Set nodes = info != null ? info.getNodes() : Collections.emptySet(); + // Every reachable watch gets messages; a nearby (Bluetooth-linked) + // one is listed first so it is the name the phone UI shows. + List reachable = new ArrayList<>(); + for (Node node : nodes) { + if (node.isNearby()) reachable.add(0, node); else reachable.add(node); + } + boolean changed; + String name; + synchronized (lock) { + changed = !sameNodeIds(watchNodes, reachable); + watchNodes.clear(); + watchNodes.addAll(reachable); + name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); + } + if (changed) { + Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name); + boolean reachableNow = !reachable.isEmpty(); + FaceclawWearListener target; + Handler handler; + synchronized (lock) { + target = listener; + handler = listenerHandler; + } + if (target != null && handler != null) { + handler.post(() -> { + try { + target.onWatchConnection(reachableNow, name); + } catch (Exception error) { + Log.w(TAG, "watch connection listener failed", error); + } + }); + } + } + } + + private static boolean sameNodeIds(List a, List b) { + if (a.size() != b.size()) return false; + for (int i = 0; i < a.size(); i++) { + if (!a.get(i).getId().equals(b.get(i).getId())) return false; + } + return true; + } + + /** Post to the JS listener; false when there is none to post to. */ + private boolean dispatchMessage(String path, String json, String nodeId) { + FaceclawWearListener target; + Handler handler; + synchronized (lock) { + target = listener; + handler = listenerHandler; + } + if (target == null || handler == null) return false; + handler.post(() -> { + try { + target.onMessage(path, json, nodeId); + } catch (Exception error) { + Log.w(TAG, "wear message listener failed for " + path, error); + } + }); + return true; + } + + private static long readSeq(String json) { + try { + return new JSONObject(json).optLong("seq", 0); + } catch (JSONException error) { + return 0; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java new file mode 100644 index 00000000..590c87d6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java @@ -0,0 +1,19 @@ +package com.faceclaw.app; + +/** + * Callbacks from the Android Wear (Wear OS) bridge; implemented in JS. Every + * call is delivered on the thread that registered the listener (see + * FaceclawWearBridge.setListener). + */ +public interface FaceclawWearListener { + /** + * A message from the watch app. `path` is the Data Layer message path + * (e.g. "/faceclaw/input"); `json` is the UTF-8 payload (always a JSON + * object, "{}" when the watch sent none); `nodeId` identifies the watch + * so a reply can be addressed to it. + */ + void onMessage(String path, String json, String nodeId); + + /** The set of reachable watches running the Faceclaw watch app changed. */ + void onWatchConnection(boolean reachable, String watchName); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java new file mode 100644 index 00000000..7520d780 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java @@ -0,0 +1,26 @@ +package com.faceclaw.app; + +import android.util.Log; + +import com.google.android.gms.wearable.CapabilityInfo; +import com.google.android.gms.wearable.MessageEvent; +import com.google.android.gms.wearable.WearableListenerService; + +/** + * Play services starts this service (in the app process) for every Data + * Layer message under /faceclaw and for capability changes, whether or not + * the app is running. It only forwards to FaceclawWearBridge, which owns the + * hand-off to the JS side. + */ +public class FaceclawWearListenerService extends WearableListenerService { + @Override + public void onMessageReceived(MessageEvent event) { + Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId()); + FaceclawWearBridge.getInstance(this).handleMessage(event); + } + + @Override + public void onCapabilityChanged(CapabilityInfo info) { + FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java index fd1de064..4b599d67 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java @@ -15,13 +15,18 @@ public final class PreviewBitmapUtil { private PreviewBitmapUtil() {} public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) { + return fromGray(gray, width, height, brightenGamma, false); + } + + /** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */ + public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma, boolean green) { if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { throw new IllegalArgumentException("invalid gray preview buffer"); } int[] lut = new int[256]; for (int g = 0; g < 256; g++) { int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))); - lut[g] = 0xff000000 | (v << 16) | (v << 8) | v; + lut[g] = green ? (0xff000000 | (v << 8)) : (0xff000000 | (v << 16) | (v << 8) | v); } byte[] bytes = new byte[width * height]; gray.get(bytes); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java index f1efc7f1..b5660a5b 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java @@ -49,6 +49,14 @@ public class BleProtocol { // protobuf fields to stock implementations, so they are safely ignored. public static final int FACECLAW_WAKE_CONTROL_FIELD = 101; public static final int FACECLAW_WAKE_EVENT_FIELD = 102; + /** + * CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host + * writes an ['M','C',ver,op,...] record as settings field 103; each temple + * reports a 21-byte ['M','C',ver,...] status as field 104, both appended + * to settings read responses and as standalone commandId=3 pushes. + */ + public static final int FACECLAW_MIC_CONTROL_FIELD = 103; + public static final int FACECLAW_MIC_STATUS_FIELD = 104; public static final int FACECLAW_WAKE_OP_ACQUIRE = 1; public static final int FACECLAW_WAKE_OP_RELEASE = 2; public static final int FACECLAW_WAKE_OP_CLAIM = 3; @@ -405,6 +413,40 @@ public class BleProtocol { )); } + /** + * Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a + * settings write carrying CFW field 103. Magic zero makes it + * fire-and-forget like the wake lease: the firmware consumes the field + * before its stock nanopb decoder discards the unknown tag, and the + * "ack" is the field-104 status notify that follows. + */ + public static byte[] buildFaceclawMicControl(byte[] record) { + return concat(CollectionUtils.listOf( + encodeVarintField(1, 1), + encodeVarintField(2, 0), + encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record) + )); + } + + /** + * Extract the CFW mic status record (field 104) from a settings frame, or + * null when absent. The 21-byte body layout is decoded on the TS side; + * here we only validate the 'M','C',version prelude. + */ + public static byte[] parseFaceclawMicStatus(byte[] pb) { + if (pb == null) { + return null; + } + byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD); + if (body == null + || body.length < 3 + || body[0] != 'M' + || body[1] != 'C') { + return null; + } + return body; + } + /** * Return the uint16 wake nonce from a CFW field-102 notification, or -1 * when this is an ordinary settings frame. diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java index eb31a930..82a7ca44 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java @@ -267,6 +267,21 @@ public class MessageBuilder { ); } + /** CFW mic_control write (settings field 103) targeted at one temple. */ + public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) { + return new OutboundMessage( + "mic-control", + "mic control " + label + (leftArm ? " L" : " R"), + BleProtocol.SID_UI_SETTING, + BleProtocol.FLAG_REQUEST, + 0, + BleProtocol.buildFaceclawMicControl(record), + ACK_TIMEOUT_MS, + -1, + leftArm + ); + } + public OutboundMessage faceclawWearQuery(boolean leftArm) { return new OutboundMessage( "wear-query-control", diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FastClusteringConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FastClusteringConfig.java new file mode 100644 index 00000000..f2e95725 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FastClusteringConfig.java @@ -0,0 +1,44 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class FastClusteringConfig { + private final int numClusters; + private final float threshold; + + private FastClusteringConfig(Builder builder) { + this.numClusters = builder.numClusters; + this.threshold = builder.threshold; + } + + public static Builder builder() { + return new Builder(); + } + + public int getNumClusters() { + return numClusters; + } + + public float getThreshold() { + return threshold; + } + + public static class Builder { + private int numClusters = -1; + private float threshold = 0.5f; + + public FastClusteringConfig build() { + return new FastClusteringConfig(this); + } + + public Builder setNumClusters(int numClusters) { + this.numClusters = numClusters; + return this; + } + + public Builder setThreshold(float threshold) { + this.threshold = threshold; + return this; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java new file mode 100644 index 00000000..20c9ae7e --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java @@ -0,0 +1,62 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarization { + private long ptr = 0; + + public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = newFromFile(config); + if (ptr == 0) { + throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization"); + } + } + + public int getSampleRate() { + return getSampleRate(ptr); + } + + // Only config.clustering is used. All other fields are ignored + public void setConfig(OfflineSpeakerDiarizationConfig config) { + setConfig(ptr, config); + } + + public OfflineSpeakerDiarizationSegment[] process(float[] samples) { + return process(ptr, samples); + } + + public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) { + return processWithCallback(ptr, samples, callback, 0); + } + + public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) { + return processWithCallback(ptr, samples, callback, arg); + } + + protected void finalize() throws Throwable { + release(); + } + + // You'd better call it manually if it is not used anymore + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + private native int getSampleRate(long ptr); + + private native void delete(long ptr); + + private native long newFromFile(OfflineSpeakerDiarizationConfig config); + + private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config); + + private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples); + + private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java new file mode 100644 index 00000000..7787386d --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java @@ -0,0 +1,8 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +@FunctionalInterface +public interface OfflineSpeakerDiarizationCallback { + Integer invoke(int numProcessedChunks, int numTotalCunks, long arg); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java new file mode 100644 index 00000000..9965c574 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java @@ -0,0 +1,79 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarizationConfig { + private final OfflineSpeakerSegmentationModelConfig segmentation; + private final SpeakerEmbeddingExtractorConfig embedding; + private final FastClusteringConfig clustering; + private final float minDurationOn; + private final float minDurationOff; + + private OfflineSpeakerDiarizationConfig(Builder builder) { + this.segmentation = builder.segmentation; + this.embedding = builder.embedding; + this.clustering = builder.clustering; + this.minDurationOff = builder.minDurationOff; + this.minDurationOn = builder.minDurationOn; + } + + public static Builder builder() { + return new Builder(); + } + + public OfflineSpeakerSegmentationModelConfig getSegmentation() { + return segmentation; + } + + public SpeakerEmbeddingExtractorConfig getEmbedding() { + return embedding; + } + + public FastClusteringConfig getClustering() { + return clustering; + } + + public float getMinDurationOff() { + return minDurationOff; + } + + public float getMinDurationOn() { + return minDurationOn; + } + + public static class Builder { + private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build(); + private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build(); + private FastClusteringConfig clustering = FastClusteringConfig.builder().build(); + private float minDurationOn = 0.2f; + private float minDurationOff = 0.5f; + + public OfflineSpeakerDiarizationConfig build() { + return new OfflineSpeakerDiarizationConfig(this); + } + + public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) { + this.segmentation = segmentation; + return this; + } + + public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) { + this.embedding = embedding; + return this; + } + + public Builder setClustering(FastClusteringConfig clustering) { + this.clustering = clustering; + return this; + } + + public Builder setMinDurationOff(float minDurationOff) { + this.minDurationOff = minDurationOff; + return this; + } + + public Builder setMinDurationOn(float minDurationOn) { + this.minDurationOn = minDurationOn; + return this; + } + } + +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java new file mode 100644 index 00000000..1bb1a763 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java @@ -0,0 +1,27 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarizationSegment { + private final float start; + private final float end; + private final int speaker; + + public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) { + this.start = start; + this.end = end; + this.speaker = speaker; + } + + public float getStart() { + return start; + } + + public float getEnd() { + return end; + } + + public int getSpeaker() { + return speaker; + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java new file mode 100644 index 00000000..55df6c29 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java @@ -0,0 +1,52 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerSegmentationModelConfig { + private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote; + private final int numThreads; + private final boolean debug; + private final String provider; + + private OfflineSpeakerSegmentationModelConfig(Builder builder) { + this.pyannote = builder.pyannote; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build(); + private int numThreads = 1; + private boolean debug = true; + private String provider = "cpu"; + + public OfflineSpeakerSegmentationModelConfig build() { + return new OfflineSpeakerSegmentationModelConfig(this); + } + + public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) { + this.pyannote = pyannote; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setDebug(boolean debug) { + this.debug = debug; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + } +} \ No newline at end of file diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java new file mode 100644 index 00000000..51fd9987 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java @@ -0,0 +1,32 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerSegmentationPyannoteModelConfig { + private final String model; + + private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) { + this.model = builder.model; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + public static class Builder { + private String model = ""; + + public OfflineSpeakerSegmentationPyannoteModelConfig build() { + return new OfflineSpeakerSegmentationPyannoteModelConfig(this); + } + + public Builder setModel(String model) { + this.model = model; + return this; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java new file mode 100644 index 00000000..985ce8de --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java @@ -0,0 +1,77 @@ +// Copyright 2022-2023 by zhaoming +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OnlineStream { + private long ptr = 0; + + public OnlineStream() { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + this.ptr = 0; + } + + public OnlineStream(long ptr) { + this.ptr = ptr; + } + + public long getPtr() { + return ptr; + } + + public void setPtr(long ptr) { + this.ptr = ptr; + } + + public void acceptWaveform(float[] samples, int sampleRate) { + acceptWaveform(this.ptr, samples, sampleRate); + } + + public void inputFinished() { + inputFinished(this.ptr); + } + + public void setOption(String key, String value) { + setOption(this.ptr, key, value); + } + + public String getOption(String key) { + return getOption(this.ptr, key); + } + + public boolean hasOption(String key) { + return hasOption(this.ptr, key); + } + + public void release() { + close(); + } + + public void close() { + // stream object must be release after used + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + @Override + protected void finalize() throws Throwable { + close(); + super.finalize(); + } + + private native void acceptWaveform(long ptr, float[] samples, int sampleRate); + + private native void inputFinished(long ptr); + + private native void setOption(long ptr, String key, String value); + + private native String getOption(long ptr, String key); + + private native boolean hasOption(long ptr, String key); + + private native void delete(long ptr); +} \ No newline at end of file diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractor.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractor.java new file mode 100644 index 00000000..026b2ed6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractor.java @@ -0,0 +1,58 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class SpeakerEmbeddingExtractor { + private long ptr = 0; + + public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = newFromFile(config); + if (ptr == 0) { + throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor"); + } + } + + @Override + protected void finalize() throws Throwable { + release(); + } + + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + public OnlineStream createStream() { + long p = createStream(ptr); + return new OnlineStream(p); + } + + public boolean isReady(OnlineStream s) { + return isReady(ptr, s.getPtr()); + } + + public float[] compute(OnlineStream s) { + return compute(ptr, s.getPtr()); + } + + public int getDim() { + return dim(ptr); + } + + private native void delete(long ptr); + + private native long newFromFile(SpeakerEmbeddingExtractorConfig config); + + private native long createStream(long ptr); + + private native boolean isReady(long ptr, long streamPtr); + + private native float[] compute(long ptr, long streamPtr); + + private native int dim(long ptr); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java new file mode 100644 index 00000000..80f800cd --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java @@ -0,0 +1,53 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class SpeakerEmbeddingExtractorConfig { + private final String model; + private final int numThreads; + private final boolean debug; + private final String provider; + + private SpeakerEmbeddingExtractorConfig(Builder builder) { + this.model = builder.model; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String model = ""; + private int numThreads = 1; + private boolean debug = true; + private String provider = "cpu"; + + public SpeakerEmbeddingExtractorConfig build() { + return new SpeakerEmbeddingExtractorConfig(this); + } + + + public Builder setModel(String model) { + this.model = model; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setDebug(boolean debug) { + this.debug = debug; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingManager.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingManager.java new file mode 100644 index 00000000..29bc7cdc --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingManager.java @@ -0,0 +1,78 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class SpeakerEmbeddingManager { + private long ptr = 0; + + public SpeakerEmbeddingManager(int dim) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = create(dim); + } + + @Override + protected void finalize() throws Throwable { + release(); + } + + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + public boolean add(String name, float[] embedding) { + return add(ptr, name, embedding); + } + + public boolean add(String name, float[][] embedding) { + return addList(ptr, name, embedding); + } + + public boolean remove(String name) { + return remove(ptr, name); + } + + public String search(float[] embedding, float threshold) { + return search(ptr, embedding, threshold); + } + + public boolean verify(String name, float[] embedding, float threshold) { + return verify(ptr, name, embedding, threshold); + } + + public boolean contains(String name) { + return contains(ptr, name); + } + + public int getNumSpeakers() { + return numSpeakers(ptr); + } + + public String[] getAllSpeakerNames() { + return allSpeakerNames(ptr); + } + + private native long create(int dim); + + private native void delete(long ptr); + + private native boolean add(long ptr, String name, float[] embedding); + + private native boolean addList(long ptr, String name, float[][] embedding); + + private native boolean remove(long ptr, String name); + + private native String search(long ptr, float[] embedding, float threshold); + + private native boolean verify(long ptr, String name, float[] embedding, float threshold); + + private native boolean contains(long ptr, String name); + + private native int numSpeakers(long ptr); + + private native String[] allSpeakerNames(long ptr); +} diff --git a/App_Resources/Android/src/main/res/drawable/ic_tab_mic.xml b/App_Resources/Android/src/main/res/drawable/ic_tab_mic.xml new file mode 100644 index 00000000..3132c4ea --- /dev/null +++ b/App_Resources/Android/src/main/res/drawable/ic_tab_mic.xml @@ -0,0 +1,18 @@ + + + + + + diff --git a/App_Resources/Android/src/main/res/values/styles.xml b/App_Resources/Android/src/main/res/values/styles.xml index 5cb0a093..0eef3110 100644 --- a/App_Resources/Android/src/main/res/values/styles.xml +++ b/App_Resources/Android/src/main/res/values/styles.xml @@ -25,6 +25,17 @@ @color/ns_primary @color/ns_primaryDark @color/ns_accent + + + @style/FaceclawOverflowButtonStyle + + + + diff --git a/wear/app/src/main/res/values/wear.xml b/wear/app/src/main/res/values/wear.xml new file mode 100644 index 00000000..ebb4878f --- /dev/null +++ b/wear/app/src/main/res/values/wear.xml @@ -0,0 +1,8 @@ + + + + + faceclaw_watch + + diff --git a/wear/build.gradle.kts b/wear/build.gradle.kts new file mode 100644 index 00000000..0fdc2bca --- /dev/null +++ b/wear/build.gradle.kts @@ -0,0 +1,9 @@ +// Faceclaw watch app (Wear OS companion). A standalone Gradle project: it +// shares nothing with the NativeScript build except the applicationId and +// (for release builds) the signing key, both of which the Wearable Data +// Layer requires to match the phone app. +plugins { + id("com.android.application") version "8.7.3" apply false + id("org.jetbrains.kotlin.android") version "2.1.10" apply false + id("org.jetbrains.kotlin.plugin.compose") version "2.1.10" apply false +} diff --git a/wear/gradle.properties b/wear/gradle.properties new file mode 100644 index 00000000..5f4cff37 --- /dev/null +++ b/wear/gradle.properties @@ -0,0 +1,4 @@ +org.gradle.jvmargs=-Xmx2g -Dfile.encoding=UTF-8 +android.useAndroidX=true +android.nonTransitiveRClass=true +kotlin.code.style=official diff --git a/wear/gradle/wrapper/gradle-wrapper.jar b/wear/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..249e5832 Binary files /dev/null and b/wear/gradle/wrapper/gradle-wrapper.jar differ diff --git a/wear/gradle/wrapper/gradle-wrapper.properties b/wear/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..d4081da4 --- /dev/null +++ b/wear/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/wear/gradlew b/wear/gradlew new file mode 100755 index 00000000..a69d9cb6 --- /dev/null +++ b/wear/gradlew @@ -0,0 +1,240 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/wear/gradlew.bat b/wear/gradlew.bat new file mode 100644 index 00000000..f127cfd4 --- /dev/null +++ b/wear/gradlew.bat @@ -0,0 +1,91 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/wear/settings.gradle.kts b/wear/settings.gradle.kts new file mode 100644 index 00000000..b273625d --- /dev/null +++ b/wear/settings.gradle.kts @@ -0,0 +1,16 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} +rootProject.name = "faceclaw-wear" +include(":app") diff --git a/website/CNAME b/website/CNAME new file mode 100644 index 00000000..c248600c --- /dev/null +++ b/website/CNAME @@ -0,0 +1 @@ +faceclaw.org diff --git a/website/README.md b/website/README.md new file mode 100644 index 00000000..bc65a419 --- /dev/null +++ b/website/README.md @@ -0,0 +1,59 @@ +# faceclaw.org + +The Faceclaw website: plain HTML and one stylesheet, no build step and no +external dependencies. GitHub Actions publishes this directory to GitHub Pages +on every push to `main` that touches it (`.github/workflows/pages.yml`). + +## Layout + + index.html landing page + install.html install instructions + privacy.html privacy policy (body generated, see below) + style.css the whole stylesheet + content/ canonical copies of content shared with the repo root + images/ logo and favicon, derived from ../images/FaceclawAppIcon.png + screenshots/ glasses captures, also used by the root README + CNAME custom domain + +## Shared content + +Two things appear both here and in the repository root, and are generated +rather than copied by hand: + +| Source | Destinations | +| --------------------------- | ------------------------------------- | +| `content/features.md` | `README.md`, `index.html` | +| `../PRIVACY` | `privacy.html` | + +Edit the source, then run: + + node scripts/sync-site.mjs + +Each destination has a `` / `` +pair; the script rewrites what is between them and leaves the rest alone. CI +runs `node scripts/sync-site.mjs --check` and fails the deploy if a +destination is stale. + +## Previewing + +Any static file server will do, since there is nothing to compile: + + python3 -m http.server -d website 8000 + +## Setting up the custom domain + +Publishing needs Settings > Pages > Source set to "GitHub Actions". For +`faceclaw.org`, point the apex at GitHub's Pages addresses: + + A faceclaw.org 185.199.108.153 + A faceclaw.org 185.199.109.153 + A faceclaw.org 185.199.110.153 + A faceclaw.org 185.199.111.153 + AAAA faceclaw.org 2606:50c0:8000::153 + AAAA faceclaw.org 2606:50c0:8001::153 + AAAA faceclaw.org 2606:50c0:8002::153 + AAAA faceclaw.org 2606:50c0:8003::153 + CNAME www jimrandomh.github.io. + +Then set the custom domain in Settings > Pages and enable "Enforce HTTPS" +once the certificate has been issued. diff --git a/website/content/features.md b/website/content/features.md new file mode 100644 index 00000000..b59baa8d --- /dev/null +++ b/website/content/features.md @@ -0,0 +1,24 @@ + + +- **A voice assistant** that wakes up when you say "Hey Even", transcribes text with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and responds to queries and commands using an onboard model (Qwen3 4B, slow) or with an Anthropic or OpenAI model (requires an API key), or using your own long-running OpenClaw agent. +- **Multitasking**, with an app-switcher sidebar and app launcher. +- **Mostly-compatible with EvenHub apps.** +- **A lock screen**; glasses lock automatically when you take them off and unlock when you unlock your phone. +- **The full display.** Full-screen apps can use the full 640x480 display area, rather than the 576x288 that EvenHub apps can use. +- **Integration with Android notifications**: a top bar that shows the same icons your phone does, popups when notifications arrive, and menu items to dismiss or use Android-app-provided custom actions like mark as read or quick reply. +- **Terminal mirroring.** Mirror terminal apps such as Claude Code or Codex CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), view them on the glasses, and send them inputs with the voice assistant. +- **Media player controls** including playlist and media library navigation, compatible with most Android media players. +- **Turn-by-turn directions** (requires a Mapbox API token). +- **Nightscout**, an app for viewing blood-glucose data (requires a cloud server and API token). +- **Power management**: the glasses go to sleep properly when the screen is off, and wake when you double-tap the ring or speak the wakeword, allowing battery life similar to the stock Even app. +- **Connection management** with auto-reconnect, and autodetection of conflict with the official Even Realities app. +- **A Wear OS watch app** that replaces (and outdoes) the R1 ring: tap, swipe, hold and crown gestures, side buttons, app launching and window switching, voice or keyboard queries to the assistant with the reply on your wrist, typing into apps, and glasses status/lock/display control. +- **On-phone screen mirroring with touch control**: tap what you see on the mirror (sidebar icons, launcher cells), or use the phone's own touchpad, d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus a compact ring simulator. A display-mode picker (576×288 band, 576×480 tall, or the full 640×480 panel with an auto-hiding sidebar) and a brightness slider with an Auto toggle sit beside the mirror. +- **Bluetooth pairing** that scans for nearby glasses and identifies each pair before connecting: model, frame shape, and colour decoded from the advertised serial (with product photos), left and right arms matched to each other by that serial, an estimated distance so the pair in your hand sorts first, and the optional R1 ring. +- **Dual-language NativeScript architecture**, with Java for the multithreaded Android API and bluetooth stack bits, Typescript for the bits you want to hack on. diff --git a/website/images/faceclaw-logo.png b/website/images/faceclaw-logo.png new file mode 100644 index 00000000..36053dff Binary files /dev/null and b/website/images/faceclaw-logo.png differ diff --git a/website/images/favicon.png b/website/images/favicon.png new file mode 100644 index 00000000..805cc0c0 Binary files /dev/null and b/website/images/favicon.png differ diff --git a/website/index.html b/website/index.html new file mode 100644 index 00000000..755c4dfb --- /dev/null +++ b/website/index.html @@ -0,0 +1,147 @@ + + + + + +Faceclaw — an unofficial interface for the Even Realities G2 + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ +

Faceclaw

+

An unofficial user interface for the Even Realities G2 smart glasses. Apps, a voice assistant, notifications and a watch remote — running from your Android phone.

+ +
+
+ +
+
+
+

Unofficial

+

Faceclaw is not made by, endorsed by, or supported by Even Realities. It comes with no warranty from them or from anyone. It runs on Android, and it requires installing custom firmware on the glasses — the app can install and uninstall that firmware for you, downloading the stock firmware from Even and applying patches generated from g2flash.

+
+
+
+ +
+
+

What it looks like

+
+
+ The Faceclaw app launcher on the glasses display +
The app launcher, with the app-switcher sidebar on the left.
+
+
+ The music player on the glasses display +
Media controls, working with most Android media players.
+
+
+ Assistant settings on the glasses display +
Assistant settings, edited on the glasses themselves.
+
+
+ Display settings on the glasses display +
Display settings, including height and vertical position.
+
+
+
+
+ +
+
+

Features

+ +
    +
  • A voice assistant that wakes up when you say "Hey Even", transcribes text with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and responds to queries and commands using an onboard model (Qwen3 4B, slow) or with an Anthropic or OpenAI model (requires an API key), or using your own long-running OpenClaw agent.
  • +
  • Multitasking, with an app-switcher sidebar and app launcher.
  • +
  • Mostly-compatible with EvenHub apps.
  • +
  • A lock screen; glasses lock automatically when you take them off and unlock when you unlock your phone.
  • +
  • The full display. Full-screen apps can use the full 640x480 display area, rather than the 576x288 that EvenHub apps can use.
  • +
  • Integration with Android notifications: a top bar that shows the same icons your phone does, popups when notifications arrive, and menu items to dismiss or use Android-app-provided custom actions like mark as read or quick reply.
  • +
  • Terminal mirroring. Mirror terminal apps such as Claude Code or Codex CLI with g2mirror, view them on the glasses, and send them inputs with the voice assistant.
  • +
  • Media player controls including playlist and media library navigation, compatible with most Android media players.
  • +
  • Turn-by-turn directions (requires a Mapbox API token).
  • +
  • Nightscout, an app for viewing blood-glucose data (requires a cloud server and API token).
  • +
  • Power management: the glasses go to sleep properly when the screen is off, and wake when you double-tap the ring or speak the wakeword, allowing battery life similar to the stock Even app.
  • +
  • Connection management with auto-reconnect, and autodetection of conflict with the official Even Realities app.
  • +
  • A Wear OS watch app that replaces (and outdoes) the R1 ring: tap, swipe, hold and crown gestures, side buttons, app launching and window switching, voice or keyboard queries to the assistant with the reply on your wrist, typing into apps, and glasses status/lock/display control.
  • +
  • On-phone screen mirroring with touch control: tap what you see on the mirror (sidebar icons, launcher cells), or use the phone's own touchpad, d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus a compact ring simulator. A display-mode picker (576×288 band, 576×480 tall, or the full 640×480 panel with an auto-hiding sidebar) and a brightness slider with an Auto toggle sit beside the mirror.
  • +
  • Bluetooth pairing that scans for nearby glasses and identifies each pair before connecting: model, frame shape, and colour decoded from the advertised serial (with product photos), left and right arms matched to each other by that serial, an estimated distance so the pair in your hand sorts first, and the optional R1 ring.
  • +
  • Dual-language NativeScript architecture, with Java for the multithreaded Android API and bluetooth stack bits, Typescript for the bits you want to hack on.
  • +
+ +
+
+ +
+
+

Beyond the glasses

+
+
+

Watch remote

+

A Wear OS companion app turns the whole watch face into a touchpad that outdoes the R1 ring: swipes, crown scrolling, side buttons, app launching, and spoken or typed queries with the reply on your wrist.

+
+
+

EvenHub apps

+

Faceclaw runs EvenHub apps through an emulation layer. Install them from the Hub, or sideload your own by opening an .ehpk file on your phone.

+
+
+

Your own agent

+

Instead of calling an LLM API from the phone, the assistant can route queries to your own long-running OpenClaw agent, which also gets access to the glasses' tools.

+
+
+
+
+ +
+
+

Contributing

+

Be bold. Modify Faceclaw into the app that you want it to be for yourself, without worrying about whether other people will like your version — then, if you think your changes might be useful to others, open a pull request. Faceclaw is Free Software under the GPLv3. See the contributing notes for what to test before you send a change.

+
+
+ +
+ + + + + diff --git a/website/install.html b/website/install.html new file mode 100644 index 00000000..66323def --- /dev/null +++ b/website/install.html @@ -0,0 +1,80 @@ + + + + + +Install — Faceclaw + + + + + + +
+ +
+ +
+
+
+

Installing Faceclaw

+ +
+

Read this first

+

Faceclaw is unofficial and unsupported, and it replaces the firmware on your glasses. Installing it means running code that Even Realities did not write and will not help you with. The app can put the stock firmware back, but you are doing this at your own risk.

+
+ +

What you need

+
    +
  • A pair of Even Realities G2 glasses.
  • +
  • An Android phone. Faceclaw runs as a background app and talks to the glasses over Bluetooth LE.
  • +
  • Optionally, a Wear OS 3+ watch, if you want the watch remote instead of the R1 ring.
  • +
+

There is no iOS version, and there will not be one — the app depends on Android APIs for background Bluetooth, notification access and media control.

+ +

Install the app

+

Faceclaw is not on the Play Store. Download the most recent APK from the releases page and open it on your phone; Android will ask you to allow installing apps from that source.

+ +

Install the firmware

+

Faceclaw needs custom firmware on the glasses, and installs it for you. On first run, the onboarding flow will offer to download the stock firmware from Even Realities, apply the g2flash patches locally, and flash the result. You confirm the flash on the glasses themselves.

+

Onboarding also offers a preview-only path, which connects to unmodified glasses so you can look around before committing. Faceclaw can restore the stock firmware later from the same screens.

+ +

Grant permissions

+

Android does not reliably prompt for everything Faceclaw uses. After installing, go to Settings → Apps → Faceclaw and check that permissions are granted — notification access and Bluetooth in particular — or parts of the app will silently do nothing.

+

Also exempt Faceclaw from battery optimization. Being a background app on a locked Android phone is fraught, and manufacturer-specific battery software may pause it or throttle it to low CPU usage.

+ +

Optional extras

+
    +
  • Watch remote. The Wear OS app is a separate Gradle build in wear/, sideloaded over adb; it needs no Play Store, but the phone side requires Google Play services.
  • +
  • API keys. The voice assistant needs a model provider key (Anthropic or OpenAI), cloud transcription needs a key for OpenAI Whisper, ElevenLabs or Soniox, and turn-by-turn directions need a Mapbox token. On-device transcription needs no key.
  • +
  • Terminal mirroring. Install g2mirror on the machine whose terminal you want to see.
  • +
  • External agent. To route assistant queries to your own long-running agent, install the faceclaw-agent-bridge OpenClaw plugin; setup lives in that repository's README.
  • +
+ +

Building from source

+

Faceclaw is a NativeScript app written in TypeScript and Java, and builds with an Android SDK 35 environment and a JDK 21. The README has the current build and adb instructions; they are kept there rather than duplicated here, so they stay in step with the code.

+
+
+
+ +
+
+ +

Faceclaw is Free Software, licensed under the GPLv3. It is unofficial, and is not affiliated with Even Realities.

+
+
+ + + diff --git a/website/privacy.html b/website/privacy.html new file mode 100644 index 00000000..3cd84806 --- /dev/null +++ b/website/privacy.html @@ -0,0 +1,80 @@ + + + + + +Privacy policy — Faceclaw + + + + + + +
+ +
+ +
+
+
+ +

Faceclaw Privacy Policy

+

Faceclaw is a privacy-conscious app. The developers of Faceclaw don't collect or store user data. Faceclaw does integrate with some third-party services, which have their own privacy policies; this document describes what those services are, what they're used for and what data they collect, and links to their privacy policies.

+

Audio

+

Faceclaw allows users to transcribe audio (for use with the voice assistant, the Transcribe app, typing into Terminal, and other apps). The default setting is that this is done using an on-device model running on your phone. If an on-device model is selected, audio never leaves your device, and is not stored after it is processed.

+

Users can also provide their own API key for a third-party transcription service such as OpenAI Whisper, ElevenLabs, or Soniox, which may offer better accuracy. Faceclaw never transmits audio to any party other than a user-selected transcription service. If a third-party transcription service is selected, audio will be sent to that third party, is linked to the account with which that API key was created, and is subject to that party's privacy and retention policies:

+ +

Third party data retention and privacy policies may differ based on the API key provided. API keys are part of a relationship between the user and the API provider; Faceclaw does not manage accounts or process billing information.

+

AI Assistant Transcripts

+

Faceclaw has a builtin AI voice assistant. Using the voice assistant requires providing an API key for a model provider such as Anthropic or OpenAI. Prompts and responses are subject to that model provider's privacy policy:

+ +

AI conversation histories and memory files may be stored on your phone. Faceclaw does not transmit voice-assistant prompts or responses to any party other than a user-selected AI model provider. The voice assistant is given tools with which it is able to access most things on the user's phone, including all content displayed on the glasses, location, notifications, and content accessible through the Terminal app.

+

The AI assistant can take actions on the user's behalf, including sharing any information it has access to. The user is responsible for any instructions they give it.

+

Location

+

Faceclaw has a builtin Navigate app, which provides turn-by-turn directions. Using the Navigate app requires providing an API key for Mapbox. While the Navigate app is active, the user's exact location and selected destination are transmitted to Mapbox, and are subject to Mapbox's privacy policy.

+ +

The Weather app will retrieve the user's coarse location, and will use it to query a third-party weather information provider.

+

EvenHub

+

The EvenHub app store is subject to Even Realities' privacy policy: https://support.evenrealities.com/hc/en-us/articles/14127464826511-Privacy-Policy. Apps downloaded from the EvenHub app store come with their own privacy policies.

+

Wear OS Watch

+

The optional Faceclaw watch app (wear/) talks to the phone over Google Play services' Wearable Data Layer, the same channel the watch uses for notifications and other companion apps. Gestures, commands, the watch's own speech-recognition results, and the mirrored glasses status and assistant replies travel between your watch and your phone over their Bluetooth (or, when paired that way, Wi-Fi) link; Faceclaw does not send any of it to a server. Speech recognized on the watch uses the watch's system speech recognizer, which may send audio to Google under Google's privacy policy:

+ +

The phone does nothing watch-related on phones without Google Play services.

+ +

This page is generated from the PRIVACY file in the Faceclaw repository, which is the authoritative copy.

+
+
+
+ + + + + diff --git a/screenshots/launcher.png b/website/screenshots/launcher.png similarity index 100% rename from screenshots/launcher.png rename to website/screenshots/launcher.png diff --git a/screenshots/music-player.png b/website/screenshots/music-player.png similarity index 100% rename from screenshots/music-player.png rename to website/screenshots/music-player.png diff --git a/screenshots/settings-assistant.png b/website/screenshots/settings-assistant.png similarity index 100% rename from screenshots/settings-assistant.png rename to website/screenshots/settings-assistant.png diff --git a/screenshots/settings-display.png b/website/screenshots/settings-display.png similarity index 100% rename from screenshots/settings-display.png rename to website/screenshots/settings-display.png diff --git a/website/style.css b/website/style.css new file mode 100644 index 00000000..e067c5e5 --- /dev/null +++ b/website/style.css @@ -0,0 +1,178 @@ +/* Faceclaw website. Hand-written, no build step, no external assets. */ + +:root { + --bg: #0d1226; + --bg-raised: #141b38; + --line: #26305a; + --fg: #e9ecf7; + --fg-dim: #98a1c4; + --accent: #f5c531; + --accent-warm: #e8503a; + --lens: #000; + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + --sans: system-ui, -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--bg); + color: var(--fg); + font: 16px/1.65 var(--sans); + -webkit-font-smoothing: antialiased; +} + +.wrap { width: 100%; max-width: 960px; margin: 0 auto; padding: 0 24px; } + +a { color: var(--accent); text-decoration-color: rgba(245, 197, 49, 0.4); text-underline-offset: 2px; } +a:hover { text-decoration-color: var(--accent); } + +h1, h2, h3 { line-height: 1.2; letter-spacing: -0.015em; } +h2 { font-size: 1.5rem; margin: 0 0 16px; } +h3 { font-size: 1.05rem; margin: 0 0 8px; } +p { margin: 0 0 16px; } + +code, kbd { font-family: var(--mono); font-size: 0.9em; } +code { + background: var(--bg-raised); + border: 1px solid var(--line); + border-radius: 4px; + padding: 1px 5px; +} +pre { + background: var(--bg-raised); + border: 1px solid var(--line); + border-radius: 8px; + padding: 14px 16px; + overflow-x: auto; + margin: 0 0 16px; +} +pre code { background: none; border: 0; padding: 0; } + +/* ---- site chrome ---- */ + +.topbar { + border-bottom: 1px solid var(--line); + position: sticky; + top: 0; + background: rgba(13, 18, 38, 0.9); + backdrop-filter: blur(8px); + z-index: 10; +} +.topbar .wrap { display: flex; align-items: center; gap: 20px; height: 56px; } +.brand { display: flex; align-items: center; gap: 9px; font-weight: 650; color: var(--fg); text-decoration: none; } +.brand img { width: 22px; height: 22px; } +.topbar nav { margin-left: auto; display: flex; gap: 20px; font-size: 0.94rem; } +.topbar nav a { color: var(--fg-dim); text-decoration: none; } +.topbar nav a:hover, .topbar nav a[aria-current="page"] { color: var(--fg); } + +section { padding: 56px 0; border-top: 1px solid var(--line); } +section:first-of-type { border-top: 0; } + +footer { + border-top: 1px solid var(--line); + padding: 32px 0 56px; + color: var(--fg-dim); + font-size: 0.9rem; +} +footer a { color: var(--fg-dim); } +footer .links { display: flex; flex-wrap: wrap; gap: 18px; margin-bottom: 12px; } + +/* ---- hero ---- */ + +.hero { text-align: center; padding: 72px 0 56px; } +.hero img.logo { width: 104px; height: 104px; } +.hero h1 { font-size: clamp(2.2rem, 6vw, 3.1rem); margin: 18px 0 10px; } +.hero .tagline { font-size: 1.2rem; color: var(--fg-dim); max-width: 34em; margin: 0 auto 28px; } + +.buttons { display: flex; gap: 12px; justify-content: center; flex-wrap: wrap; } +.btn { + display: inline-block; + padding: 11px 22px; + border-radius: 8px; + border: 1px solid var(--line); + font-weight: 600; + text-decoration: none; + color: var(--fg); +} +.btn:hover { border-color: var(--fg-dim); } +.btn.primary { background: var(--accent); border-color: var(--accent); color: #1a1400; } +.btn.primary:hover { background: #ffd44f; border-color: #ffd44f; } + +.eyebrow { + font-family: var(--mono); + font-size: 0.78rem; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--accent-warm); +} + +/* ---- callout ---- */ + +.callout { + border: 1px solid var(--line); + border-left: 3px solid var(--accent-warm); + border-radius: 8px; + background: var(--bg-raised); + padding: 16px 20px; + margin: 0 0 24px; +} +.callout p:last-child { margin-bottom: 0; } + +/* ---- screenshots ---- */ + +.shots { display: grid; grid-template-columns: repeat(2, 1fr); gap: 24px; } +.shot { margin: 0; } +.shot img { + display: block; + width: 100%; + background: var(--lens); + border: 1px solid var(--line); + border-radius: 6px; + image-rendering: pixelated; +} +.shot figcaption { color: var(--fg-dim); font-size: 0.88rem; margin-top: 8px; } + +/* ---- features ---- */ + +.features { list-style: none; padding: 0; margin: 0; columns: 2; column-gap: 40px; } +.features li { + break-inside: avoid; + margin: 0 0 18px; + padding-left: 20px; + position: relative; + color: var(--fg-dim); +} +.features li::before { content: "▸"; position: absolute; left: 0; color: var(--accent); } +.features li strong { color: var(--fg); font-weight: 600; } + +/* ---- cards ---- */ + +.cards { display: grid; grid-template-columns: repeat(3, 1fr); gap: 20px; } +.card { + border: 1px solid var(--line); + border-radius: 10px; + background: var(--bg-raised); + padding: 20px; +} +.card p { color: var(--fg-dim); font-size: 0.94rem; margin: 0; } + +/* ---- long-form prose (privacy, install) ---- */ + +.prose { max-width: 44em; } +.prose h2 { margin-top: 40px; padding-top: 8px; } +.prose h2:first-child { margin-top: 0; } +.prose ul { padding-left: 22px; } +.prose li { margin-bottom: 6px; } +.prose ul.links { list-style: none; padding-left: 0; font-family: var(--mono); font-size: 0.88rem; } +.prose ul.links li { margin-bottom: 4px; } +.prose .meta { color: var(--fg-dim); font-size: 0.9rem; } + +@media (max-width: 700px) { + .shots { grid-template-columns: 1fr; } + .features { columns: 1; } + .cards { grid-template-columns: 1fr; } + .topbar nav { gap: 14px; } + section { padding: 40px 0; } +}