Merge branch 'main' into ble-bandwidth-test

This commit is contained in:
James Babcock committed 2026-08-31 01:24:53 -07:00
commit 0114c521ad
316 files changed
+32980 -882

No files matched your search

@@ -77,6 +77,7 @@
android:name="com.tns.NativeScriptActivity"
android:label="@string/title_activity_kimera"
android:configChanges="keyboard|keyboardHidden|orientation|screenSize|smallestScreenSize|screenLayout|locale|uiMode"
android:windowSoftInputMode="adjustResize"
android:theme="@style/LaunchScreenTheme"
android:hardwareAccelerated="true"
android:launchMode="singleTask"
@@ -110,9 +111,34 @@
<action android:name="android.service.notification.NotificationListenerService" />
</intent-filter>
</service>
<!-- Wear OS: Play services routes watch-app messages (paths under
/faceclaw) and watch reachability changes here, starting the app
process if needed. See FaceclawWearBridge. -->
<service
android:name="com.faceclaw.app.FaceclawWearListenerService"
android:enabled="true"
android:exported="true">
<intent-filter>
<action android:name="com.google.android.gms.wearable.MESSAGE_RECEIVED" />
<data android:scheme="wear" android:host="*" android:pathPrefix="/faceclaw" />
</intent-filter>
<intent-filter>
<action android:name="com.google.android.gms.wearable.CAPABILITY_CHANGED" />
<data android:scheme="wear" android:host="*" android:pathPrefix="/" />
</intent-filter>
</service>
<receiver
android:name="com.faceclaw.app.FaceclawTimerReceiver"
android:enabled="true"
android:exported="false" />
<!-- adb-only settings export/import (see FaceclawSettingsPortReceiver).
DUMP is a development permission: the adb shell holds it, other
apps cannot get it granted without adb, so this stays reachable
from the command line but not from apps on the phone. -->
<receiver
android:name="com.faceclaw.app.FaceclawSettingsPortReceiver"
android:enabled="true"
android:exported="true"
android:permission="android.permission.DUMP" />
</application>
</manifest>
@@ -0,0 +1,242 @@
package com.faceclaw.app;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
* container, cutting speech recordings to roughly a tenth of their WAV size.
* Runs on its own thread; the source WAV is deleted on success when requested.
*/
public class FaceclawAudioTranscoder {
private static final String TAG = "FaceclawTranscode";
private static final long CODEC_TIMEOUT_US = 10_000;
public interface Listener {
void onDone(String outputPath, long outputBytes);
void onError(String message);
}
private final String wavPath;
private final String outputPath;
private final int bitrateBps;
private final boolean deleteSourceOnSuccess;
private final Listener listener;
public FaceclawAudioTranscoder(
String wavPath,
String outputPath,
int bitrateBps,
boolean deleteSourceOnSuccess,
Listener listener) {
this.wavPath = wavPath;
this.outputPath = outputPath;
this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000;
this.deleteSourceOnSuccess = deleteSourceOnSuccess;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawTranscode");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
private void run() {
try {
transcode();
long size = new File(outputPath).length();
if (deleteSourceOnSuccess) {
new File(wavPath).delete();
}
if (listener != null) {
listener.onDone(outputPath, size);
}
} catch (Throwable t) {
Log.w(TAG, "transcode failed for " + wavPath, t);
new File(outputPath).delete();
if (listener != null) {
listener.onError(t.getMessage() == null ? t.toString() : t.getMessage());
}
}
}
private void transcode() throws IOException {
WavInfo wav = readWavHeader(wavPath);
MediaFormat format = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
MediaMuxer muxer = null;
FileInputStream input = null;
try {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
codec.start();
muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
input = new FileInputStream(wavPath);
long skipped = 0;
while (skipped < wav.dataOffset) {
long step = input.skip(wav.dataOffset - skipped);
if (step <= 0) {
throw new IOException("could not seek past WAV header");
}
skipped += step;
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
byte[] chunk = new byte[8192];
int track = -1;
boolean muxerStarted = false;
boolean inputDone = false;
boolean outputDone = false;
long remaining = wav.dataBytes;
long presentationUs = 0;
long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity()));
int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1;
if (read <= 0) {
codec.queueInputBuffer(inIndex, 0, 0, presentationUs,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
inBuf.clear();
inBuf.put(chunk, 0, read);
codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0);
remaining -= read;
presentationUs += read * 1_000_000L / bytesPerSecond;
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US);
if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
track = muxer.addTrack(codec.getOutputFormat());
muxer.start();
muxerStarted = true;
} else if (outIndex >= 0) {
if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
if (!muxerStarted) {
throw new IOException("encoder produced data before format");
}
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
outBuf.position(info.offset);
outBuf.limit(info.offset + info.size);
muxer.writeSampleData(track, outBuf, info);
}
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
if (input != null) {
try {
input.close();
} catch (IOException ignored) {
}
}
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
if (muxer != null) {
try {
muxer.stop();
} catch (Throwable ignored) {
}
muxer.release();
}
}
}
private static final class WavInfo {
int sampleRate;
int channels;
long dataOffset;
long dataBytes;
}
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
private static WavInfo readWavHeader(String path) throws IOException {
try (FileInputStream in = new FileInputStream(path)) {
byte[] head = new byte[12];
if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') {
throw new IOException("not a WAV file");
}
WavInfo info = new WavInfo();
long offset = 12;
byte[] chunkHead = new byte[8];
while (in.read(chunkHead) == 8) {
ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN);
int id = b.getInt();
long size = b.getInt() & 0xffffffffL;
offset += 8;
if (id == 0x20746d66) { // "fmt "
byte[] fmt = new byte[(int) Math.min(size, 16)];
if (in.read(fmt) != fmt.length) {
throw new IOException("truncated fmt chunk");
}
ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN);
int audioFormat = f.getShort();
info.channels = f.getShort();
info.sampleRate = f.getInt();
f.getInt();
f.getShort();
int bits = f.getShort();
if (audioFormat != 1 || bits != 16) {
throw new IOException("only 16-bit PCM WAV is supported");
}
long skip = size - fmt.length;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
} else if (id == 0x61746164) { // "data"
info.dataOffset = offset;
// A zero size means the header was never patched (crash
// mid-recording); recover the length from the file size.
info.dataBytes = size > 0 ? size : new File(path).length() - offset;
break;
} else {
long skip = size;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
}
}
if (info.sampleRate == 0 || info.dataOffset == 0) {
throw new IOException("missing fmt or data chunk");
}
return info;
}
}
}
@@ -60,6 +60,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawCompassListener> compassListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private volatile String lastFirmwareCapabilities = "";
private volatile Thread workerThread;
private volatile boolean running;
private volatile boolean userDisconnectRequested;
@@ -389,6 +392,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/**
* Whether the glasses mic is enabled right now. The enable lives in the
* current EvenHub session, so it dies with a transport drop, the charging
* case, or a suspend — silently, from the phone's point of view. Callers
* that track a capture across those events must check this rather than
* assume their earlier enable still holds.
*/
public boolean isAudioCaptureActive() {
synchronized (lock) {
return running && sessionReady && !shutdownRequested && audioCaptureActive;
}
}
/**
* Acquire/renew or release CFW's volatile wake-takeover lease on both
* arms. Delivery (not a protocol ACK) is awaited so a caller can ensure
@@ -743,6 +759,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
public String getFirmwareCapabilities() {
return lastFirmwareCapabilities;
}
public void addMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.add(listener);
}
}
public void removeMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.remove(listener);
}
}
private void emitMicStatus(byte[] body, String address) {
if (micStatusListeners.isEmpty()) {
return;
}
String arm = address.equalsIgnoreCase(leftAddress) ? "L"
: address.equalsIgnoreCase(rightAddress) ? "R" : "?";
byte[] copy = java.util.Arrays.copyOf(body, body.length);
mainHandler.post(() -> {
for (FaceclawMicStatusListener micListener : micStatusListeners) {
try {
micListener.onMicStatus(copy, arm);
} catch (Throwable t) {
Log.w(TAG, "mic status listener failed", t);
}
}
});
}
/**
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
* field-103 write to both temples, or to a single one. Fire-and-forget:
* the firmware answers with a field-104 status notify per temple, which
* arrives through addMicStatusListener.
*/
public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) {
if (record == null || record.length < 4) {
return;
}
synchronized (lock) {
if (!running || !sessionReady) {
logLine("skip mic control (" + label + "); session not ready");
return;
}
if (rightTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false));
}
if (leftTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true));
}
logLine("queue mic control " + label);
}
interruptibleSleep.interrupt();
}
/**
* Forward render-characteristic audio packets to the listener WITHOUT
* sending the stock EvenHub audio-control enable. Used for the CFW
* mic_control streaming path, where capture is armed through settings
* field 103 and the temples emit 'SM' frames on the same characteristic
* that stock mono LC3 uses. Returns false when no session is up.
*/
public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) {
if (listener == null) {
throw new IllegalArgumentException("listener is required");
}
synchronized (lock) {
if (!running || !sessionReady || shutdownRequested) {
logLine("skip G2 audio forwarding; session not ready");
return false;
}
audioPacketListener = listener;
audioCaptureActive = true;
logLine("G2 audio forwarding enabled");
}
return true;
}
public void stopG2AudioForwarding() {
synchronized (lock) {
audioPacketListener = null;
audioCaptureActive = false;
logLine("G2 audio forwarding disabled");
}
}
public void addCompassListener(FaceclawCompassListener listener) {
if (listener != null) {
compassListeners.add(listener);
@@ -777,12 +885,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
* foreground), including worker-app frames the TS side never sees.
*/
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) {
return getCompositePreviewBitmap(brightenGamma, false);
}
/** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) {
SurfaceCompositor.Composite composite = compositor.previewComposite();
if (composite == null) {
return null;
}
return PreviewBitmapUtil.fromGray(
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma);
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green);
}
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
@@ -1433,6 +1546,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
0
);
}
if (!faceclawWakeNotification
&& frame.ok
&& frame.sid == BleProtocol.SID_UI_SETTING) {
// CFW mic status (field 104) rides both standalone pushes and
// settings read acks, from each temple on its own link.
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
if (micStatus != null) {
emitMicStatus(micStatus, address);
}
}
if (!faceclawWakeNotification
&& decodedWearState < 0
&& frame.ok
@@ -2716,6 +2839,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
if (firmwareInfo != null) {
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
&& hasCapability(firmwareInfo.capabilities, "texstr14");
@@ -0,0 +1,446 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
import com.k2fsa.sherpa.onnx.OfflineStream;
import java.io.File;
import java.util.ArrayDeque;
import java.util.Arrays;
/**
* Continuous captioning over the decoded mic PCM: segments speech into
* utterances with an adaptive energy gate, transcribes each utterance with
* the on-device Moonshine model, and attaches a speaker voice-print per
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
* and any beam-direction gating), so this engine has no BLE dependencies.
*
* Utterance boundaries are measured on the sample clock, not the wall clock:
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
* audio heard. The TS side converts startMs/endMs to wall-clock times using
* the engine start timestamp.
*/
public class FaceclawCaptionEngine {
private static final String TAG = "FaceclawCaptions";
private static final int SAMPLE_RATE = 16000;
private static final int FEATURE_DIM = 80;
private static final int MAX_QUEUE_PACKETS = 200;
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
private static final double ONSET_FACTOR = 3.0;
private static final double RELEASE_FACTOR = 1.8;
private static final double MIN_RMS = 220.0;
private static final double NOISE_EMA_ALPHA = 0.05;
private static final int PRE_ROLL_MS = 400;
private static final int DEFAULT_SILENCE_MS = 800;
private static final int MIN_UTTERANCE_MS = 350;
private static final int MAX_UTTERANCE_MS = 15000;
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
// segments cut at the quietest window (mirrors FaceclawVoiceController).
private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2;
private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000;
private static final float NORMALIZE_TARGET_PEAK = 0.9f;
private static final float NORMALIZE_MAX_GAIN = 30f;
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private final Object queueLock = new Object();
private final ArrayDeque<byte[]> queue = new ArrayDeque<>();
private volatile FaceclawCaptionEngineListener listener;
private volatile String asrModelDir;
private volatile String speakerModelPath;
private volatile int silenceMs = DEFAULT_SILENCE_MS;
private Thread workerThread;
private volatile boolean started;
private OfflineRecognizer recognizer;
private FaceclawSpeakerId speakerId;
// Segmentation state (worker thread only).
private long totalSamples;
private double noiseFloor;
private boolean inUtterance;
private long utteranceStartSample;
private long silenceRunSamples;
private double utterancePeakRms;
private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)];
private int utteranceLength;
private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000];
private int preRollLength;
public void setListener(FaceclawCaptionEngineListener listener) {
this.listener = listener;
}
/** Directory holding the Moonshine model files, or null to disable ASR. */
public void setAsrModelDir(String dir) {
this.asrModelDir = dir;
}
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
public void setSpeakerModelPath(String path) {
this.speakerModelPath = path;
}
public void setSilenceMs(int ms) {
this.silenceMs = Math.max(200, Math.min(3000, ms));
}
public void start() {
synchronized (lock) {
if (started) {
return;
}
started = true;
workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine");
workerThread.start();
}
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
if (!started) {
return;
}
started = false;
threadToJoin = workerThread;
workerThread = null;
}
synchronized (queueLock) {
queueLock.notifyAll();
}
if (threadToJoin != null) {
threadToJoin.interrupt();
if (Thread.currentThread() != threadToJoin) {
try {
threadToJoin.join(2000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
public void acceptPcm(byte[] pcm16le) {
if (!started || pcm16le == null || pcm16le.length < 2) {
return;
}
synchronized (queueLock) {
if (queue.size() >= MAX_QUEUE_PACKETS) {
queue.removeFirst();
}
queue.addLast(pcm16le);
queueLock.notifyAll();
}
}
private void runLoop() {
try {
loadModels();
resetSegmentation();
while (started && !Thread.currentThread().isInterrupted()) {
byte[] chunk = takeChunk();
if (chunk == null) {
continue;
}
processChunk(chunk);
}
// Flush a trailing utterance so its text isn't lost on stop.
if (inUtterance && utteranceLength > 0) {
finalizeUtterance();
}
} catch (Throwable t) {
Log.e(TAG, "caption engine failed", t);
emitStatus("Captions failed: " + t.getMessage());
} finally {
releaseModels();
}
}
private void loadModels() {
String modelDir = asrModelDir;
if (modelDir != null && new File(modelDir, "tokens.txt").exists()) {
emitStatus("Loading caption model...");
recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(OfflineModelConfig.builder()
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setNumThreads(1)
.build())
.build());
} else {
recognizer = null;
}
String embedModel = speakerModelPath;
if (embedModel != null && new File(embedModel).exists()) {
speakerId = new FaceclawSpeakerId(embedModel);
speakerId.ensureLoaded();
} else {
speakerId = null;
}
emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)...");
}
private void releaseModels() {
if (recognizer != null) {
recognizer.release();
recognizer = null;
}
if (speakerId != null) {
speakerId.close();
speakerId = null;
}
}
private byte[] takeChunk() {
synchronized (queueLock) {
while (started && queue.isEmpty()) {
try {
queueLock.wait(250);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
}
}
return queue.pollFirst();
}
}
private void resetSegmentation() {
totalSamples = 0;
noiseFloor = 0;
inUtterance = false;
utteranceLength = 0;
preRollLength = 0;
silenceRunSamples = 0;
utterancePeakRms = 0;
}
private void processChunk(byte[] chunk) {
int count = chunk.length / 2;
short[] pcm = new short[count];
double sumSquares = 0;
for (int i = 0; i < count; i++) {
short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8));
pcm[i] = s;
sumSquares += (double) s * s;
}
double rms = Math.sqrt(sumSquares / Math.max(1, count));
totalSamples += count;
if (!inUtterance) {
// Track the noise floor only while idle so speech doesn't raise it.
noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA;
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms >= threshold * ONSET_FACTOR) {
inUtterance = true;
utteranceLength = 0;
silenceRunSamples = 0;
utterancePeakRms = rms;
appendUtterance(preRoll, preRollLength);
utteranceStartSample = Math.max(0, totalSamples - count - preRollLength);
appendUtterance(pcm, count);
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE);
} else {
appendPreRoll(pcm, count);
}
return;
}
appendUtterance(pcm, count);
utterancePeakRms = Math.max(utterancePeakRms, rms);
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms < threshold * RELEASE_FACTOR) {
silenceRunSamples += count;
} else {
silenceRunSamples = 0;
}
long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE;
boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs;
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
finalizeUtterance();
inUtterance = false;
preRollLength = 0;
}
}
private void appendPreRoll(short[] pcm, int count) {
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
if (count >= preRoll.length) {
System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length);
preRollLength = preRoll.length;
return;
}
int keep = Math.min(preRollLength, preRoll.length - count);
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep);
System.arraycopy(pcm, 0, preRoll, keep, count);
preRollLength = keep + count;
}
private void appendUtterance(short[] pcm, int count) {
int room = utterance.length - utteranceLength;
int copied = Math.min(room, count);
if (copied > 0) {
System.arraycopy(pcm, 0, utterance, utteranceLength, copied);
utteranceLength += copied;
}
}
private void finalizeUtterance() {
int length = utteranceLength;
long startMs = utteranceStartSample * 1000 / SAMPLE_RATE;
long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE;
if (endMs - startMs < MIN_UTTERANCE_MS) {
return;
}
String text = recognizeUtterance(length);
float[] embedding = embedUtterance(length);
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms);
}
private String recognizeUtterance(int length) {
if (recognizer == null || length <= 0) {
return "";
}
StringBuilder joined = new StringBuilder();
int offset = 0;
while (offset < length) {
int remaining = length - offset;
int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES);
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
segment = findQuietCut(offset, segment);
}
String part = recognizeRange(offset, segment);
if (part.length() > 0) {
if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) {
joined.append(' ');
}
joined.append(part);
}
offset += segment;
}
return joined.toString().trim();
}
/**
* End a decode segment at the center of the quietest window near its end
* so the cut lands between words (same approach as the PTT controller).
*/
private int findQuietCut(int offset, int segment) {
int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES);
int win = CUT_WINDOW_SAMPLES;
if (segment - searchStart <= win) {
return segment;
}
double sum = 0;
for (int i = searchStart; i < searchStart + win; i++) {
double s = utterance[offset + i];
sum += s * s;
}
double best = sum;
int bestStart = searchStart;
for (int start = searchStart + 1; start + win <= segment; start++) {
double dropped = utterance[offset + start - 1];
double added = utterance[offset + start + win - 1];
sum += added * added - dropped * dropped;
if (sum < best) {
best = sum;
bestStart = start;
}
}
return bestStart + win / 2;
}
private String recognizeRange(int offset, int count) {
float[] samples = new float[count];
float peak = 0f;
for (int i = 0; i < count; i++) {
float v = utterance[offset + i] / 32768.0f;
samples[i] = v;
float a = Math.abs(v);
if (a > peak) {
peak = a;
}
}
if (peak > 0f) {
float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN);
if (gain > 1f) {
for (int i = 0; i < count; i++) {
samples[i] *= gain;
}
}
}
OfflineStream stream = recognizer.createStream();
try {
stream.acceptWaveform(samples, SAMPLE_RATE);
recognizer.decode(stream);
OfflineRecognizerResult result = recognizer.getResult(stream);
String raw = result == null ? "" : result.getText();
return raw == null ? "" : raw.trim();
} finally {
stream.release();
}
}
private float[] embedUtterance(int length) {
FaceclawSpeakerId currentSpeakerId = speakerId;
if (currentSpeakerId == null || length <= 0) {
return null;
}
int count = Math.min(length, EMBED_MAX_SAMPLES);
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
short s = utterance[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
return currentSpeakerId.embed(le, SAMPLE_RATE);
}
private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length);
mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms));
}
private void emitSpeechStart(long startMs) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeechStart(startMs));
}
private void emitStatus(String status) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onStatus(status));
}
}
@@ -0,0 +1,17 @@
package com.faceclaw.app;
public interface FaceclawCaptionEngineListener {
/**
* One finished utterance. text may be empty when no ASR model is loaded
* or recognition produced nothing; embedding is the L2-normalized
* voice-print or null when the speaker model is unavailable or the
* utterance was too short. startMs/endMs are on the engine's sample
* clock (milliseconds of audio since start()).
*/
void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms);
/** A speech onset was detected; the utterance is now being collected. */
void onSpeechStart(long startMs);
void onStatus(String status);
}
@@ -0,0 +1,778 @@
package com.faceclaw.app;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import android.util.Base64;
import android.util.Log;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
/**
* Persistent store for caption sessions, transcript segments, and speaker
* voice profiles, backing the Microphones app and its phone-side review UI.
* All complex values cross the JS bridge as JSON strings; voice-print
* embeddings are float32 LE blobs, exposed as base64.
*/
public class FaceclawConversationStore extends SQLiteOpenHelper {
private static final String TAG = "FaceclawConvStore";
private static final String DB_NAME = "faceclaw-conversations.db";
private static final int DB_VERSION = 2;
private static FaceclawConversationStore instance;
public static synchronized FaceclawConversationStore getInstance(Context context) {
if (instance == null) {
instance = new FaceclawConversationStore(context.getApplicationContext());
}
return instance;
}
private FaceclawConversationStore(Context context) {
super(context, DB_NAME, null, DB_VERSION);
}
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL("CREATE TABLE speakers ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "name TEXT NOT NULL,"
+ "color TEXT NOT NULL DEFAULT '#4FC3F7',"
+ "is_wearer INTEGER NOT NULL DEFAULT 0,"
+ "embedding BLOB,"
+ "embedding_count INTEGER NOT NULL DEFAULT 0,"
+ "created_at INTEGER NOT NULL,"
+ "last_heard_at INTEGER,"
+ "tag TEXT,"
+ "last_recap TEXT,"
+ "action_items TEXT,"
+ "facts TEXT,"
+ "insights_updated_at INTEGER,"
+ "insights_session_id INTEGER)");
db.execSQL("CREATE TABLE sessions ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER,"
+ "title TEXT,"
+ "audio_path TEXT,"
+ "audio_codec TEXT,"
+ "avg_sentiment REAL,"
+ "segment_count INTEGER NOT NULL DEFAULT 0)");
db.execSQL("CREATE TABLE segments ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "session_id INTEGER NOT NULL,"
+ "speaker_id INTEGER,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER NOT NULL,"
+ "audio_offset_ms INTEGER,"
+ "text TEXT NOT NULL,"
+ "lang TEXT,"
+ "translation TEXT,"
+ "translation_lang TEXT,"
+ "sentiment REAL,"
+ "emotion TEXT,"
+ "search_meta TEXT,"
+ "angle INTEGER,"
+ "embedding BLOB)");
db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)");
db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)");
db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)");
db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)");
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
if (oldVersion < 2) {
// v2: per-speaker conversation insights (recap of the most recent
// conversation, its action items, and accumulated inferred facts).
db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER");
}
}
// ---- sessions ----
public long startSession(long startedAtMs, String title) {
ContentValues values = new ContentValues();
values.put("started_at", startedAtMs);
values.put("title", title == null || title.isEmpty() ? null : title);
return getWritableDatabase().insert("sessions", null, values);
}
public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) {
ContentValues values = new ContentValues();
values.put("ended_at", endedAtMs);
if (audioPath != null && !audioPath.isEmpty()) {
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
}
values.put("avg_sentiment", avgSentiment);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public void setSessionAudio(long sessionId, String audioPath, String audioCodec) {
ContentValues values = new ContentValues();
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public String getSessionAudioPath(long sessionId) {
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) {
if (c.moveToFirst()) {
return c.isNull(0) ? "" : c.getString(0);
}
}
return "";
}
/**
* Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs,
* speakerId, emotion (matches any segment emotion in the session), query
* (LIKE over title and segment text/search metadata), limit.
*/
public String querySessions(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec,"
+ " s.avg_sentiment, s.segment_count FROM sessions s");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query");
if (joinSegments) {
sql.append(" LEFT JOIN segments g ON g.session_id = s.id");
}
if (filter.has("sinceMs")) {
where.add("s.started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("s.started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (filter.has("speakerId")) {
where.add("g.speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("emotion")) {
where.add("g.emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("query")) {
String like = "%" + filter.getString("query") + "%";
where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY s.started_at DESC");
int limit = filter.optInt("limit", 200);
sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
long id = c.getLong(0);
row.put("id", id);
row.put("startedAt", c.getLong(1));
row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2));
row.put("title", c.isNull(3) ? "" : c.getString(3));
row.put("audioPath", c.isNull(4) ? "" : c.getString(4));
row.put("audioCodec", c.isNull(5) ? "" : c.getString(5));
row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6));
row.put("segmentCount", c.getLong(7));
row.put("speakers", sessionSpeakersJson(id));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "querySessions failed", e);
return "[]";
}
}
private JSONArray sessionSpeakersJson(long sessionId) throws JSONException {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT DISTINCT p.id, p.name, p.color FROM segments g"
+ " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
out.put(row);
}
}
return out;
}
// ---- segments ----
/**
* Insert one transcript segment. json fields: sessionId, speakerId?,
* startedAt, endedAt, audioOffsetMs?, text, lang?, translation?,
* translationLang?, sentiment?, emotion?, searchMeta?, angle?,
* embeddingBase64?. Returns the new row id, or -1.
*/
public long insertSegment(String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
long sessionId = s.getLong("sessionId");
values.put("session_id", sessionId);
if (s.has("speakerId")) {
values.put("speaker_id", s.getLong("speakerId"));
}
values.put("started_at", s.getLong("startedAt"));
values.put("ended_at", s.getLong("endedAt"));
if (s.has("audioOffsetMs")) {
values.put("audio_offset_ms", s.getLong("audioOffsetMs"));
}
values.put("text", s.getString("text"));
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (s.has("angle")) {
values.put("angle", s.getInt("angle"));
}
if (s.has("embeddingBase64")) {
values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP));
}
SQLiteDatabase db = getWritableDatabase();
long id = db.insert("segments", null, values);
db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?",
new Object[]{sessionId});
return id;
} catch (Throwable t) {
Log.w(TAG, "insertSegment failed", t);
return -1;
}
}
private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column)
throws JSONException {
if (s.has(jsonKey) && !s.isNull(jsonKey)) {
values.put(column, s.getString(jsonKey));
}
}
/** Update mutable fields of a segment (same keys as insertSegment). */
public void updateSegment(long segmentId, String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
if (s.has("speakerId")) {
if (s.isNull("speakerId")) {
values.putNull("speaker_id");
} else {
values.put("speaker_id", s.getLong("speakerId"));
}
}
if (s.has("text")) {
values.put("text", s.getString("text"));
}
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (values.size() > 0) {
getWritableDatabase().update("segments", values, "id=?",
new String[]{String.valueOf(segmentId)});
}
} catch (Throwable t) {
Log.w(TAG, "updateSegment failed", t);
}
}
public String querySegments(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle"
+ " FROM segments WHERE session_id=? ORDER BY started_at ASC",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
out.put(segmentRowJson(c));
}
} catch (JSONException e) {
Log.w(TAG, "querySegments failed", e);
}
return out.toString();
}
private JSONObject segmentRowJson(Cursor c) throws JSONException {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1));
row.put("startedAt", c.getLong(2));
row.put("endedAt", c.getLong(3));
row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4));
row.put("text", c.getString(5));
row.put("lang", c.isNull(6) ? "" : c.getString(6));
row.put("translation", c.isNull(7) ? "" : c.getString(7));
row.put("translationLang", c.isNull(8) ? "" : c.getString(8));
row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9));
row.put("emotion", c.isNull(10) ? "" : c.getString(10));
row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11));
return row;
}
/**
* Search segments across sessions. filterJson: query?, emotion?,
* speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and
* the sentiment search metadata.
*/
public String searchSegments(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle, session_id"
+ " FROM segments");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
if (filter.has("query") && !filter.getString("query").isEmpty()) {
String like = "%" + filter.getString("query") + "%";
where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (filter.has("emotion")) {
where.add("emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("speakerId")) {
where.add("speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("sinceMs")) {
where.add("started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY started_at DESC LIMIT ")
.append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = segmentRowJson(c);
row.put("sessionId", c.getLong(12));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "searchSegments failed", e);
return "[]";
}
}
/** Per-segment voice-print embeddings of a session (for re-diarization). */
public String querySegmentEmbeddings(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySegmentEmbeddings failed", e);
}
return out.toString();
}
// ---- speakers ----
public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) {
ContentValues values = new ContentValues();
values.put("name", name);
values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color);
values.put("is_wearer", isWearer ? 1 : 0);
values.put("created_at", System.currentTimeMillis());
values.put("last_heard_at", System.currentTimeMillis());
if (embeddingBase64 != null && !embeddingBase64.isEmpty()) {
values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP));
values.put("embedding_count", 1);
}
return getWritableDatabase().insert("speakers", null, values);
}
public String querySpeakers() {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count,"
+ " p.last_heard_at, p.tag,"
+ " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id,"
+ " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments"
+ " FROM speakers p ORDER BY p.last_heard_at DESC", null)) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
row.put("isWearer", c.getInt(3) != 0);
byte[] blob = c.isNull(4) ? null : c.getBlob(4);
row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP));
row.put("embeddingCount", c.getInt(5));
row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6));
row.put("tag", c.isNull(7) ? "" : c.getString(7));
row.put("lastRecap", c.isNull(8) ? "" : c.getString(8));
row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9));
row.put("factsJson", c.isNull(10) ? "" : c.getString(10));
row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11));
row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12));
row.put("segmentCount", c.getLong(13));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySpeakers failed", e);
}
return out.toString();
}
public void renameSpeaker(long speakerId, String name) {
ContentValues values = new ContentValues();
values.put("name", name);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerColor(long speakerId, String color) {
ContentValues values = new ContentValues();
values.put("color", color);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerTag(long speakerId, String tag) {
ContentValues values = new ContentValues();
values.put("tag", tag);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Store LLM-derived conversation insights for a speaker: a recap of their
* most recent conversation, its open action items (JSON string array), and
* the accumulated inferred facts about them (JSON string array).
*/
public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson,
String factsJson, long sessionId) {
ContentValues values = new ContentValues();
values.put("last_recap", recap == null ? "" : recap);
values.put("action_items", actionItemsJson == null ? "" : actionItemsJson);
values.put("facts", factsJson == null ? "" : factsJson);
values.put("insights_updated_at", System.currentTimeMillis());
values.put("insights_session_id", sessionId);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerWearer(long speakerId, boolean isWearer) {
SQLiteDatabase db = getWritableDatabase();
if (isWearer) {
// Exactly one wearer profile at a time.
db.execSQL("UPDATE speakers SET is_wearer = 0");
}
ContentValues values = new ContentValues();
values.put("is_wearer", isWearer ? 1 : 0);
db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Running-mean centroid update for a speaker's voice-print, capped so a
* long session cannot drown the enrolled voice.
*/
public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) {
try {
byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP);
float[] update = blobToFloats(incoming);
if (update == null) {
return;
}
float[] current = null;
int count = 0;
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT embedding, embedding_count FROM speakers WHERE id=?",
new String[]{String.valueOf(speakerId)})) {
if (c.moveToFirst()) {
current = c.isNull(0) ? null : blobToFloats(c.getBlob(0));
count = c.getInt(1);
}
}
float[] merged;
if (current == null || current.length != update.length) {
merged = update;
count = 1;
} else {
int effective = Math.min(count, Math.max(1, maxCount));
merged = new float[current.length];
for (int i = 0; i < merged.length; i++) {
merged[i] = (current[i] * effective + update[i]) / (effective + 1);
}
l2NormalizeInPlace(merged);
count = count + 1;
}
ContentValues values = new ContentValues();
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", count);
values.put("last_heard_at", System.currentTimeMillis());
getWritableDatabase().update("speakers", values, "id=?",
new String[]{String.valueOf(speakerId)});
} catch (Throwable t) {
Log.w(TAG, "updateSpeakerEmbedding failed", t);
}
}
/** Move every segment from one speaker onto another and drop the source. */
public void mergeSpeakers(long fromId, long intoId) {
if (fromId == intoId) {
return;
}
SQLiteDatabase db = getWritableDatabase();
db.beginTransaction();
try {
db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?",
new Object[]{intoId, fromId});
// Blend the two centroids weighted by their sample counts.
float[] from = null;
float[] into = null;
int fromCount = 0;
int intoCount = 0;
boolean fromWearer = false;
try (Cursor c = db.rawQuery(
"SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)",
new String[]{String.valueOf(fromId), String.valueOf(intoId)})) {
while (c.moveToNext()) {
long id = c.getLong(0);
float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1));
int count = c.getInt(2);
if (id == fromId) {
from = embedding;
fromCount = count;
fromWearer = c.getInt(3) != 0;
} else {
into = embedding;
intoCount = count;
}
}
}
float[] merged = into;
if (from != null && into != null && from.length == into.length) {
merged = new float[into.length];
int total = Math.max(1, fromCount + intoCount);
for (int i = 0; i < merged.length; i++) {
merged[i] = (into[i] * intoCount + from[i] * fromCount) / total;
}
l2NormalizeInPlace(merged);
} else if (into == null) {
merged = from;
}
ContentValues values = new ContentValues();
if (merged != null) {
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", fromCount + intoCount);
}
if (fromWearer) {
values.put("is_wearer", 1);
}
values.put("last_heard_at", System.currentTimeMillis());
// Keep the target's conversation insights; adopt the source's only
// where the target has none (e.g. merging a rich profile into a
// freshly auto-created one).
try (Cursor c = db.rawQuery(
"SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id"
+ " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END",
new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) {
boolean targetHasInsights = false;
while (c.moveToNext()) {
boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty();
if (c.getPosition() == 0) {
targetHasInsights = hasInsights;
} else if (!targetHasInsights && hasInsights) {
values.put("last_recap", c.getString(0));
values.put("action_items", c.isNull(1) ? "" : c.getString(1));
values.put("facts", c.isNull(2) ? "" : c.getString(2));
values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3));
values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4));
}
}
}
db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)});
db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)});
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
}
public void reassignSegmentSpeaker(long segmentId, long speakerId) {
ContentValues values = new ContentValues();
values.put("speaker_id", speakerId);
getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)});
}
/** Reassign every segment of a speaker within one session (split support). */
public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) {
getWritableDatabase().execSQL(
"UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?",
new Object[]{toSpeakerId, sessionId, fromSpeakerId});
}
public void deleteSpeaker(long speakerId) {
SQLiteDatabase db = getWritableDatabase();
db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId});
db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)});
}
// ---- retention ----
/**
* Delete caption sessions older than the cutoff (0 disables), and strip
* recordings older than their own cutoff while keeping the transcript.
* Returns a JSON summary of what was removed.
*/
public String applyRetention(long captionCutoffMs, long recordingCutoffMs) {
SQLiteDatabase db = getWritableDatabase();
int sessionsDeleted = 0;
int recordingsDeleted = 0;
if (recordingCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(recordingCutoffMs)})) {
while (c.moveToNext()) {
if (!c.isNull(1)) {
paths.add(c.getString(1));
}
}
}
for (String path : paths) {
if (new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?",
new Object[]{recordingCutoffMs});
}
if (captionCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(captionCutoffMs)})) {
while (c.moveToNext()) {
paths.add(c.getString(0));
}
}
for (String path : paths) {
if (path != null && new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)",
new Object[]{captionCutoffMs});
Cursor count = db.rawQuery("SELECT changes()", null);
count.close();
db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs});
try (Cursor c = db.rawQuery("SELECT changes()", null)) {
if (c.moveToFirst()) {
sessionsDeleted = c.getInt(0);
}
}
}
try {
JSONObject summary = new JSONObject();
summary.put("sessionsDeleted", sessionsDeleted);
summary.put("recordingsDeleted", recordingsDeleted);
return summary.toString();
} catch (JSONException e) {
return "{}";
}
}
public void deleteSession(long sessionId) {
SQLiteDatabase db = getWritableDatabase();
String path = getSessionAudioPath(sessionId);
if (!path.isEmpty()) {
new File(path).delete();
}
db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)});
db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)});
}
// ---- blob helpers ----
private static float[] blobToFloats(byte[] blob) {
if (blob == null || blob.length < 4 || blob.length % 4 != 0) {
return null;
}
ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
float[] out = new float[blob.length / 4];
for (int i = 0; i < out.length; i++) {
out[i] = buffer.getFloat();
}
return out;
}
private static byte[] floatsToBlob(float[] values) {
ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN);
for (float v : values) {
buffer.putFloat(v);
}
return buffer.array();
}
private static void l2NormalizeInPlace(float[] v) {
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
}
}
@@ -37,9 +37,6 @@ import java.util.Set;
* matching are decided in TypeScript (app/g2/even-advertisement.ts,
* app/g2/pairing-candidates.ts) where they can be unit-tested; this filter
* must stay a superset of the TS classifier's admission.
*
* Ported from the discovery path of SybilSight's Bluetooth SDK (G2.didDiscover
* and R1's advertisement snapshot), reduced to what Android exposes.
*/
@SuppressLint("MissingPermission")
public class FaceclawDeviceDiscovery {
@@ -296,7 +293,7 @@ public class FaceclawDeviceDiscovery {
/**
* Android strips the 2-byte company identifier from each manufacturer
* record and keys the SparseArray by it. Put the bytes back (little-endian)
* so TypeScript sees the same "ER"+… layout the iOS SDK and HCI captures
* so TypeScript sees the same "ER"+… layout the HCI captures
* document. Prefers Even's own id; otherwise returns the first record so an
* unexpected company id still reaches the diagnostics log.
*/
@@ -0,0 +1,262 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FastClusteringConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* Offline re-diarization of a stored conversation recording (16-bit mono
* WAV): pyannote segmentation plus the shared speaker-embedding model with
* clustering, via sherpa-onnx. Runs on its own thread; results are handed to
* TS as JSON turns for reconciliation against the transcript segments.
*/
public class FaceclawDiarizer {
private static final String TAG = "FaceclawDiarizer";
public interface Listener {
/** progress in [0, 1]. */
void onProgress(double progress);
/** turns: [{startMs, endMs, cluster}], sorted by start time. */
void onDone(String turnsJson);
void onError(String message);
}
private final String segmentationModelPath;
private final String embeddingModelPath;
private final String wavPath;
private final int numClusters;
private final float threshold;
private final Listener listener;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private volatile boolean cancelled;
public FaceclawDiarizer(
String segmentationModelPath,
String embeddingModelPath,
String wavPath,
int numClusters,
float threshold,
Listener listener) {
this.segmentationModelPath = segmentationModelPath;
this.embeddingModelPath = embeddingModelPath;
this.wavPath = wavPath;
this.numClusters = numClusters;
this.threshold = threshold <= 0 ? 0.5f : threshold;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawDiarizer");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
public void cancel() {
cancelled = true;
}
private void run() {
OfflineSpeakerDiarization diarization = null;
try {
float[] samples = wavPath.endsWith(".wav")
? readWavMono16(wavPath)
: decodeCompressedMono(wavPath);
OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder()
.setSegmentation(OfflineSpeakerSegmentationModelConfig.builder()
.setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder()
.setModel(segmentationModelPath)
.build())
.setNumThreads(1)
.build())
.setEmbedding(SpeakerEmbeddingExtractorConfig.builder()
.setModel(embeddingModelPath)
.setNumThreads(1)
.build())
.setClustering(FastClusteringConfig.builder()
.setNumClusters(numClusters > 0 ? numClusters : -1)
.setThreshold(threshold)
.build())
.setMinDurationOn(0.3f)
.setMinDurationOff(0.5f)
.build();
diarization = new OfflineSpeakerDiarization(config);
OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback(
samples,
(numProcessedChunks, numTotalChunks, arg) -> {
emitProgress(numTotalChunks <= 0 ? 0
: (double) numProcessedChunks / numTotalChunks);
return cancelled ? 1 : 0;
});
JSONArray turns = new JSONArray();
if (segments != null) {
for (OfflineSpeakerDiarizationSegment segment : segments) {
JSONObject turn = new JSONObject();
turn.put("startMs", Math.round(segment.getStart() * 1000));
turn.put("endMs", Math.round(segment.getEnd() * 1000));
turn.put("cluster", segment.getSpeaker());
turns.put(turn);
}
}
String result = turns.toString();
mainHandler.post(() -> listener.onDone(result));
} catch (Throwable t) {
Log.w(TAG, "diarization failed for " + wavPath, t);
String message = t.getMessage() == null ? t.toString() : t.getMessage();
mainHandler.post(() -> listener.onError(message));
} finally {
if (diarization != null) {
diarization.release();
}
}
}
private void emitProgress(double progress) {
mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress))));
}
/**
* Decode a compressed recording (M4A/AAC from the transcode sweep) into
* [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written
* as 16 kHz mono, so no resampling is needed.
*/
private static float[] decodeCompressedMono(String path) throws IOException {
android.media.MediaExtractor extractor = new android.media.MediaExtractor();
android.media.MediaCodec codec = null;
java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream();
try {
extractor.setDataSource(path);
int trackIndex = -1;
android.media.MediaFormat format = null;
for (int i = 0; i < extractor.getTrackCount(); i++) {
android.media.MediaFormat candidate = extractor.getTrackFormat(i);
String mime = candidate.getString(android.media.MediaFormat.KEY_MIME);
if (mime != null && mime.startsWith("audio/")) {
trackIndex = i;
format = candidate;
break;
}
}
if (trackIndex < 0 || format == null) {
throw new IOException("no audio track in " + path);
}
extractor.selectTrack(trackIndex);
codec = android.media.MediaCodec.createDecoderByType(
format.getString(android.media.MediaFormat.KEY_MIME));
codec.configure(format, null, null, 0);
codec.start();
android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo();
boolean inputDone = false;
boolean outputDone = false;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(10_000);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int size = extractor.readSampleData(inBuf, 0);
if (size < 0) {
codec.queueInputBuffer(inIndex, 0, 0, 0,
android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0);
extractor.advance();
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, 10_000);
if (outIndex >= 0) {
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
byte[] chunk = new byte[info.size];
outBuf.position(info.offset);
outBuf.get(chunk);
pcm.write(chunk);
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
extractor.release();
if (codec != null) {
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
}
}
byte[] bytes = pcm.toByteArray();
int count = bytes.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f;
}
return samples;
}
/** Load a 16-bit mono PCM WAV into [-1, 1] floats. */
private static float[] readWavMono16(String path) throws IOException {
File file = new File(path);
long fileLength = file.length();
try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
byte[] header = new byte[44];
in.readFully(header);
if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') {
throw new IOException("not a WAV file");
}
int channels = (header[22] & 0xff) | (header[23] << 8);
int bits = (header[34] & 0xff) | (header[35] << 8);
if (channels != 1 || bits != 16) {
throw new IOException("only 16-bit mono WAV is supported");
}
long dataBytes = fileLength - 44;
int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE);
float[] samples = new float[count];
byte[] chunk = new byte[65536];
int sampleIndex = 0;
int carried = 0;
byte carriedByte = 0;
int read;
while (sampleIndex < count && (read = in.read(chunk)) > 0) {
int offset = 0;
if (carried == 1) {
samples[sampleIndex++] =
((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f;
offset = 1;
carried = 0;
}
int pairs = (read - offset) / 2;
for (int i = 0; i < pairs && sampleIndex < count; i++) {
int base = offset + i * 2;
samples[sampleIndex++] =
((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f;
}
if (((read - offset) & 1) == 1) {
carried = 1;
carriedByte = chunk[read - 1];
}
}
return samples;
}
}
}
@@ -26,6 +26,10 @@ import java.util.concurrent.TimeUnit;
* protobuf helpers (BleProtocol), but owns its own GATT connection and runs its
* whole lifecycle on a single worker thread. It expects the main app to be
* disconnected while it runs (onboarding, before flashing).
*
* Only the right arm is used: the lenses relay messages to each other, so
* anything sent to the right arm reaches both, and acks/events only ever come
* from the right arm anyway.
*/
public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private static final String TAG = "FaceclawFlashPrompt";
@@ -43,7 +47,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private final Context context;
private final String rightAddress;
private final String leftAddress;
private final String warningText;
private final FaceclawBleManager bleManager;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
@@ -58,7 +61,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private volatile boolean cancelled = false;
private volatile boolean finished = false;
private volatile boolean rightConnected = false;
private volatile boolean leftConnected = false;
private volatile boolean rightLost = false;
private volatile Boolean approved = null;
private volatile ScheduledExecutorService heartbeatExecutor;
@@ -66,10 +68,9 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private int nextMagic = 100;
private int nextSeq = 0x40;
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String leftAddress, String warningText) {
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String warningText) {
this.context = context.getApplicationContext();
this.rightAddress = rightAddress == null ? "" : rightAddress;
this.leftAddress = leftAddress == null ? "" : leftAddress;
this.warningText = warningText == null ? "" : warningText;
this.bleManager = new FaceclawBleManager(this.context);
this.bleManager.setListener(this);
@@ -116,18 +117,8 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
}
emitState("connecting", "");
connectArm(rightAddress, "right");
connectArm(rightAddress);
rightConnected = true;
boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress);
if (haveLeft) {
try {
connectArm(leftAddress, "left");
leftConnected = true;
} catch (Exception e) {
emitLog("left arm connect failed (continuing on right only): " + e.getMessage());
}
}
if (cancelled) {
teardown();
return;
@@ -135,14 +126,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
emitState("connected", "");
bringUpArm(rightAddress);
if (leftConnected) {
try {
bringUpArm(leftAddress);
} catch (Exception e) {
emitLog("left arm prompt failed (continuing on right only): " + e.getMessage());
leftConnected = false;
}
}
startHeartbeat();
emitState("prompting", "");
@@ -158,7 +141,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
if (approved != null) {
boolean result = Boolean.TRUE.equals(approved);
try {
sendShutdownBoth();
sendShutdown();
} catch (Exception ignored) {
}
emitResult(result);
@@ -182,7 +165,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
}
}
private void connectArm(String address, String labelForLog) {
private void connectArm(String address) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw new IllegalStateException("connect failed: " + address);
}
@@ -194,7 +177,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
throw new IllegalStateException("enableNotifications failed: " + address);
}
emitLog("connected " + labelForLog + " arm");
emitLog("connected right arm");
}
private void bringUpArm(String address) throws InterruptedException {
@@ -301,9 +284,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
}
if (leftConnected) {
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
}
} catch (Exception e) {
Log.w(TAG, "heartbeat failed: " + e.getMessage());
}
@@ -317,14 +297,11 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
}
}
private void sendShutdownBoth() {
private void sendShutdown() {
byte[] shutdown = BleProtocol.buildShutdown(allocMagic(), 0);
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
}
if (leftConnected) {
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
}
}
private void teardown() {
@@ -335,7 +312,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
} catch (Exception ignored) {
}
rightConnected = false;
leftConnected = false;
}
@Override
@@ -355,10 +331,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
return;
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
// Async events (taps/selections) only ever come from the right arm.
if (address.equalsIgnoreCase(rightAddress)) {
handleEvent(frame);
}
handleEvent(frame);
return;
}
if (frame.msgSeq >= 0) {
@@ -407,8 +380,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
selectionLatch.countDown();
}
}
} else if (address.equalsIgnoreCase(leftAddress)) {
leftConnected = false;
}
}
@@ -12,6 +12,12 @@ public class FaceclawLc3Decoder {
public static final int COUNTER_OFFSET = 204;
public static final int SAMPLES_PER_FRAME = 160;
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
// glasses DSP appends two signed 16-bit LE values computed on the raw
// stereo capture before the mono downmix — a signal-strength ratio and the
// asin-derived direction-of-arrival in degrees — then the packet counter.
public static final int SSR_OFFSET = 200;
public static final int ANGLE_OFFSET = 202;
static {
System.loadLibrary("faceclaw_lc3");
@@ -23,6 +29,8 @@ public class FaceclawLc3Decoder {
private long duplicatePackets;
private long missingPackets;
private long decodeErrors;
private int lastSsr;
private int lastAngleDegrees;
public FaceclawLc3Decoder() {
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
@@ -64,9 +72,21 @@ public class FaceclawLc3Decoder {
}
realPackets++;
lastCounter = counter;
lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8));
lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8));
return decoded;
}
/** Firmware signal-strength ratio from the most recent decoded packet. */
public synchronized int getLastSsr() {
return lastSsr;
}
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
public synchronized int getLastAngleDegrees() {
return lastAngleDegrees;
}
public synchronized long getRealPackets() {
return realPackets;
}
@@ -0,0 +1,10 @@
package com.faceclaw.app;
public interface FaceclawMicStatusListener {
/**
* A CFW mic_control field-104 status record from one temple. body is the
* raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is
* "L" or "R" for the temple's own BLE link, "?" if unmatched.
*/
void onMicStatus(byte[] body, String arm);
}
@@ -0,0 +1,266 @@
package com.faceclaw.app;
import android.media.audiofx.AcousticEchoCanceler;
import android.media.audiofx.AudioEffect;
import android.media.audiofx.AutomaticGainControl;
import android.media.audiofx.NoiseSuppressor;
import java.util.ArrayList;
import java.util.Arrays;
/**
* Phone-side noise suppression for the microphone stream.
*
* Android's built-in noise cancellation (the {@link NoiseSuppressor} capture
* effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio
* arrives as raw PCM over BLE from the glasses, which that effect chain never
* sees, so this class picks the best engine that can actually reach the
* samples:
*
* - Audio captured by the phone itself: {@link #attachPlatformEffects(int)}
* enables the device's built-in NS/AEC/AGC chain (hardware or OEM software,
* whatever the Android build ships) on the AudioRecord session.
* - Audio from the glasses (the normal case): {@link #process(byte[])} runs a
* WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann
* frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an
* over-subtracted Wiener gain with a residual floor so speech never pumps
* in and out of silence.
*/
public final class FaceclawNoiseSuppressor {
private static final int FFT_SIZE = 256;
private static final int HOP = FFT_SIZE / 2;
private static final float OVERSUBTRACTION = 1.6f;
/** About -16 dB of residual: keeps the noise bed natural instead of gated. */
private static final float GAIN_FLOOR = 0.15f;
private static final float GAIN_SMOOTHING = 0.6f;
/** Frames of pure noise-floor learning after start/reset (~100 ms). */
private static final int NOISE_WARMUP_FRAMES = 12;
private final int sampleRate;
private final float[] window = new float[FFT_SIZE];
private final float[] history = new float[FFT_SIZE];
private final float[] overlap = new float[HOP];
private final float[] noise = new float[FFT_SIZE / 2 + 1];
private final float[] gain = new float[FFT_SIZE / 2 + 1];
private final float[] re = new float[FFT_SIZE];
private final float[] im = new float[FFT_SIZE];
private final short[] pending = new short[HOP];
private int pendingCount = 0;
private int warmupFrames = 0;
public FaceclawNoiseSuppressor(int sampleRate) {
this.sampleRate = sampleRate;
// sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap.
for (int i = 0; i < FFT_SIZE; i++) {
window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE);
}
reset();
}
public int getSampleRate() {
return sampleRate;
}
/** Forget the learned noise floor (e.g. after the suppressor was toggled off). */
public void reset() {
Arrays.fill(history, 0f);
Arrays.fill(overlap, 0f);
Arrays.fill(noise, 0f);
Arrays.fill(gain, 1f);
pendingCount = 0;
warmupFrames = 0;
}
/**
* Suppress noise in 16-bit little-endian mono PCM. Streaming: input not
* filling a whole 8 ms hop is carried into the next call, so the output
* may be up to one hop shorter than the input (and longer on a later
* call); total latency is one hop.
*/
public byte[] process(byte[] pcm16le) {
if (pcm16le == null || pcm16le.length < 2) {
return pcm16le;
}
int sampleCount = pcm16le.length / 2;
int hops = (pendingCount + sampleCount) / HOP;
byte[] out = new byte[hops * HOP * 2];
int outPos = 0;
int inPos = 0;
while (inPos < sampleCount) {
int take = Math.min(HOP - pendingCount, sampleCount - inPos);
for (int i = 0; i < take; i++) {
int lo = pcm16le[(inPos + i) * 2] & 0xff;
int hi = pcm16le[(inPos + i) * 2 + 1];
pending[pendingCount + i] = (short) ((hi << 8) | lo);
}
pendingCount += take;
inPos += take;
if (pendingCount == HOP) {
outPos = processHop(out, outPos);
pendingCount = 0;
}
}
return out;
}
private int processHop(byte[] out, int outPos) {
System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP);
for (int i = 0; i < HOP; i++) {
history[FFT_SIZE - HOP + i] = pending[i] / 32768f;
}
for (int i = 0; i < FFT_SIZE; i++) {
re[i] = history[i] * window[i];
im[i] = 0f;
}
fft(re, im, false);
boolean warming = warmupFrames < NOISE_WARMUP_FRAMES;
if (warming) {
warmupFrames++;
}
for (int b = 0; b <= FFT_SIZE / 2; b++) {
float power = re[b] * re[b] + im[b] * im[b];
if (warming) {
noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames;
} else if (power < noise[b]) {
// Track the floor down quickly; drift up slowly so speech
// energy never gets learned as noise.
noise[b] += (power - noise[b]) * 0.2f;
} else {
noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f);
}
float clean = power - OVERSUBTRACTION * noise[b];
float g = clean > 0f ? clean / power : 0f;
if (g < GAIN_FLOOR) {
g = GAIN_FLOOR;
}
g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g;
gain[b] = g;
re[b] *= g;
im[b] *= g;
if (b > 0 && b < FFT_SIZE / 2) {
re[FFT_SIZE - b] *= g;
im[FFT_SIZE - b] *= g;
}
}
fft(re, im, true);
for (int i = 0; i < HOP; i++) {
float value = (overlap[i] + re[i] * window[i]) * 32767f;
int sample = Math.round(value);
if (sample > 32767) {
sample = 32767;
}
if (sample < -32768) {
sample = -32768;
}
out[outPos + i * 2] = (byte) (sample & 0xff);
out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff);
}
for (int i = 0; i < HOP; i++) {
overlap[i] = re[HOP + i] * window[HOP + i];
}
return outPos + HOP * 2;
}
/** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */
private static void fft(float[] re, float[] im, boolean inverse) {
int n = re.length;
for (int i = 1, j = 0; i < n; i++) {
int bit = n >> 1;
for (; (j & bit) != 0; bit >>= 1) {
j ^= bit;
}
j ^= bit;
if (i < j) {
float t = re[i];
re[i] = re[j];
re[j] = t;
t = im[i];
im[i] = im[j];
im[j] = t;
}
}
for (int len = 2; len <= n; len <<= 1) {
double ang = 2 * Math.PI / len * (inverse ? 1 : -1);
float wRe = (float) Math.cos(ang);
float wIm = (float) Math.sin(ang);
for (int i = 0; i < n; i += len) {
float curRe = 1f;
float curIm = 0f;
for (int k = 0; k < len / 2; k++) {
int a = i + k;
int b = a + len / 2;
float xr = re[b] * curRe - im[b] * curIm;
float xi = re[b] * curIm + im[b] * curRe;
re[b] = re[a] - xr;
im[b] = im[a] - xi;
re[a] += xr;
im[a] += xi;
float nextRe = curRe * wRe - curIm * wIm;
curIm = curRe * wIm + curIm * wRe;
curRe = nextRe;
}
}
}
if (inverse) {
for (int i = 0; i < n; i++) {
re[i] /= n;
im[i] /= n;
}
}
}
/** Whether this Android build ships the built-in capture noise suppressor. */
public static boolean platformSuppressorAvailable() {
try {
return NoiseSuppressor.isAvailable();
} catch (Throwable t) {
return false;
}
}
/** Short engine name for the UI. */
public static String engineDescription() {
return "spectral (phone DSP)";
}
/**
* Attach the platform's built-in capture-effect chain (NoiseSuppressor,
* AcousticEchoCanceler, AutomaticGainControl — whichever this Android
* build provides) to a phone AudioRecord session. Only applicable when the
* phone itself captures the audio; the BLE glasses stream must go through
* {@link #process(byte[])} instead. The caller keeps the returned effects
* for the life of the recording and release()s them with it.
*/
public static AudioEffect[] attachPlatformEffects(int audioSessionId) {
ArrayList<AudioEffect> effects = new ArrayList<>();
try {
if (NoiseSuppressor.isAvailable()) {
NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId);
if (ns != null) {
ns.setEnabled(true);
effects.add(ns);
}
}
if (AcousticEchoCanceler.isAvailable()) {
AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId);
if (aec != null) {
aec.setEnabled(true);
effects.add(aec);
}
}
if (AutomaticGainControl.isAvailable()) {
AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId);
if (agc != null) {
agc.setEnabled(true);
effects.add(agc);
}
}
} catch (Throwable t) {
// A missing effect just means this tier is unavailable.
}
return effects.toArray(new AudioEffect[0]);
}
}
@@ -0,0 +1,185 @@
package com.faceclaw.app;
import android.content.Context;
/**
* Headless display target for preview-only mode (no glasses paired). Hosts
* the same SurfaceCompositor the BLE communicator uses, so the TS side can
* run its full surface/render pipeline unchanged; composited frames simply
* have nowhere to go, and the phone-mirror preview, screenshots, and GIF
* recordings read the retained composite back exactly as they do on a live
* connection.
*
* The method signatures mirror the display subset of FaceclawBleCommunicator
* (the TS DisplayTarget interface is typed against that subset).
*/
public final class FaceclawPreviewCompositor {
private final Context appContext;
private final SurfaceCompositor compositor = new SurfaceCompositor();
private final android.os.Handler mainHandler = new android.os.Handler(android.os.Looper.getMainLooper());
// Notified (on the main looper, where the phone UI's JS runs) after each
// applied surface frame, standing in for the BLE path's per-frame metrics
// callback so the phone mirror refreshes promptly instead of waiting for
// its safety-net poll.
private volatile Runnable frameListener;
// Active animated-GIF screen recording, or null when idle. Frames are
// pushed by recordScreenFrame(), which the TS side calls at each
// phone-preview flush.
private volatile GifScreenRecorder screenRecorder;
// Worker-app isolates locate their frame target through a static, like
// FaceclawBleCommunicator.getActive(); Java statics are shared across
// isolates, unlike anything on the JS side.
private static volatile FaceclawPreviewCompositor activeInstance;
public static FaceclawPreviewCompositor getActive() {
return activeInstance;
}
public FaceclawPreviewCompositor(Context context) {
this.appContext = context.getApplicationContext();
}
/** Publish this compositor as the workers' frame target. */
public void makeActive() {
activeInstance = this;
}
/** Withdraw from workers (a real connection is taking over). */
public void release() {
if (activeInstance == this) {
activeInstance = null;
}
frameListener = null;
}
public void setFrameListener(Runnable listener) {
this.frameListener = listener;
}
/** Set the compositor's output frame size. Call before configuring surfaces. */
public void configureCompositorScreen(int width, int height) {
compositor.configureScreen(width, height);
}
public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) {
compositor.configureSurface(id, x, y, width, height, zOrder, transparency);
}
public void removeSurface(String id) {
compositor.removeSurface(id);
}
public void setSurfaceVisible(String id, boolean visible) {
compositor.setSurfaceVisible(id, visible);
}
/** Blank (screen off) or unblank the output; retained surface state survives. */
public void setScreenBlanked(boolean blanked) {
compositor.setBlanked(blanked);
}
/**
* Apply an update to one compositor surface. Unlike the connected path
* there is no desired-frame store or transmit pipeline downstream; the
* preview/screenshot calls recomposite the retained state on demand, so
* applying the update is the whole job and the frame finishes here.
*/
public void submitSurfaceFrame(
java.nio.ByteBuffer pixels8bpp,
String surfaceId,
int rectX,
int rectY,
int rectWidth,
int rectHeight,
String contentFingerprint,
int paintMs,
int frameId,
java.nio.ByteBuffer glyphs
) {
compositor.applyAndComposite(
surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs);
FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)");
Runnable listener = frameListener;
if (listener != null) {
mainHandler.post(listener);
}
}
/**
* The current composited screen as a phone-UI preview bitmap, or null
* before any surface has been configured.
*/
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) {
SurfaceCompositor.Composite composite = compositor.previewComposite();
if (composite == null) {
return null;
}
return PreviewBitmapUtil.fromGray(
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green);
}
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
public String saveCompositePngScreenshot() throws java.io.IOException {
SurfaceCompositor.Composite composite = compositor.previewComposite();
if (composite == null) {
return "";
}
return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height);
}
/**
* Save the current composite cropped to the given screen rect (the region
* the shell says is actually occupied). The rect is clamped to the screen;
* a degenerate rect falls back to the full screen.
*/
public String saveCompositePngScreenshot(int cropX, int cropY, int cropWidth, int cropHeight)
throws java.io.IOException {
SurfaceCompositor.Composite composite = compositor.previewComposite();
if (composite == null) {
return "";
}
int x = Math.max(0, cropX);
int y = Math.max(0, cropY);
int width = Math.min(composite.width - x, cropWidth - (x - cropX));
int height = Math.min(composite.height - y, cropHeight - (y - cropY));
if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) {
return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height);
}
byte[] cropped = new byte[width * height];
for (int row = 0; row < height; row++) {
System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width);
}
return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height);
}
/** Begin collecting composite frames for an animated-GIF screen recording. */
public void startScreenRecording() {
screenRecorder = new GifScreenRecorder();
}
/** Capture the current composite into the active recording; no-op when idle. */
public void recordScreenFrame() {
GifScreenRecorder recorder = screenRecorder;
if (recorder == null) {
return;
}
SurfaceCompositor.Composite composite = compositor.previewComposite();
if (composite == null) {
return;
}
recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis());
}
/** Finish the recording and save it as an animated GIF; returns the path or "". */
public String stopScreenRecording() throws java.io.IOException {
GifScreenRecorder recorder = screenRecorder;
screenRecorder = null;
if (recorder == null) {
return "";
}
return recorder.save(appContext);
}
}
@@ -0,0 +1,141 @@
package com.faceclaw.app;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import android.util.Xml;
import org.xmlpull.v1.XmlPullParser;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
/**
* adb-triggered export/import of the faceclaw_settings SharedPreferences
* file, so configuration (API keys, device addresses, ...) can be moved off a
* release build, where run-as is unavailable. See scripts/pull_config.sh and
* scripts/push_config.sh, which use run-as on debuggable builds and fall back
* to this receiver otherwise.
*
* adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \
* -a com.faceclaw.app.SETTINGS_EXPORT
* adb pull /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-export.xml
*
* adb push settings.xml /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-import.xml
* adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \
* -a com.faceclaw.app.SETTINGS_IMPORT
*
* Security: the settings file contains API tokens, so the manifest guards
* this receiver with android.permission.DUMP — a development permission the
* adb shell holds but ordinary apps cannot obtain without adb's help, keeping
* the trigger reachable from adb only. The transfer file lives in
* getExternalFilesDir(), which other apps cannot read on Android 11+ (and
* which the scripts delete as soon as the transfer completes); on the
* Android 7-10 devices minSdk still admits, apps holding READ_EXTERNAL_STORAGE
* could read it during that window.
*/
public class FaceclawSettingsPortReceiver extends BroadcastReceiver {
private static final String TAG = "FaceclawSettingsPort";
public static final String ACTION_EXPORT = "com.faceclaw.app.SETTINGS_EXPORT";
public static final String ACTION_IMPORT = "com.faceclaw.app.SETTINGS_IMPORT";
private static final String PREFS_FILE = "faceclaw_settings.xml";
private static final String EXPORT_NAME = "faceclaw-settings-export.xml";
private static final String IMPORT_NAME = "faceclaw-settings-import.xml";
@Override
public void onReceive(Context context, Intent intent) {
String action = intent == null ? null : intent.getAction();
try {
if (ACTION_EXPORT.equals(action)) {
setResultData(exportSettings(context));
} else if (ACTION_IMPORT.equals(action)) {
setResultData(importSettings(context));
// Exit once the broadcast result has been delivered: this
// process may hold stale in-memory SharedPreferences that a
// later apply() would flush over the imported file, and the
// next launch must re-read from disk.
new Handler(Looper.getMainLooper()).postDelayed(() -> {
Log.i(TAG, "Exiting after settings import");
System.exit(0);
}, 500);
}
} catch (Exception e) {
Log.e(TAG, "Settings " + action + " failed", e);
setResultData("error: " + e);
}
}
private File prefsFile(Context context) {
return new File(context.getApplicationInfo().dataDir, "shared_prefs/" + PREFS_FILE);
}
private File externalFile(Context context, String name) throws IOException {
File dir = context.getExternalFilesDir(null);
if (dir == null) {
throw new IOException("external files dir unavailable");
}
return new File(dir, name);
}
private String exportSettings(Context context) throws IOException {
File prefs = prefsFile(context);
if (!prefs.exists()) {
return "error: no settings file yet (" + prefs + ")";
}
File out = externalFile(context, EXPORT_NAME);
copy(prefs, out);
return "exported: " + out;
}
private String importSettings(Context context) throws Exception {
File staged = externalFile(context, IMPORT_NAME);
if (!staged.exists()) {
return "error: nothing staged at " + staged;
}
validatePrefsXml(staged);
File prefs = prefsFile(context);
prefs.getParentFile().mkdirs();
if (prefs.exists()) {
copy(prefs, new File(prefs.getPath() + ".bak"));
}
copy(staged, prefs);
staged.delete();
return "imported: " + prefs + " (previous settings in " + PREFS_FILE + ".bak)";
}
/** Reject files that are not a SharedPreferences <map> document. */
private void validatePrefsXml(File file) throws Exception {
try (InputStream in = new FileInputStream(file)) {
XmlPullParser parser = Xml.newPullParser();
parser.setInput(in, null);
int event = parser.next();
while (event != XmlPullParser.START_TAG && event != XmlPullParser.END_DOCUMENT) {
event = parser.next();
}
if (event != XmlPullParser.START_TAG || !"map".equals(parser.getName())) {
throw new IOException("not a SharedPreferences <map> document");
}
// Walk the rest so malformed XML is caught before it replaces
// the live settings file.
while (parser.next() != XmlPullParser.END_DOCUMENT) { }
}
}
private void copy(File from, File to) throws IOException {
try (InputStream in = new FileInputStream(from);
OutputStream out = new FileOutputStream(to)) {
byte[] buffer = new byte[8192];
int n;
while ((n = in.read(buffer)) > 0) {
out.write(buffer, 0, n);
}
}
}
}
@@ -0,0 +1,111 @@
package com.faceclaw.app;
import android.util.Log;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
/**
* Speaker voice-print extraction over sherpa-onnx. One embedding per
* utterance; matching/enrollment happens on the TS side (cosine similarity
* against the stored speaker profiles).
*/
public class FaceclawSpeakerId {
private static final String TAG = "FaceclawSpeakerId";
private SpeakerEmbeddingExtractor extractor;
private final String modelPath;
public FaceclawSpeakerId(String modelPath) {
this.modelPath = modelPath;
}
/** Lazily loads the model; returns false when it cannot be loaded. */
public synchronized boolean ensureLoaded() {
if (extractor != null) {
return true;
}
try {
SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder()
.setModel(modelPath)
.setNumThreads(1)
.setDebug(false)
.setProvider("cpu")
.build();
extractor = new SpeakerEmbeddingExtractor(config);
return true;
} catch (Throwable t) {
Log.w(TAG, "could not load speaker model at " + modelPath, t);
extractor = null;
return false;
}
}
public synchronized int embeddingDim() {
return extractor == null ? 0 : extractor.getDim();
}
/**
* Compute an L2-normalized voice-print for one utterance of 16 kHz mono
* S16LE PCM. Returns null when the model is unavailable or the utterance
* is too short to embed (under ~0.5 s).
*/
public synchronized float[] embed(byte[] pcm16le, int sampleRate) {
if (pcm16le == null || pcm16le.length < sampleRate) {
return null;
}
if (!ensureLoaded()) {
return null;
}
int count = pcm16le.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8));
samples[i] = s / 32768.0f;
}
OnlineStream stream = null;
try {
stream = extractor.createStream();
stream.acceptWaveform(samples, sampleRate);
stream.inputFinished();
if (!extractor.isReady(stream)) {
return null;
}
float[] embedding = extractor.compute(stream);
return l2Normalize(embedding);
} catch (Throwable t) {
Log.w(TAG, "embedding failed", t);
return null;
} finally {
if (stream != null) {
stream.release();
}
}
}
private static float[] l2Normalize(float[] v) {
if (v == null) {
return null;
}
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return v;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
return v;
}
public synchronized void close() {
if (extractor != null) {
extractor.release();
extractor = null;
}
}
}
@@ -76,6 +76,15 @@ public class FaceclawVoiceController {
private volatile FaceclawBleCommunicator communicator;
private Thread workerThread;
private volatile boolean started;
// Set once the worker has the glasses mic enabled for this session.
// Read and written under `lock`, so it flips with `started` atomically.
private boolean audioStarted;
// Capture from the phone's own microphone instead of the G2 over BLE
// (preview-only mode, where no glasses are connected). Latched into
// activePhoneMic at start() (under `lock`) so a mid-session setter call
// can't switch pipelines underneath the worker.
private volatile boolean usePhoneMic;
private boolean activePhoneMic;
private VoiceInputMode mode = VoiceInputMode.CLOUD;
private OfflineRecognizer recognizer;
private FaceclawLc3Decoder lc3Decoder;
@@ -90,6 +99,26 @@ public class FaceclawVoiceController {
private volatile boolean endpointing;
private final EndpointDetector endpointDetector = new EndpointDetector();
private java.io.ByteArrayOutputStream recordingPcm;
// Speaker verification against the enrolled wearer voice-print ("my voice
// only" command gating). Configured before start(); the utterance PCM is
// buffered (capped) and verified once at session end.
private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10;
private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE;
private volatile String verifySpeakerModelPath;
private volatile float[] verifyWearerEmbedding;
private volatile float verifyThreshold = 0.8f;
private short[] verifyBuffer;
private int verifyCount;
// Global mic processing (Microphones app config): spectral noise
// suppression and firmware-DoA beam gating, applied to every capture
// session that opts in (assistant push-to-talk, Transcribe, hands-free).
// The raw tap opts out — raw means raw, and the Microphones session does
// its own beam-compensated processing on that path.
private volatile boolean suppressionEnabled;
private volatile boolean beamFilterEnabled;
private volatile int beamCenterDeg;
private volatile int beamHalfWidthDeg = 180;
private FaceclawNoiseSuppressor suppressor;
private long queuedPackets;
private long queueDroppedPackets;
private long decodedSamples;
@@ -111,6 +140,11 @@ public class FaceclawVoiceController {
this.communicator = communicator;
}
/** Source the next capture from the phone microphone (no glasses paired). */
public void setUsePhoneMic(boolean usePhoneMic) {
this.usePhoneMic = usePhoneMic;
}
/** When true, the decoded mic PCM for each session is saved as a WAV. */
public void setSaveRecordings(boolean saveRecordings) {
this.saveRecordings = saveRecordings;
@@ -125,23 +159,117 @@ public class FaceclawVoiceController {
this.endpointing = endpointing;
}
/**
* Verify this session's speaker against the enrolled wearer voice-print
* and report the result via onSpeakerVerified just before the final
* transcript. Must be set before {@link #start}; pass a null model path
* to disable.
*/
public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) {
this.verifySpeakerModelPath = speakerModelPath;
this.verifyWearerEmbedding = wearerEmbedding;
this.verifyThreshold = threshold > 0 ? threshold : 0.8f;
}
public void clearSpeakerVerification() {
this.verifySpeakerModelPath = null;
this.verifyWearerEmbedding = null;
}
/** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */
public void setNoiseSuppression(boolean enabled) {
this.suppressionEnabled = enabled;
}
/**
* Direction gating from the Sonic Radar beam: packets whose firmware
* direction-of-arrival falls outside centerDeg ± halfWidthDeg (device
* frame, 0 = straight ahead, positive right) are dropped before any
* consumer sees them. Safe to update mid-run.
*/
public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) {
this.beamFilterEnabled = enabled;
this.beamCenterDeg = centerDeg;
this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg));
}
private boolean withinBeam(int angleDegrees) {
int delta = angleDegrees - beamCenterDeg;
while (delta > 180) delta -= 360;
while (delta < -180) delta += 360;
return Math.abs(delta) <= beamHalfWidthDeg;
}
private void applySuppression(short[] pcm, int count) {
try {
if (suppressor == null) {
suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE);
}
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
le[i * 2] = (byte) (pcm[i] & 0xff);
le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff);
}
byte[] cleaned = suppressor.process(le);
int cleanedCount = Math.min(count, cleaned.length / 2);
for (int i = 0; i < cleanedCount; i++) {
pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8));
}
} catch (Throwable t) {
Log.w(TAG, "noise suppression failed; passing audio through", t);
suppressionEnabled = false;
}
}
public void start(String requestedMode) {
synchronized (lock) {
if (started) {
emitStatus("Voice control is already listening.");
return;
}
if (communicator == null || !communicator.isSessionReady()) {
if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) {
emitStatus("Voice control needs an active G2 connection.");
return;
}
mode = parseMode(requestedMode);
activePhoneMic = usePhoneMic;
started = true;
audioStarted = false;
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
workerThread.start();
}
}
/**
* Whether mic audio is actually flowing. {@link #start} only records
* intent: the enable lives in the glasses' EvenHub session, so a transport
* drop or a session suspend can leave this controller started with a
* worker that will never see another packet. Anything deciding whether to
* (re)start capture must ask this rather than assume its own bookkeeping.
*/
public boolean isCapturing() {
boolean audioUp;
boolean phoneMic;
synchronized (lock) {
if (!started) {
return false;
}
audioUp = audioStarted;
phoneMic = activePhoneMic;
}
if (!audioUp) {
// The worker is still bringing the mic up; report it as running so
// a concurrent request shares it instead of restarting it.
return true;
}
if (phoneMic) {
// AudioRecord has no session to lose the enable to; it runs until stop().
return true;
}
FaceclawBleCommunicator currentCommunicator = communicator;
return currentCommunicator != null && currentCommunicator.isAudioCaptureActive();
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
@@ -193,18 +321,54 @@ public class FaceclawVoiceController {
resetTranscriptState();
lastTranscript = "";
}
lc3Decoder = new FaceclawLc3Decoder();
endpointDetector.reset();
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.");
return;
if (suppressor != null) {
suppressor.reset();
}
emitStatus(currentMode == VoiceInputMode.CLOUD
? "Listening (cloud)..."
: "Listening...");
processG2Audio();
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null;
verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null;
verifyCount = 0;
if (activePhoneMic) {
android.media.AudioRecord record = openPhoneMic();
if (record == null) {
emitStatus("Could not start the phone microphone.");
return;
}
synchronized (lock) {
audioStarted = true;
}
emitStatus(currentMode == VoiceInputMode.CLOUD
? "Listening (cloud)..."
: "Listening...");
try {
processPhoneAudio(record);
} finally {
try {
record.stop();
} catch (Throwable ignored) {
// Already stopped or never recording; release below either way.
}
record.release();
}
} else {
lc3Decoder = new FaceclawLc3Decoder();
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.");
return;
}
synchronized (lock) {
audioStarted = true;
}
emitStatus(currentMode == VoiceInputMode.CLOUD
? "Listening (cloud)..."
: "Listening...");
processG2Audio();
}
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
// on (the callbacks are posted in order to the main handler).
runSpeakerVerification();
// Button released / stop requested: emit one final full-utterance
// transcript so the UI can freeze it.
if (currentMode == VoiceInputMode.ONBOARD) {
@@ -220,6 +384,7 @@ public class FaceclawVoiceController {
releaseLc3();
synchronized (lock) {
started = false;
audioStarted = false;
workerThread = null;
}
}
@@ -374,25 +539,123 @@ public class FaceclawVoiceController {
continue;
}
decodedSamples += count;
if (recordingPcm != null) {
appendRecording(pcm, count);
}
if (endpointing && endpointDetector.accept(pcm, count)) {
emitSpeechEnd();
}
if (mode == VoiceInputMode.CLOUD) {
emitPcm(pcm, count);
} else {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
processRecognizer(samples);
// Global mic processing from the Microphones app config: the
// beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker
// for every consumer, recognition included), and the spectral
// noise suppressor cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
int angleDegrees = currentDecoder.getLastAngleDegrees();
int ssr = currentDecoder.getLastSsr();
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr);
maybeEmitAudioStats(false);
continue;
}
processPcmChunk(pcm, count, angleDegrees, ssr, true);
maybeEmitAudioStats(false);
}
}
/**
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
* decode and the beam filter. hasFrameMeta is false for the phone mic,
* which has no firmware DSP metadata to report.
*/
private void processPcmChunk(short[] pcm, int count, int angleDegrees, int ssr, boolean hasFrameMeta) {
if (suppressionEnabled) {
applySuppression(pcm, count);
}
if (recordingPcm != null) {
appendRecording(pcm, count);
}
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount);
System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied);
verifyCount += copied;
}
if (endpointing && endpointDetector.accept(pcm, count)) {
emitSpeechEnd();
}
// PCM and frame metadata flow in every mode so levels, recording,
// and the Microphones radar keep working alongside onboard ASR.
emitPcm(pcm, count);
if (hasFrameMeta) {
emitFrameMeta(angleDegrees, ssr);
}
if (mode != VoiceInputMode.CLOUD) {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
processRecognizer(samples);
}
}
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
// (endpointing, transcript pacing) is tuned for.
private static final int PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20;
/**
* Open the phone's own microphone at the pipeline's native format
* (16 kHz mono PCM16), or null when it cannot start — the permission is
* missing (SecurityException) or the device refuses the configuration.
*/
private android.media.AudioRecord openPhoneMic() {
android.media.AudioRecord record = null;
try {
int minBytes = android.media.AudioRecord.getMinBufferSize(
SAMPLE_RATE,
android.media.AudioFormat.CHANNEL_IN_MONO,
android.media.AudioFormat.ENCODING_PCM_16BIT);
int bufferBytes = Math.max(minBytes, PHONE_MIC_CHUNK_SAMPLES * 2 * 4);
record = new android.media.AudioRecord(
android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION,
SAMPLE_RATE,
android.media.AudioFormat.CHANNEL_IN_MONO,
android.media.AudioFormat.ENCODING_PCM_16BIT,
bufferBytes);
if (record.getState() != android.media.AudioRecord.STATE_INITIALIZED) {
record.release();
return null;
}
record.startRecording();
if (record.getRecordingState() != android.media.AudioRecord.RECORDSTATE_RECORDING) {
record.release();
return null;
}
return record;
} catch (Throwable t) {
Log.w(TAG, "phone mic open failed", t);
if (record != null) {
record.release();
}
return null;
}
}
/**
* Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame
* metadata — AudioRecord already delivers the pipeline's PCM format. The
* blocking read returns every chunk (50 ms), which bounds how long a
* stop() waits for the loop to notice `started` dropped.
*/
private void processPhoneAudio(android.media.AudioRecord record) {
short[] pcm = new short[PHONE_MIC_CHUNK_SAMPLES];
while (started && !Thread.currentThread().isInterrupted()) {
int read = record.read(pcm, 0, pcm.length);
if (read < 0) {
Log.w(TAG, "phone mic read failed: " + read);
return;
}
if (read == 0) {
continue;
}
decodedSamples += read;
processPcmChunk(pcm, read, 0, 0, false);
}
}
private void processRecognizer(float[] samples) {
appendTranscriptSamples(samples);
long now = SystemClock.elapsedRealtime();
@@ -582,6 +845,83 @@ public class FaceclawVoiceController {
mainHandler.post(() -> currentListener.onPcm(le));
}
/**
* Embed the session's buffered utterance and compare it to the enrolled
* wearer voice-print. Fails open: a session too short to verify, or a
* model that will not load, counts as the wearer rather than silencing
* every command.
*/
private void runSpeakerVerification() {
short[] buffer = verifyBuffer;
float[] wearer = verifyWearerEmbedding;
String modelPath = verifySpeakerModelPath;
verifyBuffer = null;
if (buffer == null || wearer == null || modelPath == null) {
return;
}
if (verifyCount < VERIFY_MIN_SAMPLES) {
emitSpeakerVerified(true, 0f);
return;
}
FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath);
try {
byte[] le = new byte[verifyCount * 2];
for (int i = 0; i < verifyCount; i++) {
short s = buffer[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
float[] embedding = speakerId.embed(le, SAMPLE_RATE);
if (embedding == null || embedding.length != wearer.length) {
emitSpeakerVerified(true, 0f);
return;
}
double dot = 0;
for (int i = 0; i < embedding.length; i++) {
dot += (double) embedding[i] * wearer[i];
}
boolean isWearer = dot >= verifyThreshold;
Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot)
+ " threshold=" + verifyThreshold + " isWearer=" + isWearer);
emitSpeakerVerified(isWearer, (float) dot);
} catch (Throwable t) {
Log.w(TAG, "speaker verification failed", t);
emitSpeakerVerified(true, 0f);
}
}
// The 28 MB embedding model takes seconds to load; keep one instance
// across capture sessions so verification adds only the embed time.
private static FaceclawSpeakerId sharedSpeakerId;
private static String sharedSpeakerIdPath;
private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) {
if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) {
if (sharedSpeakerId != null) {
sharedSpeakerId.close();
}
sharedSpeakerId = new FaceclawSpeakerId(modelPath);
sharedSpeakerIdPath = modelPath;
}
return sharedSpeakerId;
}
private void emitSpeakerVerified(boolean isWearer, float similarity) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity));
}
private void emitFrameMeta(int angleDegrees, int ssr) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr));
}
private void emitSpeechEnd() {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
@@ -11,15 +11,34 @@ public interface FaceclawVoiceControllerListener {
void onTranscript(String text, boolean isFinal);
/**
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
* little-endian PCM. Empty/absent in onboard mode.
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
* keep working while the onboard recognizer owns the transcript.
*/
void onPcm(byte[] pcm16le);
/**
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
* packet, computed on the raw stereo capture before its mono downmix: the
* direction-of-arrival in signed degrees (0 = straight ahead, positive to
* the wearer's right) and a signal-strength ratio that serves as a
* confidence/energy proxy for the angle.
*/
void onFrameMeta(int angleDegrees, int ssr);
/**
* The speaker stopped, in a hands-free session that has no button release
* to end it. Only fires when endpointing was enabled for the session; see
* FaceclawVoiceController.setEndpointing.
*/
void onSpeechEnd();
/**
* Result of speaker verification against the enrolled wearer voice-print,
* emitted once per capture session just before the final transcript.
* Only fires when verification was configured for the session; fails
* open (isWearer=true) when the utterance was too short to verify or the
* speaker model could not be loaded.
*/
void onSpeakerVerified(boolean isWearer, float similarity);
}
@@ -0,0 +1,100 @@
package com.faceclaw.app;
import android.util.Log;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
/**
* Streaming WAV writer for conversation recordings: the header is written up
* front with zero sizes and patched on finish, so a crash mid-session leaves
* a recoverable file (data length can be derived from the file size).
*/
public class FaceclawWavRecorder {
private static final String TAG = "FaceclawWavRecorder";
private final String path;
private final int sampleRate;
private final int channels;
private RandomAccessFile file;
private long dataBytes;
public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException {
this.path = path;
this.sampleRate = sampleRate;
this.channels = channels;
File parent = new File(path).getParentFile();
if (parent != null && !parent.exists() && !parent.mkdirs()) {
throw new IOException("could not create " + parent);
}
file = new RandomAccessFile(path, "rw");
file.setLength(0);
file.write(buildHeader(0));
}
public synchronized void append(byte[] pcm16le) {
if (file == null || pcm16le == null || pcm16le.length == 0) {
return;
}
try {
file.write(pcm16le);
dataBytes += pcm16le.length;
} catch (IOException e) {
Log.w(TAG, "append failed", e);
}
}
public synchronized long getDurationMs() {
return dataBytes * 1000L / (sampleRate * channels * 2L);
}
public String getPath() {
return path;
}
/** Patch the header and close. Returns the PCM byte count (0 = empty). */
public synchronized long finish() {
RandomAccessFile current = file;
file = null;
if (current == null) {
return dataBytes;
}
try {
current.seek(0);
current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE)));
current.close();
} catch (IOException e) {
Log.w(TAG, "finish failed", e);
}
return dataBytes;
}
/** Delete the (finished or abandoned) recording. */
public synchronized void discard() {
finish();
new File(path).delete();
}
private byte[] buildHeader(int pcmBytes) {
int byteRate = sampleRate * channels * 2;
ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN);
b.put("RIFF".getBytes(StandardCharsets.US_ASCII));
b.putInt(36 + pcmBytes);
b.put("WAVE".getBytes(StandardCharsets.US_ASCII));
b.put("fmt ".getBytes(StandardCharsets.US_ASCII));
b.putInt(16);
b.putShort((short) 1);
b.putShort((short) channels);
b.putInt(sampleRate);
b.putInt(byteRate);
b.putShort((short) (channels * 2));
b.putShort((short) 16);
b.put("data".getBytes(StandardCharsets.US_ASCII));
b.putInt(pcmBytes);
return b.array();
}
}
@@ -0,0 +1,383 @@
package com.faceclaw.app;
import android.content.Context;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.google.android.gms.common.ConnectionResult;
import com.google.android.gms.common.GoogleApiAvailability;
import com.google.android.gms.wearable.CapabilityClient;
import com.google.android.gms.wearable.CapabilityInfo;
import com.google.android.gms.wearable.DataMap;
import com.google.android.gms.wearable.MessageEvent;
import com.google.android.gms.wearable.Node;
import com.google.android.gms.wearable.PutDataMapRequest;
import com.google.android.gms.wearable.PutDataRequest;
import com.google.android.gms.wearable.Wearable;
import org.json.JSONException;
import org.json.JSONObject;
import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
/**
* Phone side of the Wear OS integration: the single place that talks to the
* Wearable Data Layer (Google Play services), so the JS side only ever sees
* JSON strings and never touches Play services types.
*
* Inbound: FaceclawWearListenerService hands every watch message to
* {@link #handleMessage}; it is forwarded to the JS listener on the thread
* that registered it (the main isolate), mirroring FaceclawSettings. Messages
* that arrive before a listener exists (the watch poked a phone whose JS
* dashboard is not up yet) are acked with jsReady=false so the watch can say
* "open Faceclaw on your phone"; only idempotent state requests are kept for
* replay on registration — the watch already reported gestures and commands
* as failed, and executing them tens of seconds later would act on whatever
* the glasses happen to show by then.
*
* Outbound: {@link #publishState} mirrors the dashboard state into a Data
* Layer item (delivered even if the watch is out of range right now, and
* available to the watch app the moment it opens), while {@link #sendToWatch}
* sends fire-and-forget messages (acks, assistant text) to every reachable
* watch that advertises the Faceclaw watch-app capability.
*
* Everything is a no-op on phones without Google Play services.
*/
public final class FaceclawWearBridge {
private static final String TAG = "FaceclawWear";
/** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */
public static final String CAPABILITY_WATCH = "faceclaw_watch";
/** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */
public static final String CAPABILITY_PHONE = "faceclaw_phone";
public static final String PATH_PREFIX = "/faceclaw";
public static final String PATH_STATE = "/faceclaw/state";
public static final String PATH_STATE_REQUEST = "/faceclaw/state/request";
public static final String PATH_ACK = "/faceclaw/ack";
public static final String PATH_EVENT = "/faceclaw/event";
private static final String STATE_KEY_JSON = "json";
private static final String STATE_KEY_UPDATED_AT = "updatedAt";
/** Messages kept for a listener that is not registered yet. */
private static final int PENDING_LIMIT = 16;
private static final long PENDING_MAX_AGE_MS = 30_000;
private static volatile FaceclawWearBridge instance;
private final Context context;
private final boolean available;
private final Object lock = new Object();
private FaceclawWearListener listener;
private Handler listenerHandler;
private final ArrayDeque<PendingMessage> pending = new ArrayDeque<>();
private final List<Node> watchNodes = new ArrayList<>();
private String lastPublishedState = null;
private static final class PendingMessage {
final String path;
final String json;
final String nodeId;
final long receivedAt;
PendingMessage(String path, String json, String nodeId, long receivedAt) {
this.path = path;
this.json = json;
this.nodeId = nodeId;
this.receivedAt = receivedAt;
}
}
private FaceclawWearBridge(Context context) {
this.context = context.getApplicationContext();
boolean playServices;
try {
playServices = GoogleApiAvailability.getInstance()
.isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS;
} catch (Throwable error) {
Log.w(TAG, "Play services availability check failed", error);
playServices = false;
}
this.available = playServices;
if (!available) {
Log.i(TAG, "Google Play services unavailable; watch integration disabled");
}
}
/** Initialize (idempotent) and return the singleton. */
public static FaceclawWearBridge getInstance(Context context) {
if (instance == null) {
synchronized (FaceclawWearBridge.class) {
if (instance == null) {
instance = new FaceclawWearBridge(context);
}
}
}
return instance;
}
/** Whether the Wearable Data Layer can be used on this phone at all. */
public boolean isAvailable() {
return available;
}
/**
* Register the JS listener. Must be called from the thread whose isolate
* owns it; that thread's Looper is captured for dispatch. Replays any
* recently queued watch messages and refreshes the reachable-watch set.
*/
public void setListener(FaceclawWearListener newListener) {
Looper looper = Looper.myLooper();
if (looper == null) {
Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified");
}
List<PendingMessage> replay;
synchronized (lock) {
listener = newListener;
listenerHandler = looper != null ? new Handler(looper) : null;
replay = new ArrayList<>(pending);
pending.clear();
}
long now = System.currentTimeMillis();
for (PendingMessage message : replay) {
if (now - message.receivedAt <= PENDING_MAX_AGE_MS) {
dispatchMessage(message.path, message.json, message.nodeId);
}
}
refreshWatchNodes();
}
public void clearListener() {
synchronized (lock) {
listener = null;
listenerHandler = null;
}
}
public boolean isWatchReachable() {
synchronized (lock) {
return !watchNodes.isEmpty();
}
}
/** Display name of a reachable watch ("" if none). */
public String getWatchName() {
synchronized (lock) {
return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
}
}
/** Re-query which watches advertise the Faceclaw watch-app capability. */
public void refreshWatchNodes() {
if (!available) return;
try {
Wearable.getCapabilityClient(context)
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
.addOnSuccessListener(this::updateWatchNodes)
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
} catch (Throwable error) {
Log.w(TAG, "capability query failed", error);
}
}
/**
* Mirror the dashboard state to the watch. Identical consecutive states
* are not re-sent (the Data Layer would drop them anyway).
*/
public void publishState(String json) {
publishState(json, false);
}
/** As {@link #publishState(String)}; `force` re-sends an unchanged state. */
public void publishState(String json, boolean force) {
if (!available || json == null) return;
synchronized (lock) {
if (!force && json.equals(lastPublishedState)) return;
lastPublishedState = json;
}
try {
PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE);
DataMap map = request.getDataMap();
map.putString(STATE_KEY_JSON, json);
map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis());
PutDataRequest put = request.asPutDataRequest().setUrgent();
Wearable.getDataClient(context)
.putDataItem(put)
.addOnFailureListener(error -> Log.w(TAG, "state publish failed", error));
} catch (Throwable error) {
Log.w(TAG, "state publish failed", error);
}
}
/** Send a message to every reachable watch running the Faceclaw watch app. */
public void sendToWatch(String path, String json) {
if (!available) return;
List<Node> targets;
synchronized (lock) {
targets = new ArrayList<>(watchNodes);
}
if (targets.isEmpty()) {
// Nothing known yet (first message after boot): look the watch up, then send.
try {
Wearable.getCapabilityClient(context)
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
.addOnSuccessListener(info -> {
updateWatchNodes(info);
for (Node node : info.getNodes()) {
sendToNode(node.getId(), path, json);
}
})
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
} catch (Throwable error) {
Log.w(TAG, "capability query failed", error);
}
return;
}
for (Node node : targets) {
sendToNode(node.getId(), path, json);
}
}
/** Send a message to one specific node (e.g. an ack back to the sender). */
public void sendToNode(String nodeId, String path, String json) {
if (!available || nodeId == null || nodeId.isEmpty()) return;
try {
byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8);
Wearable.getMessageClient(context)
.sendMessage(nodeId, path, payload)
.addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error));
} catch (Throwable error) {
Log.w(TAG, "send " + path + " failed", error);
}
}
/** Reply to a watch message; `seq` echoes the watch's sequence number. */
public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) {
JSONObject ack = new JSONObject();
try {
ack.put("seq", seq);
ack.put("ok", ok);
ack.put("jsReady", jsReady);
ack.put("message", message != null ? message : "");
} catch (JSONException ignored) {
// The keys are constants; this cannot happen.
}
sendToNode(nodeId, PATH_ACK, ack.toString());
}
/** Entry point for FaceclawWearListenerService; may run on any thread. */
void handleMessage(MessageEvent event) {
if (event == null) return;
String path = event.getPath();
if (path == null || !path.startsWith(PATH_PREFIX)) return;
byte[] data = event.getData();
String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}";
String nodeId = event.getSourceNodeId();
Log.d(TAG, "recv " + path + " seq=" + readSeq(json));
boolean delivered = dispatchMessage(path, json, nodeId);
if (delivered) return;
// No JS listener yet: tell the watch so it can explain why nothing is
// happening. Only state requests are kept for replay (see class doc);
// everything else is dropped after the failure ack.
if (PATH_STATE_REQUEST.equals(path)) {
synchronized (lock) {
long now = System.currentTimeMillis();
while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) {
pending.pollFirst();
}
if (pending.size() >= PENDING_LIMIT) {
pending.pollFirst();
}
pending.addLast(new PendingMessage(path, json, nodeId, now));
}
}
sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses.");
}
/** Entry point for FaceclawWearListenerService's capability callback. */
void handleCapabilityChanged(CapabilityInfo info) {
if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return;
updateWatchNodes(info);
}
private void updateWatchNodes(CapabilityInfo info) {
Set<Node> nodes = info != null ? info.getNodes() : Collections.<Node>emptySet();
// Every reachable watch gets messages; a nearby (Bluetooth-linked)
// one is listed first so it is the name the phone UI shows.
List<Node> reachable = new ArrayList<>();
for (Node node : nodes) {
if (node.isNearby()) reachable.add(0, node); else reachable.add(node);
}
boolean changed;
String name;
synchronized (lock) {
changed = !sameNodeIds(watchNodes, reachable);
watchNodes.clear();
watchNodes.addAll(reachable);
name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
}
if (changed) {
Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name);
boolean reachableNow = !reachable.isEmpty();
FaceclawWearListener target;
Handler handler;
synchronized (lock) {
target = listener;
handler = listenerHandler;
}
if (target != null && handler != null) {
handler.post(() -> {
try {
target.onWatchConnection(reachableNow, name);
} catch (Exception error) {
Log.w(TAG, "watch connection listener failed", error);
}
});
}
}
}
private static boolean sameNodeIds(List<Node> a, List<Node> b) {
if (a.size() != b.size()) return false;
for (int i = 0; i < a.size(); i++) {
if (!a.get(i).getId().equals(b.get(i).getId())) return false;
}
return true;
}
/** Post to the JS listener; false when there is none to post to. */
private boolean dispatchMessage(String path, String json, String nodeId) {
FaceclawWearListener target;
Handler handler;
synchronized (lock) {
target = listener;
handler = listenerHandler;
}
if (target == null || handler == null) return false;
handler.post(() -> {
try {
target.onMessage(path, json, nodeId);
} catch (Exception error) {
Log.w(TAG, "wear message listener failed for " + path, error);
}
});
return true;
}
private static long readSeq(String json) {
try {
return new JSONObject(json).optLong("seq", 0);
} catch (JSONException error) {
return 0;
}
}
}
@@ -0,0 +1,19 @@
package com.faceclaw.app;
/**
* Callbacks from the Android Wear (Wear OS) bridge; implemented in JS. Every
* call is delivered on the thread that registered the listener (see
* FaceclawWearBridge.setListener).
*/
public interface FaceclawWearListener {
/**
* A message from the watch app. `path` is the Data Layer message path
* (e.g. "/faceclaw/input"); `json` is the UTF-8 payload (always a JSON
* object, "{}" when the watch sent none); `nodeId` identifies the watch
* so a reply can be addressed to it.
*/
void onMessage(String path, String json, String nodeId);
/** The set of reachable watches running the Faceclaw watch app changed. */
void onWatchConnection(boolean reachable, String watchName);
}
@@ -0,0 +1,26 @@
package com.faceclaw.app;
import android.util.Log;
import com.google.android.gms.wearable.CapabilityInfo;
import com.google.android.gms.wearable.MessageEvent;
import com.google.android.gms.wearable.WearableListenerService;
/**
* Play services starts this service (in the app process) for every Data
* Layer message under /faceclaw and for capability changes, whether or not
* the app is running. It only forwards to FaceclawWearBridge, which owns the
* hand-off to the JS side.
*/
public class FaceclawWearListenerService extends WearableListenerService {
@Override
public void onMessageReceived(MessageEvent event) {
Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId());
FaceclawWearBridge.getInstance(this).handleMessage(event);
}
@Override
public void onCapabilityChanged(CapabilityInfo info) {
FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info);
}
}
@@ -15,13 +15,18 @@ public final class PreviewBitmapUtil {
private PreviewBitmapUtil() {}
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) {
return fromGray(gray, width, height, brightenGamma, false);
}
/** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma, boolean green) {
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
throw new IllegalArgumentException("invalid gray preview buffer");
}
int[] lut = new int[256];
for (int g = 0; g < 256; g++) {
int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma))));
lut[g] = 0xff000000 | (v << 16) | (v << 8) | v;
lut[g] = green ? (0xff000000 | (v << 8)) : (0xff000000 | (v << 16) | (v << 8) | v);
}
byte[] bytes = new byte[width * height];
gray.get(bytes);
@@ -49,6 +49,14 @@ public class BleProtocol {
// protobuf fields to stock implementations, so they are safely ignored.
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
/**
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
* to settings read responses and as standalone commandId=3 pushes.
*/
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
public static final int FACECLAW_WAKE_OP_ACQUIRE = 1;
public static final int FACECLAW_WAKE_OP_RELEASE = 2;
public static final int FACECLAW_WAKE_OP_CLAIM = 3;
@@ -405,6 +413,40 @@ public class BleProtocol {
));
}
/**
* Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a
* settings write carrying CFW field 103. Magic zero makes it
* fire-and-forget like the wake lease: the firmware consumes the field
* before its stock nanopb decoder discards the unknown tag, and the
* "ack" is the field-104 status notify that follows.
*/
public static byte[] buildFaceclawMicControl(byte[] record) {
return concat(CollectionUtils.listOf(
encodeVarintField(1, 1),
encodeVarintField(2, 0),
encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record)
));
}
/**
* Extract the CFW mic status record (field 104) from a settings frame, or
* null when absent. The 21-byte body layout is decoded on the TS side;
* here we only validate the 'M','C',version prelude.
*/
public static byte[] parseFaceclawMicStatus(byte[] pb) {
if (pb == null) {
return null;
}
byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD);
if (body == null
|| body.length < 3
|| body[0] != 'M'
|| body[1] != 'C') {
return null;
}
return body;
}
/**
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
* when this is an ordinary settings frame.
@@ -267,6 +267,21 @@ public class MessageBuilder {
);
}
/** CFW mic_control write (settings field 103) targeted at one temple. */
public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) {
return new OutboundMessage(
"mic-control",
"mic control " + label + (leftArm ? " L" : " R"),
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
0,
BleProtocol.buildFaceclawMicControl(record),
ACK_TIMEOUT_MS,
-1,
leftArm
);
}
public OutboundMessage faceclawWearQuery(boolean leftArm) {
return new OutboundMessage(
"wear-query-control",
@@ -0,0 +1,44 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class FastClusteringConfig {
private final int numClusters;
private final float threshold;
private FastClusteringConfig(Builder builder) {
this.numClusters = builder.numClusters;
this.threshold = builder.threshold;
}
public static Builder builder() {
return new Builder();
}
public int getNumClusters() {
return numClusters;
}
public float getThreshold() {
return threshold;
}
public static class Builder {
private int numClusters = -1;
private float threshold = 0.5f;
public FastClusteringConfig build() {
return new FastClusteringConfig(this);
}
public Builder setNumClusters(int numClusters) {
this.numClusters = numClusters;
return this;
}
public Builder setThreshold(float threshold) {
this.threshold = threshold;
return this;
}
}
}
@@ -0,0 +1,62 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarization {
private long ptr = 0;
public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization");
}
}
public int getSampleRate() {
return getSampleRate(ptr);
}
// Only config.clustering is used. All other fields are ignored
public void setConfig(OfflineSpeakerDiarizationConfig config) {
setConfig(ptr, config);
}
public OfflineSpeakerDiarizationSegment[] process(float[] samples) {
return process(ptr, samples);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) {
return processWithCallback(ptr, samples, callback, 0);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) {
return processWithCallback(ptr, samples, callback, arg);
}
protected void finalize() throws Throwable {
release();
}
// You'd better call it manually if it is not used anymore
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
private native int getSampleRate(long ptr);
private native void delete(long ptr);
private native long newFromFile(OfflineSpeakerDiarizationConfig config);
private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config);
private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples);
private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg);
}
@@ -0,0 +1,8 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
@FunctionalInterface
public interface OfflineSpeakerDiarizationCallback {
Integer invoke(int numProcessedChunks, int numTotalCunks, long arg);
}
@@ -0,0 +1,79 @@
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationConfig {
private final OfflineSpeakerSegmentationModelConfig segmentation;
private final SpeakerEmbeddingExtractorConfig embedding;
private final FastClusteringConfig clustering;
private final float minDurationOn;
private final float minDurationOff;
private OfflineSpeakerDiarizationConfig(Builder builder) {
this.segmentation = builder.segmentation;
this.embedding = builder.embedding;
this.clustering = builder.clustering;
this.minDurationOff = builder.minDurationOff;
this.minDurationOn = builder.minDurationOn;
}
public static Builder builder() {
return new Builder();
}
public OfflineSpeakerSegmentationModelConfig getSegmentation() {
return segmentation;
}
public SpeakerEmbeddingExtractorConfig getEmbedding() {
return embedding;
}
public FastClusteringConfig getClustering() {
return clustering;
}
public float getMinDurationOff() {
return minDurationOff;
}
public float getMinDurationOn() {
return minDurationOn;
}
public static class Builder {
private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build();
private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build();
private FastClusteringConfig clustering = FastClusteringConfig.builder().build();
private float minDurationOn = 0.2f;
private float minDurationOff = 0.5f;
public OfflineSpeakerDiarizationConfig build() {
return new OfflineSpeakerDiarizationConfig(this);
}
public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) {
this.segmentation = segmentation;
return this;
}
public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) {
this.embedding = embedding;
return this;
}
public Builder setClustering(FastClusteringConfig clustering) {
this.clustering = clustering;
return this;
}
public Builder setMinDurationOff(float minDurationOff) {
this.minDurationOff = minDurationOff;
return this;
}
public Builder setMinDurationOn(float minDurationOn) {
this.minDurationOn = minDurationOn;
return this;
}
}
}
@@ -0,0 +1,27 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationSegment {
private final float start;
private final float end;
private final int speaker;
public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) {
this.start = start;
this.end = end;
this.speaker = speaker;
}
public float getStart() {
return start;
}
public float getEnd() {
return end;
}
public int getSpeaker() {
return speaker;
}
}
@@ -0,0 +1,52 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerSegmentationModelConfig {
private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote;
private final int numThreads;
private final boolean debug;
private final String provider;
private OfflineSpeakerSegmentationModelConfig(Builder builder) {
this.pyannote = builder.pyannote;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build();
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public OfflineSpeakerSegmentationModelConfig build() {
return new OfflineSpeakerSegmentationModelConfig(this);
}
public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) {
this.pyannote = pyannote;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -0,0 +1,32 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerSegmentationPyannoteModelConfig {
private final String model;
private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public OfflineSpeakerSegmentationPyannoteModelConfig build() {
return new OfflineSpeakerSegmentationPyannoteModelConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
}
}
@@ -0,0 +1,77 @@
// Copyright 2022-2023 by zhaoming
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OnlineStream {
private long ptr = 0;
public OnlineStream() {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
this.ptr = 0;
}
public OnlineStream(long ptr) {
this.ptr = ptr;
}
public long getPtr() {
return ptr;
}
public void setPtr(long ptr) {
this.ptr = ptr;
}
public void acceptWaveform(float[] samples, int sampleRate) {
acceptWaveform(this.ptr, samples, sampleRate);
}
public void inputFinished() {
inputFinished(this.ptr);
}
public void setOption(String key, String value) {
setOption(this.ptr, key, value);
}
public String getOption(String key) {
return getOption(this.ptr, key);
}
public boolean hasOption(String key) {
return hasOption(this.ptr, key);
}
public void release() {
close();
}
public void close() {
// stream object must be release after used
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
@Override
protected void finalize() throws Throwable {
close();
super.finalize();
}
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
private native void inputFinished(long ptr);
private native void setOption(long ptr, String key, String value);
private native String getOption(long ptr, String key);
private native boolean hasOption(long ptr, String key);
private native void delete(long ptr);
}
@@ -0,0 +1,58 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractor {
private long ptr = 0;
public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor");
}
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public OnlineStream createStream() {
long p = createStream(ptr);
return new OnlineStream(p);
}
public boolean isReady(OnlineStream s) {
return isReady(ptr, s.getPtr());
}
public float[] compute(OnlineStream s) {
return compute(ptr, s.getPtr());
}
public int getDim() {
return dim(ptr);
}
private native void delete(long ptr);
private native long newFromFile(SpeakerEmbeddingExtractorConfig config);
private native long createStream(long ptr);
private native boolean isReady(long ptr, long streamPtr);
private native float[] compute(long ptr, long streamPtr);
private native int dim(long ptr);
}
@@ -0,0 +1,53 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractorConfig {
private final String model;
private final int numThreads;
private final boolean debug;
private final String provider;
private SpeakerEmbeddingExtractorConfig(Builder builder) {
this.model = builder.model;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private String model = "";
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public SpeakerEmbeddingExtractorConfig build() {
return new SpeakerEmbeddingExtractorConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -0,0 +1,78 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingManager {
private long ptr = 0;
public SpeakerEmbeddingManager(int dim) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = create(dim);
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public boolean add(String name, float[] embedding) {
return add(ptr, name, embedding);
}
public boolean add(String name, float[][] embedding) {
return addList(ptr, name, embedding);
}
public boolean remove(String name) {
return remove(ptr, name);
}
public String search(float[] embedding, float threshold) {
return search(ptr, embedding, threshold);
}
public boolean verify(String name, float[] embedding, float threshold) {
return verify(ptr, name, embedding, threshold);
}
public boolean contains(String name) {
return contains(ptr, name);
}
public int getNumSpeakers() {
return numSpeakers(ptr);
}
public String[] getAllSpeakerNames() {
return allSpeakerNames(ptr);
}
private native long create(int dim);
private native void delete(long ptr);
private native boolean add(long ptr, String name, float[] embedding);
private native boolean addList(long ptr, String name, float[][] embedding);
private native boolean remove(long ptr, String name);
private native String search(long ptr, float[] embedding, float threshold);
private native boolean verify(long ptr, String name, float[] embedding, float threshold);
private native boolean contains(long ptr, String name);
private native int numSpeakers(long ptr);
private native String[] allSpeakerNames(long ptr);
}
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Microphone icon for the main page's tab-bar wakeword button.
Converted from ~/Downloads/noun-microphone-1368757.svg (Noun Project
icon #1368757); base fill is overridden by tint-color in app.css. -->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="1200"
android:viewportHeight="1200">
<path
android:fillColor="#FF333333"
android:fillType="evenOdd"
android:pathData="m821.23 215.45v382.24c0 289.57-442.47 291.89-442.47 0v-382.24c0-287.26 442.47-287.26 442.47 0z" />
<path
android:fillColor="#FF333333"
android:fillType="evenOdd"
android:pathData="m543.24 963.71c-294.21-27.801-328.96-298.84-328.96-546.72 0-41.699 88.031-41.699 97.297-4.6328 2.3164 199.23 4.6328 454.05 287.26 454.05 284.94 0 287.26-254.83 287.26-454.05 6.9492-39.383 99.613-37.066 99.613 4.6328 0 247.88-34.75 518.92-333.59 546.72v139h152.89c67.18 0 69.5 97.297 0 97.297h-414.67c-64.863 0-62.547-97.297 0-97.297h152.89v-139z" />
</vector>
@@ -25,6 +25,17 @@
<item name="colorPrimary">@color/ns_primary</item>
<item name="colorPrimaryDark">@color/ns_primaryDark</item>
<item name="colorAccent">@color/ns_accent</item>
<!-- NativeScript builds the toolbar programmatically, so the
NativeScriptToolbarTheme overlay below (an inflation-time "theme"
item) never applies and the overflow icon falls back to the light
activity theme's dark textColorSecondary: black-on-dark. Tint the
overflow button itself instead; every page's bar is dark. -->
<item name="actionOverflowButtonStyle">@style/FaceclawOverflowButtonStyle</item>
</style>
<style name="FaceclawOverflowButtonStyle" parent="Widget.AppCompat.ActionButton.Overflow">
<item name="android:tint">#FFFFFF</item>
</style>
<style name="AppTheme" parent="AppThemeBase">
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Wearable Data Layer capability advertised to the watch, so the Faceclaw
watch app can find the phone that has Faceclaw installed. The watch app
advertises "faceclaw_watch" in return (see wear/). -->
<resources>
<string-array name="android_wear_capabilities">
<item>faceclaw_phone</item>
</string-array>
</resources>