Merge branch 'calculator' of https://github.com/kalanihelekunihi/faceclaw into calculator

This commit is contained in:
James Babcock committed 2026-08-26 00:49:20 -07:00
commit f3ad55ca3d
126 files changed
+22253 -58

No files matched your search

+4
View File
@@ -15,6 +15,10 @@ dependencies {
// Wearable Data Layer (MessageClient / DataClient / CapabilityClient) for
// the Wear OS watch remote; see FaceclawWearBridge and wear/.
implementation 'com.google.android.gms:play-services-wearable:19.0.0'
// On-device language identification and translation for the Microphones
// app's captions (translation models download on demand, per language).
implementation 'com.google.mlkit:language-id:17.0.6'
implementation 'com.google.mlkit:translate:17.0.3'
}
def faceclawSherpaOnnxVersion = "1.13.0"
@@ -0,0 +1,242 @@
package com.faceclaw.app;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
* container, cutting speech recordings to roughly a tenth of their WAV size.
* Runs on its own thread; the source WAV is deleted on success when requested.
*/
public class FaceclawAudioTranscoder {
private static final String TAG = "FaceclawTranscode";
private static final long CODEC_TIMEOUT_US = 10_000;
public interface Listener {
void onDone(String outputPath, long outputBytes);
void onError(String message);
}
private final String wavPath;
private final String outputPath;
private final int bitrateBps;
private final boolean deleteSourceOnSuccess;
private final Listener listener;
public FaceclawAudioTranscoder(
String wavPath,
String outputPath,
int bitrateBps,
boolean deleteSourceOnSuccess,
Listener listener) {
this.wavPath = wavPath;
this.outputPath = outputPath;
this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000;
this.deleteSourceOnSuccess = deleteSourceOnSuccess;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawTranscode");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
private void run() {
try {
transcode();
long size = new File(outputPath).length();
if (deleteSourceOnSuccess) {
new File(wavPath).delete();
}
if (listener != null) {
listener.onDone(outputPath, size);
}
} catch (Throwable t) {
Log.w(TAG, "transcode failed for " + wavPath, t);
new File(outputPath).delete();
if (listener != null) {
listener.onError(t.getMessage() == null ? t.toString() : t.getMessage());
}
}
}
private void transcode() throws IOException {
WavInfo wav = readWavHeader(wavPath);
MediaFormat format = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
MediaMuxer muxer = null;
FileInputStream input = null;
try {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
codec.start();
muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
input = new FileInputStream(wavPath);
long skipped = 0;
while (skipped < wav.dataOffset) {
long step = input.skip(wav.dataOffset - skipped);
if (step <= 0) {
throw new IOException("could not seek past WAV header");
}
skipped += step;
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
byte[] chunk = new byte[8192];
int track = -1;
boolean muxerStarted = false;
boolean inputDone = false;
boolean outputDone = false;
long remaining = wav.dataBytes;
long presentationUs = 0;
long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity()));
int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1;
if (read <= 0) {
codec.queueInputBuffer(inIndex, 0, 0, presentationUs,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
inBuf.clear();
inBuf.put(chunk, 0, read);
codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0);
remaining -= read;
presentationUs += read * 1_000_000L / bytesPerSecond;
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US);
if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
track = muxer.addTrack(codec.getOutputFormat());
muxer.start();
muxerStarted = true;
} else if (outIndex >= 0) {
if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
if (!muxerStarted) {
throw new IOException("encoder produced data before format");
}
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
outBuf.position(info.offset);
outBuf.limit(info.offset + info.size);
muxer.writeSampleData(track, outBuf, info);
}
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
if (input != null) {
try {
input.close();
} catch (IOException ignored) {
}
}
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
if (muxer != null) {
try {
muxer.stop();
} catch (Throwable ignored) {
}
muxer.release();
}
}
}
private static final class WavInfo {
int sampleRate;
int channels;
long dataOffset;
long dataBytes;
}
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
private static WavInfo readWavHeader(String path) throws IOException {
try (FileInputStream in = new FileInputStream(path)) {
byte[] head = new byte[12];
if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') {
throw new IOException("not a WAV file");
}
WavInfo info = new WavInfo();
long offset = 12;
byte[] chunkHead = new byte[8];
while (in.read(chunkHead) == 8) {
ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN);
int id = b.getInt();
long size = b.getInt() & 0xffffffffL;
offset += 8;
if (id == 0x20746d66) { // "fmt "
byte[] fmt = new byte[(int) Math.min(size, 16)];
if (in.read(fmt) != fmt.length) {
throw new IOException("truncated fmt chunk");
}
ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN);
int audioFormat = f.getShort();
info.channels = f.getShort();
info.sampleRate = f.getInt();
f.getInt();
f.getShort();
int bits = f.getShort();
if (audioFormat != 1 || bits != 16) {
throw new IOException("only 16-bit PCM WAV is supported");
}
long skip = size - fmt.length;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
} else if (id == 0x61746164) { // "data"
info.dataOffset = offset;
// A zero size means the header was never patched (crash
// mid-recording); recover the length from the file size.
info.dataBytes = size > 0 ? size : new File(path).length() - offset;
break;
} else {
long skip = size;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
}
}
if (info.sampleRate == 0 || info.dataOffset == 0) {
throw new IOException("missing fmt or data chunk");
}
return info;
}
}
}
@@ -60,6 +60,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawCompassListener> compassListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private volatile String lastFirmwareCapabilities = "";
private volatile Thread workerThread;
private volatile boolean running;
private volatile boolean userDisconnectRequested;
@@ -535,6 +538,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
public String getFirmwareCapabilities() {
return lastFirmwareCapabilities;
}
public void addMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.add(listener);
}
}
public void removeMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.remove(listener);
}
}
private void emitMicStatus(byte[] body, String address) {
if (micStatusListeners.isEmpty()) {
return;
}
String arm = address.equalsIgnoreCase(leftAddress) ? "L"
: address.equalsIgnoreCase(rightAddress) ? "R" : "?";
byte[] copy = java.util.Arrays.copyOf(body, body.length);
mainHandler.post(() -> {
for (FaceclawMicStatusListener micListener : micStatusListeners) {
try {
micListener.onMicStatus(copy, arm);
} catch (Throwable t) {
Log.w(TAG, "mic status listener failed", t);
}
}
});
}
/**
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
* field-103 write to both temples, or to a single one. Fire-and-forget:
* the firmware answers with a field-104 status notify per temple, which
* arrives through addMicStatusListener.
*/
public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) {
if (record == null || record.length < 4) {
return;
}
synchronized (lock) {
if (!running || !sessionReady) {
logLine("skip mic control (" + label + "); session not ready");
return;
}
if (rightTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false));
}
if (leftTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true));
}
logLine("queue mic control " + label);
}
interruptibleSleep.interrupt();
}
/**
* Forward render-characteristic audio packets to the listener WITHOUT
* sending the stock EvenHub audio-control enable. Used for the CFW
* mic_control streaming path, where capture is armed through settings
* field 103 and the temples emit 'SM' frames on the same characteristic
* that stock mono LC3 uses. Returns false when no session is up.
*/
public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) {
if (listener == null) {
throw new IllegalArgumentException("listener is required");
}
synchronized (lock) {
if (!running || !sessionReady || shutdownRequested) {
logLine("skip G2 audio forwarding; session not ready");
return false;
}
audioPacketListener = listener;
audioCaptureActive = true;
logLine("G2 audio forwarding enabled");
}
return true;
}
public void stopG2AudioForwarding() {
synchronized (lock) {
audioPacketListener = null;
audioCaptureActive = false;
logLine("G2 audio forwarding disabled");
}
}
public void addCompassListener(FaceclawCompassListener listener) {
if (listener != null) {
compassListeners.add(listener);
@@ -1230,6 +1325,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
0
);
}
if (!faceclawWakeNotification
&& frame.ok
&& frame.sid == BleProtocol.SID_UI_SETTING) {
// CFW mic status (field 104) rides both standalone pushes and
// settings read acks, from each temple on its own link.
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
if (micStatus != null) {
emitMicStatus(micStatus, address);
}
}
if (!faceclawWakeNotification
&& decodedWearState < 0
&& frame.ok
@@ -2502,6 +2607,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
if (firmwareInfo != null) {
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
&& hasCapability(firmwareInfo.capabilities, "texstr14");
@@ -0,0 +1,446 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
import com.k2fsa.sherpa.onnx.OfflineStream;
import java.io.File;
import java.util.ArrayDeque;
import java.util.Arrays;
/**
* Continuous captioning over the decoded mic PCM: segments speech into
* utterances with an adaptive energy gate, transcribes each utterance with
* the on-device Moonshine model, and attaches a speaker voice-print per
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
* and any beam-direction gating), so this engine has no BLE dependencies.
*
* Utterance boundaries are measured on the sample clock, not the wall clock:
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
* audio heard. The TS side converts startMs/endMs to wall-clock times using
* the engine start timestamp.
*/
public class FaceclawCaptionEngine {
private static final String TAG = "FaceclawCaptions";
private static final int SAMPLE_RATE = 16000;
private static final int FEATURE_DIM = 80;
private static final int MAX_QUEUE_PACKETS = 200;
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
private static final double ONSET_FACTOR = 3.0;
private static final double RELEASE_FACTOR = 1.8;
private static final double MIN_RMS = 220.0;
private static final double NOISE_EMA_ALPHA = 0.05;
private static final int PRE_ROLL_MS = 400;
private static final int DEFAULT_SILENCE_MS = 800;
private static final int MIN_UTTERANCE_MS = 350;
private static final int MAX_UTTERANCE_MS = 15000;
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
// segments cut at the quietest window (mirrors FaceclawVoiceController).
private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2;
private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000;
private static final float NORMALIZE_TARGET_PEAK = 0.9f;
private static final float NORMALIZE_MAX_GAIN = 30f;
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private final Object queueLock = new Object();
private final ArrayDeque<byte[]> queue = new ArrayDeque<>();
private volatile FaceclawCaptionEngineListener listener;
private volatile String asrModelDir;
private volatile String speakerModelPath;
private volatile int silenceMs = DEFAULT_SILENCE_MS;
private Thread workerThread;
private volatile boolean started;
private OfflineRecognizer recognizer;
private FaceclawSpeakerId speakerId;
// Segmentation state (worker thread only).
private long totalSamples;
private double noiseFloor;
private boolean inUtterance;
private long utteranceStartSample;
private long silenceRunSamples;
private double utterancePeakRms;
private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)];
private int utteranceLength;
private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000];
private int preRollLength;
public void setListener(FaceclawCaptionEngineListener listener) {
this.listener = listener;
}
/** Directory holding the Moonshine model files, or null to disable ASR. */
public void setAsrModelDir(String dir) {
this.asrModelDir = dir;
}
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
public void setSpeakerModelPath(String path) {
this.speakerModelPath = path;
}
public void setSilenceMs(int ms) {
this.silenceMs = Math.max(200, Math.min(3000, ms));
}
public void start() {
synchronized (lock) {
if (started) {
return;
}
started = true;
workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine");
workerThread.start();
}
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
if (!started) {
return;
}
started = false;
threadToJoin = workerThread;
workerThread = null;
}
synchronized (queueLock) {
queueLock.notifyAll();
}
if (threadToJoin != null) {
threadToJoin.interrupt();
if (Thread.currentThread() != threadToJoin) {
try {
threadToJoin.join(2000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
public void acceptPcm(byte[] pcm16le) {
if (!started || pcm16le == null || pcm16le.length < 2) {
return;
}
synchronized (queueLock) {
if (queue.size() >= MAX_QUEUE_PACKETS) {
queue.removeFirst();
}
queue.addLast(pcm16le);
queueLock.notifyAll();
}
}
private void runLoop() {
try {
loadModels();
resetSegmentation();
while (started && !Thread.currentThread().isInterrupted()) {
byte[] chunk = takeChunk();
if (chunk == null) {
continue;
}
processChunk(chunk);
}
// Flush a trailing utterance so its text isn't lost on stop.
if (inUtterance && utteranceLength > 0) {
finalizeUtterance();
}
} catch (Throwable t) {
Log.e(TAG, "caption engine failed", t);
emitStatus("Captions failed: " + t.getMessage());
} finally {
releaseModels();
}
}
private void loadModels() {
String modelDir = asrModelDir;
if (modelDir != null && new File(modelDir, "tokens.txt").exists()) {
emitStatus("Loading caption model...");
recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(OfflineModelConfig.builder()
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setNumThreads(1)
.build())
.build());
} else {
recognizer = null;
}
String embedModel = speakerModelPath;
if (embedModel != null && new File(embedModel).exists()) {
speakerId = new FaceclawSpeakerId(embedModel);
speakerId.ensureLoaded();
} else {
speakerId = null;
}
emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)...");
}
private void releaseModels() {
if (recognizer != null) {
recognizer.release();
recognizer = null;
}
if (speakerId != null) {
speakerId.close();
speakerId = null;
}
}
private byte[] takeChunk() {
synchronized (queueLock) {
while (started && queue.isEmpty()) {
try {
queueLock.wait(250);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
}
}
return queue.pollFirst();
}
}
private void resetSegmentation() {
totalSamples = 0;
noiseFloor = 0;
inUtterance = false;
utteranceLength = 0;
preRollLength = 0;
silenceRunSamples = 0;
utterancePeakRms = 0;
}
private void processChunk(byte[] chunk) {
int count = chunk.length / 2;
short[] pcm = new short[count];
double sumSquares = 0;
for (int i = 0; i < count; i++) {
short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8));
pcm[i] = s;
sumSquares += (double) s * s;
}
double rms = Math.sqrt(sumSquares / Math.max(1, count));
totalSamples += count;
if (!inUtterance) {
// Track the noise floor only while idle so speech doesn't raise it.
noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA;
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms >= threshold * ONSET_FACTOR) {
inUtterance = true;
utteranceLength = 0;
silenceRunSamples = 0;
utterancePeakRms = rms;
appendUtterance(preRoll, preRollLength);
utteranceStartSample = Math.max(0, totalSamples - count - preRollLength);
appendUtterance(pcm, count);
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE);
} else {
appendPreRoll(pcm, count);
}
return;
}
appendUtterance(pcm, count);
utterancePeakRms = Math.max(utterancePeakRms, rms);
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms < threshold * RELEASE_FACTOR) {
silenceRunSamples += count;
} else {
silenceRunSamples = 0;
}
long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE;
boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs;
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
finalizeUtterance();
inUtterance = false;
preRollLength = 0;
}
}
private void appendPreRoll(short[] pcm, int count) {
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
if (count >= preRoll.length) {
System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length);
preRollLength = preRoll.length;
return;
}
int keep = Math.min(preRollLength, preRoll.length - count);
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep);
System.arraycopy(pcm, 0, preRoll, keep, count);
preRollLength = keep + count;
}
private void appendUtterance(short[] pcm, int count) {
int room = utterance.length - utteranceLength;
int copied = Math.min(room, count);
if (copied > 0) {
System.arraycopy(pcm, 0, utterance, utteranceLength, copied);
utteranceLength += copied;
}
}
private void finalizeUtterance() {
int length = utteranceLength;
long startMs = utteranceStartSample * 1000 / SAMPLE_RATE;
long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE;
if (endMs - startMs < MIN_UTTERANCE_MS) {
return;
}
String text = recognizeUtterance(length);
float[] embedding = embedUtterance(length);
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms);
}
private String recognizeUtterance(int length) {
if (recognizer == null || length <= 0) {
return "";
}
StringBuilder joined = new StringBuilder();
int offset = 0;
while (offset < length) {
int remaining = length - offset;
int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES);
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
segment = findQuietCut(offset, segment);
}
String part = recognizeRange(offset, segment);
if (part.length() > 0) {
if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) {
joined.append(' ');
}
joined.append(part);
}
offset += segment;
}
return joined.toString().trim();
}
/**
* End a decode segment at the center of the quietest window near its end
* so the cut lands between words (same approach as the PTT controller).
*/
private int findQuietCut(int offset, int segment) {
int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES);
int win = CUT_WINDOW_SAMPLES;
if (segment - searchStart <= win) {
return segment;
}
double sum = 0;
for (int i = searchStart; i < searchStart + win; i++) {
double s = utterance[offset + i];
sum += s * s;
}
double best = sum;
int bestStart = searchStart;
for (int start = searchStart + 1; start + win <= segment; start++) {
double dropped = utterance[offset + start - 1];
double added = utterance[offset + start + win - 1];
sum += added * added - dropped * dropped;
if (sum < best) {
best = sum;
bestStart = start;
}
}
return bestStart + win / 2;
}
private String recognizeRange(int offset, int count) {
float[] samples = new float[count];
float peak = 0f;
for (int i = 0; i < count; i++) {
float v = utterance[offset + i] / 32768.0f;
samples[i] = v;
float a = Math.abs(v);
if (a > peak) {
peak = a;
}
}
if (peak > 0f) {
float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN);
if (gain > 1f) {
for (int i = 0; i < count; i++) {
samples[i] *= gain;
}
}
}
OfflineStream stream = recognizer.createStream();
try {
stream.acceptWaveform(samples, SAMPLE_RATE);
recognizer.decode(stream);
OfflineRecognizerResult result = recognizer.getResult(stream);
String raw = result == null ? "" : result.getText();
return raw == null ? "" : raw.trim();
} finally {
stream.release();
}
}
private float[] embedUtterance(int length) {
FaceclawSpeakerId currentSpeakerId = speakerId;
if (currentSpeakerId == null || length <= 0) {
return null;
}
int count = Math.min(length, EMBED_MAX_SAMPLES);
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
short s = utterance[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
return currentSpeakerId.embed(le, SAMPLE_RATE);
}
private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length);
mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms));
}
private void emitSpeechStart(long startMs) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeechStart(startMs));
}
private void emitStatus(String status) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onStatus(status));
}
}
@@ -0,0 +1,17 @@
package com.faceclaw.app;
public interface FaceclawCaptionEngineListener {
/**
* One finished utterance. text may be empty when no ASR model is loaded
* or recognition produced nothing; embedding is the L2-normalized
* voice-print or null when the speaker model is unavailable or the
* utterance was too short. startMs/endMs are on the engine's sample
* clock (milliseconds of audio since start()).
*/
void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms);
/** A speech onset was detected; the utterance is now being collected. */
void onSpeechStart(long startMs);
void onStatus(String status);
}
@@ -0,0 +1,778 @@
package com.faceclaw.app;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import android.util.Base64;
import android.util.Log;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
/**
* Persistent store for caption sessions, transcript segments, and speaker
* voice profiles, backing the Microphones app and its phone-side review UI.
* All complex values cross the JS bridge as JSON strings; voice-print
* embeddings are float32 LE blobs, exposed as base64.
*/
public class FaceclawConversationStore extends SQLiteOpenHelper {
private static final String TAG = "FaceclawConvStore";
private static final String DB_NAME = "faceclaw-conversations.db";
private static final int DB_VERSION = 2;
private static FaceclawConversationStore instance;
public static synchronized FaceclawConversationStore getInstance(Context context) {
if (instance == null) {
instance = new FaceclawConversationStore(context.getApplicationContext());
}
return instance;
}
private FaceclawConversationStore(Context context) {
super(context, DB_NAME, null, DB_VERSION);
}
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL("CREATE TABLE speakers ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "name TEXT NOT NULL,"
+ "color TEXT NOT NULL DEFAULT '#4FC3F7',"
+ "is_wearer INTEGER NOT NULL DEFAULT 0,"
+ "embedding BLOB,"
+ "embedding_count INTEGER NOT NULL DEFAULT 0,"
+ "created_at INTEGER NOT NULL,"
+ "last_heard_at INTEGER,"
+ "tag TEXT,"
+ "last_recap TEXT,"
+ "action_items TEXT,"
+ "facts TEXT,"
+ "insights_updated_at INTEGER,"
+ "insights_session_id INTEGER)");
db.execSQL("CREATE TABLE sessions ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER,"
+ "title TEXT,"
+ "audio_path TEXT,"
+ "audio_codec TEXT,"
+ "avg_sentiment REAL,"
+ "segment_count INTEGER NOT NULL DEFAULT 0)");
db.execSQL("CREATE TABLE segments ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "session_id INTEGER NOT NULL,"
+ "speaker_id INTEGER,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER NOT NULL,"
+ "audio_offset_ms INTEGER,"
+ "text TEXT NOT NULL,"
+ "lang TEXT,"
+ "translation TEXT,"
+ "translation_lang TEXT,"
+ "sentiment REAL,"
+ "emotion TEXT,"
+ "search_meta TEXT,"
+ "angle INTEGER,"
+ "embedding BLOB)");
db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)");
db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)");
db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)");
db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)");
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
if (oldVersion < 2) {
// v2: per-speaker conversation insights (recap of the most recent
// conversation, its action items, and accumulated inferred facts).
db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER");
}
}
// ---- sessions ----
public long startSession(long startedAtMs, String title) {
ContentValues values = new ContentValues();
values.put("started_at", startedAtMs);
values.put("title", title == null || title.isEmpty() ? null : title);
return getWritableDatabase().insert("sessions", null, values);
}
public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) {
ContentValues values = new ContentValues();
values.put("ended_at", endedAtMs);
if (audioPath != null && !audioPath.isEmpty()) {
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
}
values.put("avg_sentiment", avgSentiment);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public void setSessionAudio(long sessionId, String audioPath, String audioCodec) {
ContentValues values = new ContentValues();
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public String getSessionAudioPath(long sessionId) {
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) {
if (c.moveToFirst()) {
return c.isNull(0) ? "" : c.getString(0);
}
}
return "";
}
/**
* Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs,
* speakerId, emotion (matches any segment emotion in the session), query
* (LIKE over title and segment text/search metadata), limit.
*/
public String querySessions(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec,"
+ " s.avg_sentiment, s.segment_count FROM sessions s");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query");
if (joinSegments) {
sql.append(" LEFT JOIN segments g ON g.session_id = s.id");
}
if (filter.has("sinceMs")) {
where.add("s.started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("s.started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (filter.has("speakerId")) {
where.add("g.speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("emotion")) {
where.add("g.emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("query")) {
String like = "%" + filter.getString("query") + "%";
where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY s.started_at DESC");
int limit = filter.optInt("limit", 200);
sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
long id = c.getLong(0);
row.put("id", id);
row.put("startedAt", c.getLong(1));
row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2));
row.put("title", c.isNull(3) ? "" : c.getString(3));
row.put("audioPath", c.isNull(4) ? "" : c.getString(4));
row.put("audioCodec", c.isNull(5) ? "" : c.getString(5));
row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6));
row.put("segmentCount", c.getLong(7));
row.put("speakers", sessionSpeakersJson(id));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "querySessions failed", e);
return "[]";
}
}
private JSONArray sessionSpeakersJson(long sessionId) throws JSONException {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT DISTINCT p.id, p.name, p.color FROM segments g"
+ " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
out.put(row);
}
}
return out;
}
// ---- segments ----
/**
* Insert one transcript segment. json fields: sessionId, speakerId?,
* startedAt, endedAt, audioOffsetMs?, text, lang?, translation?,
* translationLang?, sentiment?, emotion?, searchMeta?, angle?,
* embeddingBase64?. Returns the new row id, or -1.
*/
public long insertSegment(String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
long sessionId = s.getLong("sessionId");
values.put("session_id", sessionId);
if (s.has("speakerId")) {
values.put("speaker_id", s.getLong("speakerId"));
}
values.put("started_at", s.getLong("startedAt"));
values.put("ended_at", s.getLong("endedAt"));
if (s.has("audioOffsetMs")) {
values.put("audio_offset_ms", s.getLong("audioOffsetMs"));
}
values.put("text", s.getString("text"));
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (s.has("angle")) {
values.put("angle", s.getInt("angle"));
}
if (s.has("embeddingBase64")) {
values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP));
}
SQLiteDatabase db = getWritableDatabase();
long id = db.insert("segments", null, values);
db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?",
new Object[]{sessionId});
return id;
} catch (Throwable t) {
Log.w(TAG, "insertSegment failed", t);
return -1;
}
}
private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column)
throws JSONException {
if (s.has(jsonKey) && !s.isNull(jsonKey)) {
values.put(column, s.getString(jsonKey));
}
}
/** Update mutable fields of a segment (same keys as insertSegment). */
public void updateSegment(long segmentId, String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
if (s.has("speakerId")) {
if (s.isNull("speakerId")) {
values.putNull("speaker_id");
} else {
values.put("speaker_id", s.getLong("speakerId"));
}
}
if (s.has("text")) {
values.put("text", s.getString("text"));
}
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (values.size() > 0) {
getWritableDatabase().update("segments", values, "id=?",
new String[]{String.valueOf(segmentId)});
}
} catch (Throwable t) {
Log.w(TAG, "updateSegment failed", t);
}
}
public String querySegments(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle"
+ " FROM segments WHERE session_id=? ORDER BY started_at ASC",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
out.put(segmentRowJson(c));
}
} catch (JSONException e) {
Log.w(TAG, "querySegments failed", e);
}
return out.toString();
}
private JSONObject segmentRowJson(Cursor c) throws JSONException {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1));
row.put("startedAt", c.getLong(2));
row.put("endedAt", c.getLong(3));
row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4));
row.put("text", c.getString(5));
row.put("lang", c.isNull(6) ? "" : c.getString(6));
row.put("translation", c.isNull(7) ? "" : c.getString(7));
row.put("translationLang", c.isNull(8) ? "" : c.getString(8));
row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9));
row.put("emotion", c.isNull(10) ? "" : c.getString(10));
row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11));
return row;
}
/**
* Search segments across sessions. filterJson: query?, emotion?,
* speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and
* the sentiment search metadata.
*/
public String searchSegments(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle, session_id"
+ " FROM segments");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
if (filter.has("query") && !filter.getString("query").isEmpty()) {
String like = "%" + filter.getString("query") + "%";
where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (filter.has("emotion")) {
where.add("emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("speakerId")) {
where.add("speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("sinceMs")) {
where.add("started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY started_at DESC LIMIT ")
.append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = segmentRowJson(c);
row.put("sessionId", c.getLong(12));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "searchSegments failed", e);
return "[]";
}
}
/** Per-segment voice-print embeddings of a session (for re-diarization). */
public String querySegmentEmbeddings(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySegmentEmbeddings failed", e);
}
return out.toString();
}
// ---- speakers ----
public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) {
ContentValues values = new ContentValues();
values.put("name", name);
values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color);
values.put("is_wearer", isWearer ? 1 : 0);
values.put("created_at", System.currentTimeMillis());
values.put("last_heard_at", System.currentTimeMillis());
if (embeddingBase64 != null && !embeddingBase64.isEmpty()) {
values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP));
values.put("embedding_count", 1);
}
return getWritableDatabase().insert("speakers", null, values);
}
public String querySpeakers() {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count,"
+ " p.last_heard_at, p.tag,"
+ " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id,"
+ " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments"
+ " FROM speakers p ORDER BY p.last_heard_at DESC", null)) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
row.put("isWearer", c.getInt(3) != 0);
byte[] blob = c.isNull(4) ? null : c.getBlob(4);
row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP));
row.put("embeddingCount", c.getInt(5));
row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6));
row.put("tag", c.isNull(7) ? "" : c.getString(7));
row.put("lastRecap", c.isNull(8) ? "" : c.getString(8));
row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9));
row.put("factsJson", c.isNull(10) ? "" : c.getString(10));
row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11));
row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12));
row.put("segmentCount", c.getLong(13));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySpeakers failed", e);
}
return out.toString();
}
public void renameSpeaker(long speakerId, String name) {
ContentValues values = new ContentValues();
values.put("name", name);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerColor(long speakerId, String color) {
ContentValues values = new ContentValues();
values.put("color", color);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerTag(long speakerId, String tag) {
ContentValues values = new ContentValues();
values.put("tag", tag);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Store LLM-derived conversation insights for a speaker: a recap of their
* most recent conversation, its open action items (JSON string array), and
* the accumulated inferred facts about them (JSON string array).
*/
public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson,
String factsJson, long sessionId) {
ContentValues values = new ContentValues();
values.put("last_recap", recap == null ? "" : recap);
values.put("action_items", actionItemsJson == null ? "" : actionItemsJson);
values.put("facts", factsJson == null ? "" : factsJson);
values.put("insights_updated_at", System.currentTimeMillis());
values.put("insights_session_id", sessionId);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerWearer(long speakerId, boolean isWearer) {
SQLiteDatabase db = getWritableDatabase();
if (isWearer) {
// Exactly one wearer profile at a time.
db.execSQL("UPDATE speakers SET is_wearer = 0");
}
ContentValues values = new ContentValues();
values.put("is_wearer", isWearer ? 1 : 0);
db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Running-mean centroid update for a speaker's voice-print, capped so a
* long session cannot drown the enrolled voice.
*/
public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) {
try {
byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP);
float[] update = blobToFloats(incoming);
if (update == null) {
return;
}
float[] current = null;
int count = 0;
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT embedding, embedding_count FROM speakers WHERE id=?",
new String[]{String.valueOf(speakerId)})) {
if (c.moveToFirst()) {
current = c.isNull(0) ? null : blobToFloats(c.getBlob(0));
count = c.getInt(1);
}
}
float[] merged;
if (current == null || current.length != update.length) {
merged = update;
count = 1;
} else {
int effective = Math.min(count, Math.max(1, maxCount));
merged = new float[current.length];
for (int i = 0; i < merged.length; i++) {
merged[i] = (current[i] * effective + update[i]) / (effective + 1);
}
l2NormalizeInPlace(merged);
count = count + 1;
}
ContentValues values = new ContentValues();
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", count);
values.put("last_heard_at", System.currentTimeMillis());
getWritableDatabase().update("speakers", values, "id=?",
new String[]{String.valueOf(speakerId)});
} catch (Throwable t) {
Log.w(TAG, "updateSpeakerEmbedding failed", t);
}
}
/** Move every segment from one speaker onto another and drop the source. */
public void mergeSpeakers(long fromId, long intoId) {
if (fromId == intoId) {
return;
}
SQLiteDatabase db = getWritableDatabase();
db.beginTransaction();
try {
db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?",
new Object[]{intoId, fromId});
// Blend the two centroids weighted by their sample counts.
float[] from = null;
float[] into = null;
int fromCount = 0;
int intoCount = 0;
boolean fromWearer = false;
try (Cursor c = db.rawQuery(
"SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)",
new String[]{String.valueOf(fromId), String.valueOf(intoId)})) {
while (c.moveToNext()) {
long id = c.getLong(0);
float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1));
int count = c.getInt(2);
if (id == fromId) {
from = embedding;
fromCount = count;
fromWearer = c.getInt(3) != 0;
} else {
into = embedding;
intoCount = count;
}
}
}
float[] merged = into;
if (from != null && into != null && from.length == into.length) {
merged = new float[into.length];
int total = Math.max(1, fromCount + intoCount);
for (int i = 0; i < merged.length; i++) {
merged[i] = (into[i] * intoCount + from[i] * fromCount) / total;
}
l2NormalizeInPlace(merged);
} else if (into == null) {
merged = from;
}
ContentValues values = new ContentValues();
if (merged != null) {
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", fromCount + intoCount);
}
if (fromWearer) {
values.put("is_wearer", 1);
}
values.put("last_heard_at", System.currentTimeMillis());
// Keep the target's conversation insights; adopt the source's only
// where the target has none (e.g. merging a rich profile into a
// freshly auto-created one).
try (Cursor c = db.rawQuery(
"SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id"
+ " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END",
new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) {
boolean targetHasInsights = false;
while (c.moveToNext()) {
boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty();
if (c.getPosition() == 0) {
targetHasInsights = hasInsights;
} else if (!targetHasInsights && hasInsights) {
values.put("last_recap", c.getString(0));
values.put("action_items", c.isNull(1) ? "" : c.getString(1));
values.put("facts", c.isNull(2) ? "" : c.getString(2));
values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3));
values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4));
}
}
}
db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)});
db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)});
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
}
public void reassignSegmentSpeaker(long segmentId, long speakerId) {
ContentValues values = new ContentValues();
values.put("speaker_id", speakerId);
getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)});
}
/** Reassign every segment of a speaker within one session (split support). */
public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) {
getWritableDatabase().execSQL(
"UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?",
new Object[]{toSpeakerId, sessionId, fromSpeakerId});
}
public void deleteSpeaker(long speakerId) {
SQLiteDatabase db = getWritableDatabase();
db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId});
db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)});
}
// ---- retention ----
/**
* Delete caption sessions older than the cutoff (0 disables), and strip
* recordings older than their own cutoff while keeping the transcript.
* Returns a JSON summary of what was removed.
*/
public String applyRetention(long captionCutoffMs, long recordingCutoffMs) {
SQLiteDatabase db = getWritableDatabase();
int sessionsDeleted = 0;
int recordingsDeleted = 0;
if (recordingCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(recordingCutoffMs)})) {
while (c.moveToNext()) {
if (!c.isNull(1)) {
paths.add(c.getString(1));
}
}
}
for (String path : paths) {
if (new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?",
new Object[]{recordingCutoffMs});
}
if (captionCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(captionCutoffMs)})) {
while (c.moveToNext()) {
paths.add(c.getString(0));
}
}
for (String path : paths) {
if (path != null && new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)",
new Object[]{captionCutoffMs});
Cursor count = db.rawQuery("SELECT changes()", null);
count.close();
db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs});
try (Cursor c = db.rawQuery("SELECT changes()", null)) {
if (c.moveToFirst()) {
sessionsDeleted = c.getInt(0);
}
}
}
try {
JSONObject summary = new JSONObject();
summary.put("sessionsDeleted", sessionsDeleted);
summary.put("recordingsDeleted", recordingsDeleted);
return summary.toString();
} catch (JSONException e) {
return "{}";
}
}
public void deleteSession(long sessionId) {
SQLiteDatabase db = getWritableDatabase();
String path = getSessionAudioPath(sessionId);
if (!path.isEmpty()) {
new File(path).delete();
}
db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)});
db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)});
}
// ---- blob helpers ----
private static float[] blobToFloats(byte[] blob) {
if (blob == null || blob.length < 4 || blob.length % 4 != 0) {
return null;
}
ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
float[] out = new float[blob.length / 4];
for (int i = 0; i < out.length; i++) {
out[i] = buffer.getFloat();
}
return out;
}
private static byte[] floatsToBlob(float[] values) {
ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN);
for (float v : values) {
buffer.putFloat(v);
}
return buffer.array();
}
private static void l2NormalizeInPlace(float[] v) {
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
}
}
@@ -37,9 +37,6 @@ import java.util.Set;
* matching are decided in TypeScript (app/g2/even-advertisement.ts,
* app/g2/pairing-candidates.ts) where they can be unit-tested; this filter
* must stay a superset of the TS classifier's admission.
*
* Ported from the discovery path of SybilSight's Bluetooth SDK (G2.didDiscover
* and R1's advertisement snapshot), reduced to what Android exposes.
*/
@SuppressLint("MissingPermission")
public class FaceclawDeviceDiscovery {
@@ -296,7 +293,7 @@ public class FaceclawDeviceDiscovery {
/**
* Android strips the 2-byte company identifier from each manufacturer
* record and keys the SparseArray by it. Put the bytes back (little-endian)
* so TypeScript sees the same "ER"+… layout the iOS SDK and HCI captures
* so TypeScript sees the same "ER"+… layout the HCI captures
* document. Prefers Even's own id; otherwise returns the first record so an
* unexpected company id still reaches the diagnostics log.
*/
@@ -0,0 +1,262 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FastClusteringConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* Offline re-diarization of a stored conversation recording (16-bit mono
* WAV): pyannote segmentation plus the shared speaker-embedding model with
* clustering, via sherpa-onnx. Runs on its own thread; results are handed to
* TS as JSON turns for reconciliation against the transcript segments.
*/
public class FaceclawDiarizer {
private static final String TAG = "FaceclawDiarizer";
public interface Listener {
/** progress in [0, 1]. */
void onProgress(double progress);
/** turns: [{startMs, endMs, cluster}], sorted by start time. */
void onDone(String turnsJson);
void onError(String message);
}
private final String segmentationModelPath;
private final String embeddingModelPath;
private final String wavPath;
private final int numClusters;
private final float threshold;
private final Listener listener;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private volatile boolean cancelled;
public FaceclawDiarizer(
String segmentationModelPath,
String embeddingModelPath,
String wavPath,
int numClusters,
float threshold,
Listener listener) {
this.segmentationModelPath = segmentationModelPath;
this.embeddingModelPath = embeddingModelPath;
this.wavPath = wavPath;
this.numClusters = numClusters;
this.threshold = threshold <= 0 ? 0.5f : threshold;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawDiarizer");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
public void cancel() {
cancelled = true;
}
private void run() {
OfflineSpeakerDiarization diarization = null;
try {
float[] samples = wavPath.endsWith(".wav")
? readWavMono16(wavPath)
: decodeCompressedMono(wavPath);
OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder()
.setSegmentation(OfflineSpeakerSegmentationModelConfig.builder()
.setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder()
.setModel(segmentationModelPath)
.build())
.setNumThreads(1)
.build())
.setEmbedding(SpeakerEmbeddingExtractorConfig.builder()
.setModel(embeddingModelPath)
.setNumThreads(1)
.build())
.setClustering(FastClusteringConfig.builder()
.setNumClusters(numClusters > 0 ? numClusters : -1)
.setThreshold(threshold)
.build())
.setMinDurationOn(0.3f)
.setMinDurationOff(0.5f)
.build();
diarization = new OfflineSpeakerDiarization(config);
OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback(
samples,
(numProcessedChunks, numTotalChunks, arg) -> {
emitProgress(numTotalChunks <= 0 ? 0
: (double) numProcessedChunks / numTotalChunks);
return cancelled ? 1 : 0;
});
JSONArray turns = new JSONArray();
if (segments != null) {
for (OfflineSpeakerDiarizationSegment segment : segments) {
JSONObject turn = new JSONObject();
turn.put("startMs", Math.round(segment.getStart() * 1000));
turn.put("endMs", Math.round(segment.getEnd() * 1000));
turn.put("cluster", segment.getSpeaker());
turns.put(turn);
}
}
String result = turns.toString();
mainHandler.post(() -> listener.onDone(result));
} catch (Throwable t) {
Log.w(TAG, "diarization failed for " + wavPath, t);
String message = t.getMessage() == null ? t.toString() : t.getMessage();
mainHandler.post(() -> listener.onError(message));
} finally {
if (diarization != null) {
diarization.release();
}
}
}
private void emitProgress(double progress) {
mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress))));
}
/**
* Decode a compressed recording (M4A/AAC from the transcode sweep) into
* [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written
* as 16 kHz mono, so no resampling is needed.
*/
private static float[] decodeCompressedMono(String path) throws IOException {
android.media.MediaExtractor extractor = new android.media.MediaExtractor();
android.media.MediaCodec codec = null;
java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream();
try {
extractor.setDataSource(path);
int trackIndex = -1;
android.media.MediaFormat format = null;
for (int i = 0; i < extractor.getTrackCount(); i++) {
android.media.MediaFormat candidate = extractor.getTrackFormat(i);
String mime = candidate.getString(android.media.MediaFormat.KEY_MIME);
if (mime != null && mime.startsWith("audio/")) {
trackIndex = i;
format = candidate;
break;
}
}
if (trackIndex < 0 || format == null) {
throw new IOException("no audio track in " + path);
}
extractor.selectTrack(trackIndex);
codec = android.media.MediaCodec.createDecoderByType(
format.getString(android.media.MediaFormat.KEY_MIME));
codec.configure(format, null, null, 0);
codec.start();
android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo();
boolean inputDone = false;
boolean outputDone = false;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(10_000);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int size = extractor.readSampleData(inBuf, 0);
if (size < 0) {
codec.queueInputBuffer(inIndex, 0, 0, 0,
android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0);
extractor.advance();
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, 10_000);
if (outIndex >= 0) {
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
byte[] chunk = new byte[info.size];
outBuf.position(info.offset);
outBuf.get(chunk);
pcm.write(chunk);
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
extractor.release();
if (codec != null) {
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
}
}
byte[] bytes = pcm.toByteArray();
int count = bytes.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f;
}
return samples;
}
/** Load a 16-bit mono PCM WAV into [-1, 1] floats. */
private static float[] readWavMono16(String path) throws IOException {
File file = new File(path);
long fileLength = file.length();
try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
byte[] header = new byte[44];
in.readFully(header);
if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') {
throw new IOException("not a WAV file");
}
int channels = (header[22] & 0xff) | (header[23] << 8);
int bits = (header[34] & 0xff) | (header[35] << 8);
if (channels != 1 || bits != 16) {
throw new IOException("only 16-bit mono WAV is supported");
}
long dataBytes = fileLength - 44;
int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE);
float[] samples = new float[count];
byte[] chunk = new byte[65536];
int sampleIndex = 0;
int carried = 0;
byte carriedByte = 0;
int read;
while (sampleIndex < count && (read = in.read(chunk)) > 0) {
int offset = 0;
if (carried == 1) {
samples[sampleIndex++] =
((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f;
offset = 1;
carried = 0;
}
int pairs = (read - offset) / 2;
for (int i = 0; i < pairs && sampleIndex < count; i++) {
int base = offset + i * 2;
samples[sampleIndex++] =
((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f;
}
if (((read - offset) & 1) == 1) {
carried = 1;
carriedByte = chunk[read - 1];
}
}
return samples;
}
}
}
@@ -12,6 +12,12 @@ public class FaceclawLc3Decoder {
public static final int COUNTER_OFFSET = 204;
public static final int SAMPLES_PER_FRAME = 160;
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
// glasses DSP appends two signed 16-bit LE values computed on the raw
// stereo capture before the mono downmix — a signal-strength ratio and the
// asin-derived direction-of-arrival in degrees — then the packet counter.
public static final int SSR_OFFSET = 200;
public static final int ANGLE_OFFSET = 202;
static {
System.loadLibrary("faceclaw_lc3");
@@ -23,6 +29,8 @@ public class FaceclawLc3Decoder {
private long duplicatePackets;
private long missingPackets;
private long decodeErrors;
private int lastSsr;
private int lastAngleDegrees;
public FaceclawLc3Decoder() {
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
@@ -64,9 +72,21 @@ public class FaceclawLc3Decoder {
}
realPackets++;
lastCounter = counter;
lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8));
lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8));
return decoded;
}
/** Firmware signal-strength ratio from the most recent decoded packet. */
public synchronized int getLastSsr() {
return lastSsr;
}
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
public synchronized int getLastAngleDegrees() {
return lastAngleDegrees;
}
public synchronized long getRealPackets() {
return realPackets;
}
@@ -0,0 +1,10 @@
package com.faceclaw.app;
public interface FaceclawMicStatusListener {
/**
* A CFW mic_control field-104 status record from one temple. body is the
* raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is
* "L" or "R" for the temple's own BLE link, "?" if unmatched.
*/
void onMicStatus(byte[] body, String arm);
}
@@ -0,0 +1,266 @@
package com.faceclaw.app;
import android.media.audiofx.AcousticEchoCanceler;
import android.media.audiofx.AudioEffect;
import android.media.audiofx.AutomaticGainControl;
import android.media.audiofx.NoiseSuppressor;
import java.util.ArrayList;
import java.util.Arrays;
/**
* Phone-side noise suppression for the microphone stream.
*
* Android's built-in noise cancellation (the {@link NoiseSuppressor} capture
* effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio
* arrives as raw PCM over BLE from the glasses, which that effect chain never
* sees, so this class picks the best engine that can actually reach the
* samples:
*
* - Audio captured by the phone itself: {@link #attachPlatformEffects(int)}
* enables the device's built-in NS/AEC/AGC chain (hardware or OEM software,
* whatever the Android build ships) on the AudioRecord session.
* - Audio from the glasses (the normal case): {@link #process(byte[])} runs a
* WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann
* frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an
* over-subtracted Wiener gain with a residual floor so speech never pumps
* in and out of silence.
*/
public final class FaceclawNoiseSuppressor {
private static final int FFT_SIZE = 256;
private static final int HOP = FFT_SIZE / 2;
private static final float OVERSUBTRACTION = 1.6f;
/** About -16 dB of residual: keeps the noise bed natural instead of gated. */
private static final float GAIN_FLOOR = 0.15f;
private static final float GAIN_SMOOTHING = 0.6f;
/** Frames of pure noise-floor learning after start/reset (~100 ms). */
private static final int NOISE_WARMUP_FRAMES = 12;
private final int sampleRate;
private final float[] window = new float[FFT_SIZE];
private final float[] history = new float[FFT_SIZE];
private final float[] overlap = new float[HOP];
private final float[] noise = new float[FFT_SIZE / 2 + 1];
private final float[] gain = new float[FFT_SIZE / 2 + 1];
private final float[] re = new float[FFT_SIZE];
private final float[] im = new float[FFT_SIZE];
private final short[] pending = new short[HOP];
private int pendingCount = 0;
private int warmupFrames = 0;
public FaceclawNoiseSuppressor(int sampleRate) {
this.sampleRate = sampleRate;
// sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap.
for (int i = 0; i < FFT_SIZE; i++) {
window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE);
}
reset();
}
public int getSampleRate() {
return sampleRate;
}
/** Forget the learned noise floor (e.g. after the suppressor was toggled off). */
public void reset() {
Arrays.fill(history, 0f);
Arrays.fill(overlap, 0f);
Arrays.fill(noise, 0f);
Arrays.fill(gain, 1f);
pendingCount = 0;
warmupFrames = 0;
}
/**
* Suppress noise in 16-bit little-endian mono PCM. Streaming: input not
* filling a whole 8 ms hop is carried into the next call, so the output
* may be up to one hop shorter than the input (and longer on a later
* call); total latency is one hop.
*/
public byte[] process(byte[] pcm16le) {
if (pcm16le == null || pcm16le.length < 2) {
return pcm16le;
}
int sampleCount = pcm16le.length / 2;
int hops = (pendingCount + sampleCount) / HOP;
byte[] out = new byte[hops * HOP * 2];
int outPos = 0;
int inPos = 0;
while (inPos < sampleCount) {
int take = Math.min(HOP - pendingCount, sampleCount - inPos);
for (int i = 0; i < take; i++) {
int lo = pcm16le[(inPos + i) * 2] & 0xff;
int hi = pcm16le[(inPos + i) * 2 + 1];
pending[pendingCount + i] = (short) ((hi << 8) | lo);
}
pendingCount += take;
inPos += take;
if (pendingCount == HOP) {
outPos = processHop(out, outPos);
pendingCount = 0;
}
}
return out;
}
private int processHop(byte[] out, int outPos) {
System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP);
for (int i = 0; i < HOP; i++) {
history[FFT_SIZE - HOP + i] = pending[i] / 32768f;
}
for (int i = 0; i < FFT_SIZE; i++) {
re[i] = history[i] * window[i];
im[i] = 0f;
}
fft(re, im, false);
boolean warming = warmupFrames < NOISE_WARMUP_FRAMES;
if (warming) {
warmupFrames++;
}
for (int b = 0; b <= FFT_SIZE / 2; b++) {
float power = re[b] * re[b] + im[b] * im[b];
if (warming) {
noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames;
} else if (power < noise[b]) {
// Track the floor down quickly; drift up slowly so speech
// energy never gets learned as noise.
noise[b] += (power - noise[b]) * 0.2f;
} else {
noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f);
}
float clean = power - OVERSUBTRACTION * noise[b];
float g = clean > 0f ? clean / power : 0f;
if (g < GAIN_FLOOR) {
g = GAIN_FLOOR;
}
g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g;
gain[b] = g;
re[b] *= g;
im[b] *= g;
if (b > 0 && b < FFT_SIZE / 2) {
re[FFT_SIZE - b] *= g;
im[FFT_SIZE - b] *= g;
}
}
fft(re, im, true);
for (int i = 0; i < HOP; i++) {
float value = (overlap[i] + re[i] * window[i]) * 32767f;
int sample = Math.round(value);
if (sample > 32767) {
sample = 32767;
}
if (sample < -32768) {
sample = -32768;
}
out[outPos + i * 2] = (byte) (sample & 0xff);
out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff);
}
for (int i = 0; i < HOP; i++) {
overlap[i] = re[HOP + i] * window[HOP + i];
}
return outPos + HOP * 2;
}
/** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */
private static void fft(float[] re, float[] im, boolean inverse) {
int n = re.length;
for (int i = 1, j = 0; i < n; i++) {
int bit = n >> 1;
for (; (j & bit) != 0; bit >>= 1) {
j ^= bit;
}
j ^= bit;
if (i < j) {
float t = re[i];
re[i] = re[j];
re[j] = t;
t = im[i];
im[i] = im[j];
im[j] = t;
}
}
for (int len = 2; len <= n; len <<= 1) {
double ang = 2 * Math.PI / len * (inverse ? 1 : -1);
float wRe = (float) Math.cos(ang);
float wIm = (float) Math.sin(ang);
for (int i = 0; i < n; i += len) {
float curRe = 1f;
float curIm = 0f;
for (int k = 0; k < len / 2; k++) {
int a = i + k;
int b = a + len / 2;
float xr = re[b] * curRe - im[b] * curIm;
float xi = re[b] * curIm + im[b] * curRe;
re[b] = re[a] - xr;
im[b] = im[a] - xi;
re[a] += xr;
im[a] += xi;
float nextRe = curRe * wRe - curIm * wIm;
curIm = curRe * wIm + curIm * wRe;
curRe = nextRe;
}
}
}
if (inverse) {
for (int i = 0; i < n; i++) {
re[i] /= n;
im[i] /= n;
}
}
}
/** Whether this Android build ships the built-in capture noise suppressor. */
public static boolean platformSuppressorAvailable() {
try {
return NoiseSuppressor.isAvailable();
} catch (Throwable t) {
return false;
}
}
/** Short engine name for the UI. */
public static String engineDescription() {
return "spectral (phone DSP)";
}
/**
* Attach the platform's built-in capture-effect chain (NoiseSuppressor,
* AcousticEchoCanceler, AutomaticGainControl — whichever this Android
* build provides) to a phone AudioRecord session. Only applicable when the
* phone itself captures the audio; the BLE glasses stream must go through
* {@link #process(byte[])} instead. The caller keeps the returned effects
* for the life of the recording and release()s them with it.
*/
public static AudioEffect[] attachPlatformEffects(int audioSessionId) {
ArrayList<AudioEffect> effects = new ArrayList<>();
try {
if (NoiseSuppressor.isAvailable()) {
NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId);
if (ns != null) {
ns.setEnabled(true);
effects.add(ns);
}
}
if (AcousticEchoCanceler.isAvailable()) {
AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId);
if (aec != null) {
aec.setEnabled(true);
effects.add(aec);
}
}
if (AutomaticGainControl.isAvailable()) {
AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId);
if (agc != null) {
agc.setEnabled(true);
effects.add(agc);
}
}
} catch (Throwable t) {
// A missing effect just means this tier is unavailable.
}
return effects.toArray(new AudioEffect[0]);
}
}
@@ -0,0 +1,111 @@
package com.faceclaw.app;
import android.util.Log;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
/**
* Speaker voice-print extraction over sherpa-onnx. One embedding per
* utterance; matching/enrollment happens on the TS side (cosine similarity
* against the stored speaker profiles).
*/
public class FaceclawSpeakerId {
private static final String TAG = "FaceclawSpeakerId";
private SpeakerEmbeddingExtractor extractor;
private final String modelPath;
public FaceclawSpeakerId(String modelPath) {
this.modelPath = modelPath;
}
/** Lazily loads the model; returns false when it cannot be loaded. */
public synchronized boolean ensureLoaded() {
if (extractor != null) {
return true;
}
try {
SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder()
.setModel(modelPath)
.setNumThreads(1)
.setDebug(false)
.setProvider("cpu")
.build();
extractor = new SpeakerEmbeddingExtractor(config);
return true;
} catch (Throwable t) {
Log.w(TAG, "could not load speaker model at " + modelPath, t);
extractor = null;
return false;
}
}
public synchronized int embeddingDim() {
return extractor == null ? 0 : extractor.getDim();
}
/**
* Compute an L2-normalized voice-print for one utterance of 16 kHz mono
* S16LE PCM. Returns null when the model is unavailable or the utterance
* is too short to embed (under ~0.5 s).
*/
public synchronized float[] embed(byte[] pcm16le, int sampleRate) {
if (pcm16le == null || pcm16le.length < sampleRate) {
return null;
}
if (!ensureLoaded()) {
return null;
}
int count = pcm16le.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8));
samples[i] = s / 32768.0f;
}
OnlineStream stream = null;
try {
stream = extractor.createStream();
stream.acceptWaveform(samples, sampleRate);
stream.inputFinished();
if (!extractor.isReady(stream)) {
return null;
}
float[] embedding = extractor.compute(stream);
return l2Normalize(embedding);
} catch (Throwable t) {
Log.w(TAG, "embedding failed", t);
return null;
} finally {
if (stream != null) {
stream.release();
}
}
}
private static float[] l2Normalize(float[] v) {
if (v == null) {
return null;
}
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return v;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
return v;
}
public synchronized void close() {
if (extractor != null) {
extractor.release();
extractor = null;
}
}
}
@@ -90,6 +90,26 @@ public class FaceclawVoiceController {
private volatile boolean endpointing;
private final EndpointDetector endpointDetector = new EndpointDetector();
private java.io.ByteArrayOutputStream recordingPcm;
// Speaker verification against the enrolled wearer voice-print ("my voice
// only" command gating). Configured before start(); the utterance PCM is
// buffered (capped) and verified once at session end.
private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10;
private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE;
private volatile String verifySpeakerModelPath;
private volatile float[] verifyWearerEmbedding;
private volatile float verifyThreshold = 0.8f;
private short[] verifyBuffer;
private int verifyCount;
// Global mic processing (Microphones app config): spectral noise
// suppression and firmware-DoA beam gating, applied to every capture
// session that opts in (assistant push-to-talk, Transcribe, hands-free).
// The raw tap opts out — raw means raw, and the Microphones session does
// its own beam-compensated processing on that path.
private volatile boolean suppressionEnabled;
private volatile boolean beamFilterEnabled;
private volatile int beamCenterDeg;
private volatile int beamHalfWidthDeg = 180;
private FaceclawNoiseSuppressor suppressor;
private long queuedPackets;
private long queueDroppedPackets;
private long decodedSamples;
@@ -125,6 +145,68 @@ public class FaceclawVoiceController {
this.endpointing = endpointing;
}
/**
* Verify this session's speaker against the enrolled wearer voice-print
* and report the result via onSpeakerVerified just before the final
* transcript. Must be set before {@link #start}; pass a null model path
* to disable.
*/
public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) {
this.verifySpeakerModelPath = speakerModelPath;
this.verifyWearerEmbedding = wearerEmbedding;
this.verifyThreshold = threshold > 0 ? threshold : 0.8f;
}
public void clearSpeakerVerification() {
this.verifySpeakerModelPath = null;
this.verifyWearerEmbedding = null;
}
/** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */
public void setNoiseSuppression(boolean enabled) {
this.suppressionEnabled = enabled;
}
/**
* Direction gating from the Sonic Radar beam: packets whose firmware
* direction-of-arrival falls outside centerDeg ± halfWidthDeg (device
* frame, 0 = straight ahead, positive right) are dropped before any
* consumer sees them. Safe to update mid-run.
*/
public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) {
this.beamFilterEnabled = enabled;
this.beamCenterDeg = centerDeg;
this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg));
}
private boolean withinBeam(int angleDegrees) {
int delta = angleDegrees - beamCenterDeg;
while (delta > 180) delta -= 360;
while (delta < -180) delta += 360;
return Math.abs(delta) <= beamHalfWidthDeg;
}
private void applySuppression(short[] pcm, int count) {
try {
if (suppressor == null) {
suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE);
}
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
le[i * 2] = (byte) (pcm[i] & 0xff);
le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff);
}
byte[] cleaned = suppressor.process(le);
int cleanedCount = Math.min(count, cleaned.length / 2);
for (int i = 0; i < cleanedCount; i++) {
pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8));
}
} catch (Throwable t) {
Log.w(TAG, "noise suppression failed; passing audio through", t);
suppressionEnabled = false;
}
}
public void start(String requestedMode) {
synchronized (lock) {
if (started) {
@@ -195,7 +277,13 @@ public class FaceclawVoiceController {
}
lc3Decoder = new FaceclawLc3Decoder();
endpointDetector.reset();
if (suppressor != null) {
suppressor.reset();
}
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null;
verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null;
verifyCount = 0;
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.");
return;
@@ -205,6 +293,10 @@ public class FaceclawVoiceController {
? "Listening (cloud)..."
: "Listening...");
processG2Audio();
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
// on (the callbacks are posted in order to the main handler).
runSpeakerVerification();
// Button released / stop requested: emit one final full-utterance
// transcript so the UI can freeze it.
if (currentMode == VoiceInputMode.ONBOARD) {
@@ -374,15 +466,38 @@ public class FaceclawVoiceController {
continue;
}
decodedSamples += count;
// Global mic processing from the Microphones app config: the
// beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker
// for every consumer, recognition included), and the spectral
// noise suppressor cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
int angleDegrees = currentDecoder.getLastAngleDegrees();
int ssr = currentDecoder.getLastSsr();
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr);
maybeEmitAudioStats(false);
continue;
}
if (suppressionEnabled) {
applySuppression(pcm, count);
}
if (recordingPcm != null) {
appendRecording(pcm, count);
}
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount);
System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied);
verifyCount += copied;
}
if (endpointing && endpointDetector.accept(pcm, count)) {
emitSpeechEnd();
}
if (mode == VoiceInputMode.CLOUD) {
emitPcm(pcm, count);
} else {
// PCM and frame metadata flow in every mode so levels, recording,
// and the Microphones radar keep working alongside onboard ASR.
emitPcm(pcm, count);
emitFrameMeta(angleDegrees, ssr);
if (mode != VoiceInputMode.CLOUD) {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
@@ -582,6 +697,83 @@ public class FaceclawVoiceController {
mainHandler.post(() -> currentListener.onPcm(le));
}
/**
* Embed the session's buffered utterance and compare it to the enrolled
* wearer voice-print. Fails open: a session too short to verify, or a
* model that will not load, counts as the wearer rather than silencing
* every command.
*/
private void runSpeakerVerification() {
short[] buffer = verifyBuffer;
float[] wearer = verifyWearerEmbedding;
String modelPath = verifySpeakerModelPath;
verifyBuffer = null;
if (buffer == null || wearer == null || modelPath == null) {
return;
}
if (verifyCount < VERIFY_MIN_SAMPLES) {
emitSpeakerVerified(true, 0f);
return;
}
FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath);
try {
byte[] le = new byte[verifyCount * 2];
for (int i = 0; i < verifyCount; i++) {
short s = buffer[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
float[] embedding = speakerId.embed(le, SAMPLE_RATE);
if (embedding == null || embedding.length != wearer.length) {
emitSpeakerVerified(true, 0f);
return;
}
double dot = 0;
for (int i = 0; i < embedding.length; i++) {
dot += (double) embedding[i] * wearer[i];
}
boolean isWearer = dot >= verifyThreshold;
Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot)
+ " threshold=" + verifyThreshold + " isWearer=" + isWearer);
emitSpeakerVerified(isWearer, (float) dot);
} catch (Throwable t) {
Log.w(TAG, "speaker verification failed", t);
emitSpeakerVerified(true, 0f);
}
}
// The 28 MB embedding model takes seconds to load; keep one instance
// across capture sessions so verification adds only the embed time.
private static FaceclawSpeakerId sharedSpeakerId;
private static String sharedSpeakerIdPath;
private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) {
if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) {
if (sharedSpeakerId != null) {
sharedSpeakerId.close();
}
sharedSpeakerId = new FaceclawSpeakerId(modelPath);
sharedSpeakerIdPath = modelPath;
}
return sharedSpeakerId;
}
private void emitSpeakerVerified(boolean isWearer, float similarity) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity));
}
private void emitFrameMeta(int angleDegrees, int ssr) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr));
}
private void emitSpeechEnd() {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
@@ -11,15 +11,34 @@ public interface FaceclawVoiceControllerListener {
void onTranscript(String text, boolean isFinal);
/**
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
* little-endian PCM. Empty/absent in onboard mode.
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
* keep working while the onboard recognizer owns the transcript.
*/
void onPcm(byte[] pcm16le);
/**
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
* packet, computed on the raw stereo capture before its mono downmix: the
* direction-of-arrival in signed degrees (0 = straight ahead, positive to
* the wearer's right) and a signal-strength ratio that serves as a
* confidence/energy proxy for the angle.
*/
void onFrameMeta(int angleDegrees, int ssr);
/**
* The speaker stopped, in a hands-free session that has no button release
* to end it. Only fires when endpointing was enabled for the session; see
* FaceclawVoiceController.setEndpointing.
*/
void onSpeechEnd();
/**
* Result of speaker verification against the enrolled wearer voice-print,
* emitted once per capture session just before the final transcript.
* Only fires when verification was configured for the session; fails
* open (isWearer=true) when the utterance was too short to verify or the
* speaker model could not be loaded.
*/
void onSpeakerVerified(boolean isWearer, float similarity);
}
@@ -0,0 +1,100 @@
package com.faceclaw.app;
import android.util.Log;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
/**
* Streaming WAV writer for conversation recordings: the header is written up
* front with zero sizes and patched on finish, so a crash mid-session leaves
* a recoverable file (data length can be derived from the file size).
*/
public class FaceclawWavRecorder {
private static final String TAG = "FaceclawWavRecorder";
private final String path;
private final int sampleRate;
private final int channels;
private RandomAccessFile file;
private long dataBytes;
public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException {
this.path = path;
this.sampleRate = sampleRate;
this.channels = channels;
File parent = new File(path).getParentFile();
if (parent != null && !parent.exists() && !parent.mkdirs()) {
throw new IOException("could not create " + parent);
}
file = new RandomAccessFile(path, "rw");
file.setLength(0);
file.write(buildHeader(0));
}
public synchronized void append(byte[] pcm16le) {
if (file == null || pcm16le == null || pcm16le.length == 0) {
return;
}
try {
file.write(pcm16le);
dataBytes += pcm16le.length;
} catch (IOException e) {
Log.w(TAG, "append failed", e);
}
}
public synchronized long getDurationMs() {
return dataBytes * 1000L / (sampleRate * channels * 2L);
}
public String getPath() {
return path;
}
/** Patch the header and close. Returns the PCM byte count (0 = empty). */
public synchronized long finish() {
RandomAccessFile current = file;
file = null;
if (current == null) {
return dataBytes;
}
try {
current.seek(0);
current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE)));
current.close();
} catch (IOException e) {
Log.w(TAG, "finish failed", e);
}
return dataBytes;
}
/** Delete the (finished or abandoned) recording. */
public synchronized void discard() {
finish();
new File(path).delete();
}
private byte[] buildHeader(int pcmBytes) {
int byteRate = sampleRate * channels * 2;
ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN);
b.put("RIFF".getBytes(StandardCharsets.US_ASCII));
b.putInt(36 + pcmBytes);
b.put("WAVE".getBytes(StandardCharsets.US_ASCII));
b.put("fmt ".getBytes(StandardCharsets.US_ASCII));
b.putInt(16);
b.putShort((short) 1);
b.putShort((short) channels);
b.putInt(sampleRate);
b.putInt(byteRate);
b.putShort((short) (channels * 2));
b.putShort((short) 16);
b.put("data".getBytes(StandardCharsets.US_ASCII));
b.putInt(pcmBytes);
return b.array();
}
}
@@ -49,6 +49,14 @@ public class BleProtocol {
// protobuf fields to stock implementations, so they are safely ignored.
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
/**
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
* to settings read responses and as standalone commandId=3 pushes.
*/
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
public static final int FACECLAW_WAKE_OP_ACQUIRE = 1;
public static final int FACECLAW_WAKE_OP_RELEASE = 2;
public static final int FACECLAW_WAKE_OP_CLAIM = 3;
@@ -405,6 +413,40 @@ public class BleProtocol {
));
}
/**
* Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a
* settings write carrying CFW field 103. Magic zero makes it
* fire-and-forget like the wake lease: the firmware consumes the field
* before its stock nanopb decoder discards the unknown tag, and the
* "ack" is the field-104 status notify that follows.
*/
public static byte[] buildFaceclawMicControl(byte[] record) {
return concat(CollectionUtils.listOf(
encodeVarintField(1, 1),
encodeVarintField(2, 0),
encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record)
));
}
/**
* Extract the CFW mic status record (field 104) from a settings frame, or
* null when absent. The 21-byte body layout is decoded on the TS side;
* here we only validate the 'M','C',version prelude.
*/
public static byte[] parseFaceclawMicStatus(byte[] pb) {
if (pb == null) {
return null;
}
byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD);
if (body == null
|| body.length < 3
|| body[0] != 'M'
|| body[1] != 'C') {
return null;
}
return body;
}
/**
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
* when this is an ordinary settings frame.
@@ -267,6 +267,21 @@ public class MessageBuilder {
);
}
/** CFW mic_control write (settings field 103) targeted at one temple. */
public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) {
return new OutboundMessage(
"mic-control",
"mic control " + label + (leftArm ? " L" : " R"),
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
0,
BleProtocol.buildFaceclawMicControl(record),
ACK_TIMEOUT_MS,
-1,
leftArm
);
}
public OutboundMessage faceclawWearQuery(boolean leftArm) {
return new OutboundMessage(
"wear-query-control",
@@ -0,0 +1,44 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class FastClusteringConfig {
private final int numClusters;
private final float threshold;
private FastClusteringConfig(Builder builder) {
this.numClusters = builder.numClusters;
this.threshold = builder.threshold;
}
public static Builder builder() {
return new Builder();
}
public int getNumClusters() {
return numClusters;
}
public float getThreshold() {
return threshold;
}
public static class Builder {
private int numClusters = -1;
private float threshold = 0.5f;
public FastClusteringConfig build() {
return new FastClusteringConfig(this);
}
public Builder setNumClusters(int numClusters) {
this.numClusters = numClusters;
return this;
}
public Builder setThreshold(float threshold) {
this.threshold = threshold;
return this;
}
}
}
@@ -0,0 +1,62 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarization {
private long ptr = 0;
public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization");
}
}
public int getSampleRate() {
return getSampleRate(ptr);
}
// Only config.clustering is used. All other fields are ignored
public void setConfig(OfflineSpeakerDiarizationConfig config) {
setConfig(ptr, config);
}
public OfflineSpeakerDiarizationSegment[] process(float[] samples) {
return process(ptr, samples);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) {
return processWithCallback(ptr, samples, callback, 0);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) {
return processWithCallback(ptr, samples, callback, arg);
}
protected void finalize() throws Throwable {
release();
}
// You'd better call it manually if it is not used anymore
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
private native int getSampleRate(long ptr);
private native void delete(long ptr);
private native long newFromFile(OfflineSpeakerDiarizationConfig config);
private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config);
private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples);
private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg);
}
@@ -0,0 +1,8 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
@FunctionalInterface
public interface OfflineSpeakerDiarizationCallback {
Integer invoke(int numProcessedChunks, int numTotalCunks, long arg);
}
@@ -0,0 +1,79 @@
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationConfig {
private final OfflineSpeakerSegmentationModelConfig segmentation;
private final SpeakerEmbeddingExtractorConfig embedding;
private final FastClusteringConfig clustering;
private final float minDurationOn;
private final float minDurationOff;
private OfflineSpeakerDiarizationConfig(Builder builder) {
this.segmentation = builder.segmentation;
this.embedding = builder.embedding;
this.clustering = builder.clustering;
this.minDurationOff = builder.minDurationOff;
this.minDurationOn = builder.minDurationOn;
}
public static Builder builder() {
return new Builder();
}
public OfflineSpeakerSegmentationModelConfig getSegmentation() {
return segmentation;
}
public SpeakerEmbeddingExtractorConfig getEmbedding() {
return embedding;
}
public FastClusteringConfig getClustering() {
return clustering;
}
public float getMinDurationOff() {
return minDurationOff;
}
public float getMinDurationOn() {
return minDurationOn;
}
public static class Builder {
private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build();
private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build();
private FastClusteringConfig clustering = FastClusteringConfig.builder().build();
private float minDurationOn = 0.2f;
private float minDurationOff = 0.5f;
public OfflineSpeakerDiarizationConfig build() {
return new OfflineSpeakerDiarizationConfig(this);
}
public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) {
this.segmentation = segmentation;
return this;
}
public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) {
this.embedding = embedding;
return this;
}
public Builder setClustering(FastClusteringConfig clustering) {
this.clustering = clustering;
return this;
}
public Builder setMinDurationOff(float minDurationOff) {
this.minDurationOff = minDurationOff;
return this;
}
public Builder setMinDurationOn(float minDurationOn) {
this.minDurationOn = minDurationOn;
return this;
}
}
}
@@ -0,0 +1,27 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationSegment {
private final float start;
private final float end;
private final int speaker;
public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) {
this.start = start;
this.end = end;
this.speaker = speaker;
}
public float getStart() {
return start;
}
public float getEnd() {
return end;
}
public int getSpeaker() {
return speaker;
}
}
@@ -0,0 +1,52 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerSegmentationModelConfig {
private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote;
private final int numThreads;
private final boolean debug;
private final String provider;
private OfflineSpeakerSegmentationModelConfig(Builder builder) {
this.pyannote = builder.pyannote;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build();
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public OfflineSpeakerSegmentationModelConfig build() {
return new OfflineSpeakerSegmentationModelConfig(this);
}
public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) {
this.pyannote = pyannote;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -0,0 +1,32 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerSegmentationPyannoteModelConfig {
private final String model;
private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public OfflineSpeakerSegmentationPyannoteModelConfig build() {
return new OfflineSpeakerSegmentationPyannoteModelConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
}
}
@@ -0,0 +1,77 @@
// Copyright 2022-2023 by zhaoming
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OnlineStream {
private long ptr = 0;
public OnlineStream() {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
this.ptr = 0;
}
public OnlineStream(long ptr) {
this.ptr = ptr;
}
public long getPtr() {
return ptr;
}
public void setPtr(long ptr) {
this.ptr = ptr;
}
public void acceptWaveform(float[] samples, int sampleRate) {
acceptWaveform(this.ptr, samples, sampleRate);
}
public void inputFinished() {
inputFinished(this.ptr);
}
public void setOption(String key, String value) {
setOption(this.ptr, key, value);
}
public String getOption(String key) {
return getOption(this.ptr, key);
}
public boolean hasOption(String key) {
return hasOption(this.ptr, key);
}
public void release() {
close();
}
public void close() {
// stream object must be release after used
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
@Override
protected void finalize() throws Throwable {
close();
super.finalize();
}
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
private native void inputFinished(long ptr);
private native void setOption(long ptr, String key, String value);
private native String getOption(long ptr, String key);
private native boolean hasOption(long ptr, String key);
private native void delete(long ptr);
}
@@ -0,0 +1,58 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractor {
private long ptr = 0;
public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor");
}
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public OnlineStream createStream() {
long p = createStream(ptr);
return new OnlineStream(p);
}
public boolean isReady(OnlineStream s) {
return isReady(ptr, s.getPtr());
}
public float[] compute(OnlineStream s) {
return compute(ptr, s.getPtr());
}
public int getDim() {
return dim(ptr);
}
private native void delete(long ptr);
private native long newFromFile(SpeakerEmbeddingExtractorConfig config);
private native long createStream(long ptr);
private native boolean isReady(long ptr, long streamPtr);
private native float[] compute(long ptr, long streamPtr);
private native int dim(long ptr);
}
@@ -0,0 +1,53 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractorConfig {
private final String model;
private final int numThreads;
private final boolean debug;
private final String provider;
private SpeakerEmbeddingExtractorConfig(Builder builder) {
this.model = builder.model;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private String model = "";
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public SpeakerEmbeddingExtractorConfig build() {
return new SpeakerEmbeddingExtractorConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -0,0 +1,78 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingManager {
private long ptr = 0;
public SpeakerEmbeddingManager(int dim) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = create(dim);
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public boolean add(String name, float[] embedding) {
return add(ptr, name, embedding);
}
public boolean add(String name, float[][] embedding) {
return addList(ptr, name, embedding);
}
public boolean remove(String name) {
return remove(ptr, name);
}
public String search(float[] embedding, float threshold) {
return search(ptr, embedding, threshold);
}
public boolean verify(String name, float[] embedding, float threshold) {
return verify(ptr, name, embedding, threshold);
}
public boolean contains(String name) {
return contains(ptr, name);
}
public int getNumSpeakers() {
return numSpeakers(ptr);
}
public String[] getAllSpeakerNames() {
return allSpeakerNames(ptr);
}
private native long create(int dim);
private native void delete(long ptr);
private native boolean add(long ptr, String name, float[] embedding);
private native boolean addList(long ptr, String name, float[][] embedding);
private native boolean remove(long ptr, String name);
private native String search(long ptr, float[] embedding, float threshold);
private native boolean verify(long ptr, String name, float[] embedding, float threshold);
private native boolean contains(long ptr, String name);
private native int numSpeakers(long ptr);
private native String[] allSpeakerNames(long ptr);
}