forked from Blendman974/faceclaw
Merge branch 'calculator' of https://github.com/kalanihelekunihi/faceclaw into calculator
This commit is contained in:
commit
f3ad55ca3d
126 files changed
+22253
-58
No files matched your search
@@ -0,0 +1,242 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodecInfo;
|
||||
import android.media.MediaFormat;
|
||||
import android.media.MediaMuxer;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
|
||||
/**
|
||||
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
|
||||
* container, cutting speech recordings to roughly a tenth of their WAV size.
|
||||
* Runs on its own thread; the source WAV is deleted on success when requested.
|
||||
*/
|
||||
public class FaceclawAudioTranscoder {
|
||||
private static final String TAG = "FaceclawTranscode";
|
||||
private static final long CODEC_TIMEOUT_US = 10_000;
|
||||
|
||||
public interface Listener {
|
||||
void onDone(String outputPath, long outputBytes);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
|
||||
private final String wavPath;
|
||||
private final String outputPath;
|
||||
private final int bitrateBps;
|
||||
private final boolean deleteSourceOnSuccess;
|
||||
private final Listener listener;
|
||||
|
||||
public FaceclawAudioTranscoder(
|
||||
String wavPath,
|
||||
String outputPath,
|
||||
int bitrateBps,
|
||||
boolean deleteSourceOnSuccess,
|
||||
Listener listener) {
|
||||
this.wavPath = wavPath;
|
||||
this.outputPath = outputPath;
|
||||
this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000;
|
||||
this.deleteSourceOnSuccess = deleteSourceOnSuccess;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
Thread thread = new Thread(this::run, "FaceclawTranscode");
|
||||
thread.setPriority(Thread.MIN_PRIORITY);
|
||||
thread.start();
|
||||
}
|
||||
|
||||
private void run() {
|
||||
try {
|
||||
transcode();
|
||||
long size = new File(outputPath).length();
|
||||
if (deleteSourceOnSuccess) {
|
||||
new File(wavPath).delete();
|
||||
}
|
||||
if (listener != null) {
|
||||
listener.onDone(outputPath, size);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "transcode failed for " + wavPath, t);
|
||||
new File(outputPath).delete();
|
||||
if (listener != null) {
|
||||
listener.onError(t.getMessage() == null ? t.toString() : t.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void transcode() throws IOException {
|
||||
WavInfo wav = readWavHeader(wavPath);
|
||||
MediaFormat format = MediaFormat.createAudioFormat(
|
||||
MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels);
|
||||
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
|
||||
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps);
|
||||
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
|
||||
|
||||
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
|
||||
MediaMuxer muxer = null;
|
||||
FileInputStream input = null;
|
||||
try {
|
||||
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
|
||||
codec.start();
|
||||
muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
|
||||
input = new FileInputStream(wavPath);
|
||||
long skipped = 0;
|
||||
while (skipped < wav.dataOffset) {
|
||||
long step = input.skip(wav.dataOffset - skipped);
|
||||
if (step <= 0) {
|
||||
throw new IOException("could not seek past WAV header");
|
||||
}
|
||||
skipped += step;
|
||||
}
|
||||
|
||||
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
|
||||
byte[] chunk = new byte[8192];
|
||||
int track = -1;
|
||||
boolean muxerStarted = false;
|
||||
boolean inputDone = false;
|
||||
boolean outputDone = false;
|
||||
long remaining = wav.dataBytes;
|
||||
long presentationUs = 0;
|
||||
long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2;
|
||||
|
||||
while (!outputDone) {
|
||||
if (!inputDone) {
|
||||
int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US);
|
||||
if (inIndex >= 0) {
|
||||
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
|
||||
int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity()));
|
||||
int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1;
|
||||
if (read <= 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, presentationUs,
|
||||
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
|
||||
inputDone = true;
|
||||
} else {
|
||||
inBuf.clear();
|
||||
inBuf.put(chunk, 0, read);
|
||||
codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0);
|
||||
remaining -= read;
|
||||
presentationUs += read * 1_000_000L / bytesPerSecond;
|
||||
}
|
||||
}
|
||||
}
|
||||
int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US);
|
||||
if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
track = muxer.addTrack(codec.getOutputFormat());
|
||||
muxer.start();
|
||||
muxerStarted = true;
|
||||
} else if (outIndex >= 0) {
|
||||
if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
|
||||
if (!muxerStarted) {
|
||||
throw new IOException("encoder produced data before format");
|
||||
}
|
||||
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
|
||||
outBuf.position(info.offset);
|
||||
outBuf.limit(info.offset + info.size);
|
||||
muxer.writeSampleData(track, outBuf, info);
|
||||
}
|
||||
codec.releaseOutputBuffer(outIndex, false);
|
||||
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputDone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (input != null) {
|
||||
try {
|
||||
input.close();
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
try {
|
||||
codec.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
codec.release();
|
||||
if (muxer != null) {
|
||||
try {
|
||||
muxer.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
muxer.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final class WavInfo {
|
||||
int sampleRate;
|
||||
int channels;
|
||||
long dataOffset;
|
||||
long dataBytes;
|
||||
}
|
||||
|
||||
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
|
||||
private static WavInfo readWavHeader(String path) throws IOException {
|
||||
try (FileInputStream in = new FileInputStream(path)) {
|
||||
byte[] head = new byte[12];
|
||||
if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') {
|
||||
throw new IOException("not a WAV file");
|
||||
}
|
||||
WavInfo info = new WavInfo();
|
||||
long offset = 12;
|
||||
byte[] chunkHead = new byte[8];
|
||||
while (in.read(chunkHead) == 8) {
|
||||
ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN);
|
||||
int id = b.getInt();
|
||||
long size = b.getInt() & 0xffffffffL;
|
||||
offset += 8;
|
||||
if (id == 0x20746d66) { // "fmt "
|
||||
byte[] fmt = new byte[(int) Math.min(size, 16)];
|
||||
if (in.read(fmt) != fmt.length) {
|
||||
throw new IOException("truncated fmt chunk");
|
||||
}
|
||||
ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN);
|
||||
int audioFormat = f.getShort();
|
||||
info.channels = f.getShort();
|
||||
info.sampleRate = f.getInt();
|
||||
f.getInt();
|
||||
f.getShort();
|
||||
int bits = f.getShort();
|
||||
if (audioFormat != 1 || bits != 16) {
|
||||
throw new IOException("only 16-bit PCM WAV is supported");
|
||||
}
|
||||
long skip = size - fmt.length;
|
||||
while (skip > 0) {
|
||||
long step = in.skip(skip);
|
||||
if (step <= 0) {
|
||||
break;
|
||||
}
|
||||
skip -= step;
|
||||
}
|
||||
offset += size;
|
||||
} else if (id == 0x61746164) { // "data"
|
||||
info.dataOffset = offset;
|
||||
// A zero size means the header was never patched (crash
|
||||
// mid-recording); recover the length from the file size.
|
||||
info.dataBytes = size > 0 ? size : new File(path).length() - offset;
|
||||
break;
|
||||
} else {
|
||||
long skip = size;
|
||||
while (skip > 0) {
|
||||
long step = in.skip(skip);
|
||||
if (step <= 0) {
|
||||
break;
|
||||
}
|
||||
skip -= step;
|
||||
}
|
||||
offset += size;
|
||||
}
|
||||
}
|
||||
if (info.sampleRate == 0 || info.dataOffset == 0) {
|
||||
throw new IOException("missing fmt or data chunk");
|
||||
}
|
||||
return info;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private final java.util.List<FaceclawCompassListener> compassListeners =
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private volatile String lastFirmwareCapabilities = "";
|
||||
private volatile Thread workerThread;
|
||||
private volatile boolean running;
|
||||
private volatile boolean userDisconnectRequested;
|
||||
@@ -535,6 +538,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
|
||||
public String getFirmwareCapabilities() {
|
||||
return lastFirmwareCapabilities;
|
||||
}
|
||||
|
||||
public void addMicStatusListener(FaceclawMicStatusListener listener) {
|
||||
if (listener != null) {
|
||||
micStatusListeners.add(listener);
|
||||
}
|
||||
}
|
||||
|
||||
public void removeMicStatusListener(FaceclawMicStatusListener listener) {
|
||||
if (listener != null) {
|
||||
micStatusListeners.remove(listener);
|
||||
}
|
||||
}
|
||||
|
||||
private void emitMicStatus(byte[] body, String address) {
|
||||
if (micStatusListeners.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String arm = address.equalsIgnoreCase(leftAddress) ? "L"
|
||||
: address.equalsIgnoreCase(rightAddress) ? "R" : "?";
|
||||
byte[] copy = java.util.Arrays.copyOf(body, body.length);
|
||||
mainHandler.post(() -> {
|
||||
for (FaceclawMicStatusListener micListener : micStatusListeners) {
|
||||
try {
|
||||
micListener.onMicStatus(copy, arm);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "mic status listener failed", t);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
|
||||
* field-103 write to both temples, or to a single one. Fire-and-forget:
|
||||
* the firmware answers with a field-104 status notify per temple, which
|
||||
* arrives through addMicStatusListener.
|
||||
*/
|
||||
public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) {
|
||||
if (record == null || record.length < 4) {
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (!running || !sessionReady) {
|
||||
logLine("skip mic control (" + label + "); session not ready");
|
||||
return;
|
||||
}
|
||||
if (rightTemple) {
|
||||
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false));
|
||||
}
|
||||
if (leftTemple) {
|
||||
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true));
|
||||
}
|
||||
logLine("queue mic control " + label);
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward render-characteristic audio packets to the listener WITHOUT
|
||||
* sending the stock EvenHub audio-control enable. Used for the CFW
|
||||
* mic_control streaming path, where capture is armed through settings
|
||||
* field 103 and the temples emit 'SM' frames on the same characteristic
|
||||
* that stock mono LC3 uses. Returns false when no session is up.
|
||||
*/
|
||||
public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) {
|
||||
if (listener == null) {
|
||||
throw new IllegalArgumentException("listener is required");
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (!running || !sessionReady || shutdownRequested) {
|
||||
logLine("skip G2 audio forwarding; session not ready");
|
||||
return false;
|
||||
}
|
||||
audioPacketListener = listener;
|
||||
audioCaptureActive = true;
|
||||
logLine("G2 audio forwarding enabled");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public void stopG2AudioForwarding() {
|
||||
synchronized (lock) {
|
||||
audioPacketListener = null;
|
||||
audioCaptureActive = false;
|
||||
logLine("G2 audio forwarding disabled");
|
||||
}
|
||||
}
|
||||
|
||||
public void addCompassListener(FaceclawCompassListener listener) {
|
||||
if (listener != null) {
|
||||
compassListeners.add(listener);
|
||||
@@ -1230,6 +1325,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
0
|
||||
);
|
||||
}
|
||||
if (!faceclawWakeNotification
|
||||
&& frame.ok
|
||||
&& frame.sid == BleProtocol.SID_UI_SETTING) {
|
||||
// CFW mic status (field 104) rides both standalone pushes and
|
||||
// settings read acks, from each temple on its own link.
|
||||
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
|
||||
if (micStatus != null) {
|
||||
emitMicStatus(micStatus, address);
|
||||
}
|
||||
}
|
||||
if (!faceclawWakeNotification
|
||||
&& decodedWearState < 0
|
||||
&& frame.ok
|
||||
@@ -2502,6 +2607,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
|
||||
if (firmwareInfo != null) {
|
||||
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
|
||||
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
|
||||
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
|
||||
&& hasCapability(firmwareInfo.capabilities, "texstr14");
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FeatureConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineStream;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Continuous captioning over the decoded mic PCM: segments speech into
|
||||
* utterances with an adaptive energy gate, transcribes each utterance with
|
||||
* the on-device Moonshine model, and attaches a speaker voice-print per
|
||||
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
|
||||
* and any beam-direction gating), so this engine has no BLE dependencies.
|
||||
*
|
||||
* Utterance boundaries are measured on the sample clock, not the wall clock:
|
||||
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
|
||||
* audio heard. The TS side converts startMs/endMs to wall-clock times using
|
||||
* the engine start timestamp.
|
||||
*/
|
||||
public class FaceclawCaptionEngine {
|
||||
private static final String TAG = "FaceclawCaptions";
|
||||
private static final int SAMPLE_RATE = 16000;
|
||||
private static final int FEATURE_DIM = 80;
|
||||
private static final int MAX_QUEUE_PACKETS = 200;
|
||||
|
||||
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
|
||||
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
|
||||
private static final double ONSET_FACTOR = 3.0;
|
||||
private static final double RELEASE_FACTOR = 1.8;
|
||||
private static final double MIN_RMS = 220.0;
|
||||
private static final double NOISE_EMA_ALPHA = 0.05;
|
||||
private static final int PRE_ROLL_MS = 400;
|
||||
private static final int DEFAULT_SILENCE_MS = 800;
|
||||
private static final int MIN_UTTERANCE_MS = 350;
|
||||
private static final int MAX_UTTERANCE_MS = 15000;
|
||||
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
|
||||
// segments cut at the quietest window (mirrors FaceclawVoiceController).
|
||||
private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
|
||||
private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2;
|
||||
private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000;
|
||||
private static final float NORMALIZE_TARGET_PEAK = 0.9f;
|
||||
private static final float NORMALIZE_MAX_GAIN = 30f;
|
||||
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
|
||||
private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10;
|
||||
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private final Object lock = new Object();
|
||||
private final Object queueLock = new Object();
|
||||
private final ArrayDeque<byte[]> queue = new ArrayDeque<>();
|
||||
|
||||
private volatile FaceclawCaptionEngineListener listener;
|
||||
private volatile String asrModelDir;
|
||||
private volatile String speakerModelPath;
|
||||
private volatile int silenceMs = DEFAULT_SILENCE_MS;
|
||||
private Thread workerThread;
|
||||
private volatile boolean started;
|
||||
|
||||
private OfflineRecognizer recognizer;
|
||||
private FaceclawSpeakerId speakerId;
|
||||
|
||||
// Segmentation state (worker thread only).
|
||||
private long totalSamples;
|
||||
private double noiseFloor;
|
||||
private boolean inUtterance;
|
||||
private long utteranceStartSample;
|
||||
private long silenceRunSamples;
|
||||
private double utterancePeakRms;
|
||||
private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)];
|
||||
private int utteranceLength;
|
||||
private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000];
|
||||
private int preRollLength;
|
||||
|
||||
public void setListener(FaceclawCaptionEngineListener listener) {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
/** Directory holding the Moonshine model files, or null to disable ASR. */
|
||||
public void setAsrModelDir(String dir) {
|
||||
this.asrModelDir = dir;
|
||||
}
|
||||
|
||||
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
|
||||
public void setSpeakerModelPath(String path) {
|
||||
this.speakerModelPath = path;
|
||||
}
|
||||
|
||||
public void setSilenceMs(int ms) {
|
||||
this.silenceMs = Math.max(200, Math.min(3000, ms));
|
||||
}
|
||||
|
||||
public void start() {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
Thread threadToJoin;
|
||||
synchronized (lock) {
|
||||
if (!started) {
|
||||
return;
|
||||
}
|
||||
started = false;
|
||||
threadToJoin = workerThread;
|
||||
workerThread = null;
|
||||
}
|
||||
synchronized (queueLock) {
|
||||
queueLock.notifyAll();
|
||||
}
|
||||
if (threadToJoin != null) {
|
||||
threadToJoin.interrupt();
|
||||
if (Thread.currentThread() != threadToJoin) {
|
||||
try {
|
||||
threadToJoin.join(2000);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
|
||||
public void acceptPcm(byte[] pcm16le) {
|
||||
if (!started || pcm16le == null || pcm16le.length < 2) {
|
||||
return;
|
||||
}
|
||||
synchronized (queueLock) {
|
||||
if (queue.size() >= MAX_QUEUE_PACKETS) {
|
||||
queue.removeFirst();
|
||||
}
|
||||
queue.addLast(pcm16le);
|
||||
queueLock.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
try {
|
||||
loadModels();
|
||||
resetSegmentation();
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
byte[] chunk = takeChunk();
|
||||
if (chunk == null) {
|
||||
continue;
|
||||
}
|
||||
processChunk(chunk);
|
||||
}
|
||||
// Flush a trailing utterance so its text isn't lost on stop.
|
||||
if (inUtterance && utteranceLength > 0) {
|
||||
finalizeUtterance();
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.e(TAG, "caption engine failed", t);
|
||||
emitStatus("Captions failed: " + t.getMessage());
|
||||
} finally {
|
||||
releaseModels();
|
||||
}
|
||||
}
|
||||
|
||||
private void loadModels() {
|
||||
String modelDir = asrModelDir;
|
||||
if (modelDir != null && new File(modelDir, "tokens.txt").exists()) {
|
||||
emitStatus("Loading caption model...");
|
||||
recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setModelConfig(OfflineModelConfig.builder()
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.build());
|
||||
} else {
|
||||
recognizer = null;
|
||||
}
|
||||
String embedModel = speakerModelPath;
|
||||
if (embedModel != null && new File(embedModel).exists()) {
|
||||
speakerId = new FaceclawSpeakerId(embedModel);
|
||||
speakerId.ensureLoaded();
|
||||
} else {
|
||||
speakerId = null;
|
||||
}
|
||||
emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)...");
|
||||
}
|
||||
|
||||
private void releaseModels() {
|
||||
if (recognizer != null) {
|
||||
recognizer.release();
|
||||
recognizer = null;
|
||||
}
|
||||
if (speakerId != null) {
|
||||
speakerId.close();
|
||||
speakerId = null;
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] takeChunk() {
|
||||
synchronized (queueLock) {
|
||||
while (started && queue.isEmpty()) {
|
||||
try {
|
||||
queueLock.wait(250);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return queue.pollFirst();
|
||||
}
|
||||
}
|
||||
|
||||
private void resetSegmentation() {
|
||||
totalSamples = 0;
|
||||
noiseFloor = 0;
|
||||
inUtterance = false;
|
||||
utteranceLength = 0;
|
||||
preRollLength = 0;
|
||||
silenceRunSamples = 0;
|
||||
utterancePeakRms = 0;
|
||||
}
|
||||
|
||||
private void processChunk(byte[] chunk) {
|
||||
int count = chunk.length / 2;
|
||||
short[] pcm = new short[count];
|
||||
double sumSquares = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8));
|
||||
pcm[i] = s;
|
||||
sumSquares += (double) s * s;
|
||||
}
|
||||
double rms = Math.sqrt(sumSquares / Math.max(1, count));
|
||||
totalSamples += count;
|
||||
|
||||
if (!inUtterance) {
|
||||
// Track the noise floor only while idle so speech doesn't raise it.
|
||||
noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA;
|
||||
double threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
if (rms >= threshold * ONSET_FACTOR) {
|
||||
inUtterance = true;
|
||||
utteranceLength = 0;
|
||||
silenceRunSamples = 0;
|
||||
utterancePeakRms = rms;
|
||||
appendUtterance(preRoll, preRollLength);
|
||||
utteranceStartSample = Math.max(0, totalSamples - count - preRollLength);
|
||||
appendUtterance(pcm, count);
|
||||
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE);
|
||||
} else {
|
||||
appendPreRoll(pcm, count);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
appendUtterance(pcm, count);
|
||||
utterancePeakRms = Math.max(utterancePeakRms, rms);
|
||||
double threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
if (rms < threshold * RELEASE_FACTOR) {
|
||||
silenceRunSamples += count;
|
||||
} else {
|
||||
silenceRunSamples = 0;
|
||||
}
|
||||
long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE;
|
||||
boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs;
|
||||
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
|
||||
finalizeUtterance();
|
||||
inUtterance = false;
|
||||
preRollLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private void appendPreRoll(short[] pcm, int count) {
|
||||
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
|
||||
if (count >= preRoll.length) {
|
||||
System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length);
|
||||
preRollLength = preRoll.length;
|
||||
return;
|
||||
}
|
||||
int keep = Math.min(preRollLength, preRoll.length - count);
|
||||
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep);
|
||||
System.arraycopy(pcm, 0, preRoll, keep, count);
|
||||
preRollLength = keep + count;
|
||||
}
|
||||
|
||||
private void appendUtterance(short[] pcm, int count) {
|
||||
int room = utterance.length - utteranceLength;
|
||||
int copied = Math.min(room, count);
|
||||
if (copied > 0) {
|
||||
System.arraycopy(pcm, 0, utterance, utteranceLength, copied);
|
||||
utteranceLength += copied;
|
||||
}
|
||||
}
|
||||
|
||||
private void finalizeUtterance() {
|
||||
int length = utteranceLength;
|
||||
long startMs = utteranceStartSample * 1000 / SAMPLE_RATE;
|
||||
long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE;
|
||||
if (endMs - startMs < MIN_UTTERANCE_MS) {
|
||||
return;
|
||||
}
|
||||
String text = recognizeUtterance(length);
|
||||
float[] embedding = embedUtterance(length);
|
||||
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms);
|
||||
}
|
||||
|
||||
private String recognizeUtterance(int length) {
|
||||
if (recognizer == null || length <= 0) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder joined = new StringBuilder();
|
||||
int offset = 0;
|
||||
while (offset < length) {
|
||||
int remaining = length - offset;
|
||||
int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES);
|
||||
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
|
||||
segment = findQuietCut(offset, segment);
|
||||
}
|
||||
String part = recognizeRange(offset, segment);
|
||||
if (part.length() > 0) {
|
||||
if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) {
|
||||
joined.append(' ');
|
||||
}
|
||||
joined.append(part);
|
||||
}
|
||||
offset += segment;
|
||||
}
|
||||
return joined.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* End a decode segment at the center of the quietest window near its end
|
||||
* so the cut lands between words (same approach as the PTT controller).
|
||||
*/
|
||||
private int findQuietCut(int offset, int segment) {
|
||||
int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES);
|
||||
int win = CUT_WINDOW_SAMPLES;
|
||||
if (segment - searchStart <= win) {
|
||||
return segment;
|
||||
}
|
||||
double sum = 0;
|
||||
for (int i = searchStart; i < searchStart + win; i++) {
|
||||
double s = utterance[offset + i];
|
||||
sum += s * s;
|
||||
}
|
||||
double best = sum;
|
||||
int bestStart = searchStart;
|
||||
for (int start = searchStart + 1; start + win <= segment; start++) {
|
||||
double dropped = utterance[offset + start - 1];
|
||||
double added = utterance[offset + start + win - 1];
|
||||
sum += added * added - dropped * dropped;
|
||||
if (sum < best) {
|
||||
best = sum;
|
||||
bestStart = start;
|
||||
}
|
||||
}
|
||||
return bestStart + win / 2;
|
||||
}
|
||||
|
||||
private String recognizeRange(int offset, int count) {
|
||||
float[] samples = new float[count];
|
||||
float peak = 0f;
|
||||
for (int i = 0; i < count; i++) {
|
||||
float v = utterance[offset + i] / 32768.0f;
|
||||
samples[i] = v;
|
||||
float a = Math.abs(v);
|
||||
if (a > peak) {
|
||||
peak = a;
|
||||
}
|
||||
}
|
||||
if (peak > 0f) {
|
||||
float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN);
|
||||
if (gain > 1f) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] *= gain;
|
||||
}
|
||||
}
|
||||
}
|
||||
OfflineStream stream = recognizer.createStream();
|
||||
try {
|
||||
stream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
recognizer.decode(stream);
|
||||
OfflineRecognizerResult result = recognizer.getResult(stream);
|
||||
String raw = result == null ? "" : result.getText();
|
||||
return raw == null ? "" : raw.trim();
|
||||
} finally {
|
||||
stream.release();
|
||||
}
|
||||
}
|
||||
|
||||
private float[] embedUtterance(int length) {
|
||||
FaceclawSpeakerId currentSpeakerId = speakerId;
|
||||
if (currentSpeakerId == null || length <= 0) {
|
||||
return null;
|
||||
}
|
||||
int count = Math.min(length, EMBED_MAX_SAMPLES);
|
||||
byte[] le = new byte[count * 2];
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = utterance[i];
|
||||
le[i * 2] = (byte) (s & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
|
||||
}
|
||||
return currentSpeakerId.embed(le, SAMPLE_RATE);
|
||||
}
|
||||
|
||||
private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length);
|
||||
mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms));
|
||||
}
|
||||
|
||||
private void emitSpeechStart(long startMs) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onSpeechStart(startMs));
|
||||
}
|
||||
|
||||
private void emitStatus(String status) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onStatus(status));
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawCaptionEngineListener {
|
||||
/**
|
||||
* One finished utterance. text may be empty when no ASR model is loaded
|
||||
* or recognition produced nothing; embedding is the L2-normalized
|
||||
* voice-print or null when the speaker model is unavailable or the
|
||||
* utterance was too short. startMs/endMs are on the engine's sample
|
||||
* clock (milliseconds of audio since start()).
|
||||
*/
|
||||
void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms);
|
||||
|
||||
/** A speech onset was detected; the utterance is now being collected. */
|
||||
void onSpeechStart(long startMs);
|
||||
|
||||
void onStatus(String status);
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.ContentValues;
|
||||
import android.content.Context;
|
||||
import android.database.Cursor;
|
||||
import android.database.sqlite.SQLiteDatabase;
|
||||
import android.database.sqlite.SQLiteOpenHelper;
|
||||
import android.util.Base64;
|
||||
import android.util.Log;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Persistent store for caption sessions, transcript segments, and speaker
|
||||
* voice profiles, backing the Microphones app and its phone-side review UI.
|
||||
* All complex values cross the JS bridge as JSON strings; voice-print
|
||||
* embeddings are float32 LE blobs, exposed as base64.
|
||||
*/
|
||||
public class FaceclawConversationStore extends SQLiteOpenHelper {
|
||||
private static final String TAG = "FaceclawConvStore";
|
||||
private static final String DB_NAME = "faceclaw-conversations.db";
|
||||
private static final int DB_VERSION = 2;
|
||||
|
||||
private static FaceclawConversationStore instance;
|
||||
|
||||
public static synchronized FaceclawConversationStore getInstance(Context context) {
|
||||
if (instance == null) {
|
||||
instance = new FaceclawConversationStore(context.getApplicationContext());
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
private FaceclawConversationStore(Context context) {
|
||||
super(context, DB_NAME, null, DB_VERSION);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCreate(SQLiteDatabase db) {
|
||||
db.execSQL("CREATE TABLE speakers ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "name TEXT NOT NULL,"
|
||||
+ "color TEXT NOT NULL DEFAULT '#4FC3F7',"
|
||||
+ "is_wearer INTEGER NOT NULL DEFAULT 0,"
|
||||
+ "embedding BLOB,"
|
||||
+ "embedding_count INTEGER NOT NULL DEFAULT 0,"
|
||||
+ "created_at INTEGER NOT NULL,"
|
||||
+ "last_heard_at INTEGER,"
|
||||
+ "tag TEXT,"
|
||||
+ "last_recap TEXT,"
|
||||
+ "action_items TEXT,"
|
||||
+ "facts TEXT,"
|
||||
+ "insights_updated_at INTEGER,"
|
||||
+ "insights_session_id INTEGER)");
|
||||
db.execSQL("CREATE TABLE sessions ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "started_at INTEGER NOT NULL,"
|
||||
+ "ended_at INTEGER,"
|
||||
+ "title TEXT,"
|
||||
+ "audio_path TEXT,"
|
||||
+ "audio_codec TEXT,"
|
||||
+ "avg_sentiment REAL,"
|
||||
+ "segment_count INTEGER NOT NULL DEFAULT 0)");
|
||||
db.execSQL("CREATE TABLE segments ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "session_id INTEGER NOT NULL,"
|
||||
+ "speaker_id INTEGER,"
|
||||
+ "started_at INTEGER NOT NULL,"
|
||||
+ "ended_at INTEGER NOT NULL,"
|
||||
+ "audio_offset_ms INTEGER,"
|
||||
+ "text TEXT NOT NULL,"
|
||||
+ "lang TEXT,"
|
||||
+ "translation TEXT,"
|
||||
+ "translation_lang TEXT,"
|
||||
+ "sentiment REAL,"
|
||||
+ "emotion TEXT,"
|
||||
+ "search_meta TEXT,"
|
||||
+ "angle INTEGER,"
|
||||
+ "embedding BLOB)");
|
||||
db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)");
|
||||
db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)");
|
||||
db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)");
|
||||
db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
|
||||
if (oldVersion < 2) {
|
||||
// v2: per-speaker conversation insights (recap of the most recent
|
||||
// conversation, its action items, and accumulated inferred facts).
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER");
|
||||
}
|
||||
}
|
||||
|
||||
// ---- sessions ----
|
||||
|
||||
public long startSession(long startedAtMs, String title) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("started_at", startedAtMs);
|
||||
values.put("title", title == null || title.isEmpty() ? null : title);
|
||||
return getWritableDatabase().insert("sessions", null, values);
|
||||
}
|
||||
|
||||
public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("ended_at", endedAtMs);
|
||||
if (audioPath != null && !audioPath.isEmpty()) {
|
||||
values.put("audio_path", audioPath);
|
||||
values.put("audio_codec", audioCodec);
|
||||
}
|
||||
values.put("avg_sentiment", avgSentiment);
|
||||
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
public void setSessionAudio(long sessionId, String audioPath, String audioCodec) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("audio_path", audioPath);
|
||||
values.put("audio_codec", audioCodec);
|
||||
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
public String getSessionAudioPath(long sessionId) {
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) {
|
||||
if (c.moveToFirst()) {
|
||||
return c.isNull(0) ? "" : c.getString(0);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs,
|
||||
* speakerId, emotion (matches any segment emotion in the session), query
|
||||
* (LIKE over title and segment text/search metadata), limit.
|
||||
*/
|
||||
public String querySessions(String filterJson) {
|
||||
try {
|
||||
JSONObject filter = filterJson == null || filterJson.isEmpty()
|
||||
? new JSONObject() : new JSONObject(filterJson);
|
||||
StringBuilder sql = new StringBuilder(
|
||||
"SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec,"
|
||||
+ " s.avg_sentiment, s.segment_count FROM sessions s");
|
||||
List<String> args = new ArrayList<>();
|
||||
List<String> where = new ArrayList<>();
|
||||
boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query");
|
||||
if (joinSegments) {
|
||||
sql.append(" LEFT JOIN segments g ON g.session_id = s.id");
|
||||
}
|
||||
if (filter.has("sinceMs")) {
|
||||
where.add("s.started_at >= ?");
|
||||
args.add(String.valueOf(filter.getLong("sinceMs")));
|
||||
}
|
||||
if (filter.has("untilMs")) {
|
||||
where.add("s.started_at <= ?");
|
||||
args.add(String.valueOf(filter.getLong("untilMs")));
|
||||
}
|
||||
if (filter.has("speakerId")) {
|
||||
where.add("g.speaker_id = ?");
|
||||
args.add(String.valueOf(filter.getLong("speakerId")));
|
||||
}
|
||||
if (filter.has("emotion")) {
|
||||
where.add("g.emotion = ?");
|
||||
args.add(filter.getString("emotion"));
|
||||
}
|
||||
if (filter.has("query")) {
|
||||
String like = "%" + filter.getString("query") + "%";
|
||||
where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)");
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
}
|
||||
if (!where.isEmpty()) {
|
||||
sql.append(" WHERE ").append(String.join(" AND ", where));
|
||||
}
|
||||
sql.append(" ORDER BY s.started_at DESC");
|
||||
int limit = filter.optInt("limit", 200);
|
||||
sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000)));
|
||||
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
long id = c.getLong(0);
|
||||
row.put("id", id);
|
||||
row.put("startedAt", c.getLong(1));
|
||||
row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2));
|
||||
row.put("title", c.isNull(3) ? "" : c.getString(3));
|
||||
row.put("audioPath", c.isNull(4) ? "" : c.getString(4));
|
||||
row.put("audioCodec", c.isNull(5) ? "" : c.getString(5));
|
||||
row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6));
|
||||
row.put("segmentCount", c.getLong(7));
|
||||
row.put("speakers", sessionSpeakersJson(id));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySessions failed", e);
|
||||
return "[]";
|
||||
}
|
||||
}
|
||||
|
||||
private JSONArray sessionSpeakersJson(long sessionId) throws JSONException {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT DISTINCT p.id, p.name, p.color FROM segments g"
|
||||
+ " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("name", c.getString(1));
|
||||
row.put("color", c.getString(2));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- segments ----
|
||||
|
||||
/**
|
||||
* Insert one transcript segment. json fields: sessionId, speakerId?,
|
||||
* startedAt, endedAt, audioOffsetMs?, text, lang?, translation?,
|
||||
* translationLang?, sentiment?, emotion?, searchMeta?, angle?,
|
||||
* embeddingBase64?. Returns the new row id, or -1.
|
||||
*/
|
||||
public long insertSegment(String json) {
|
||||
try {
|
||||
JSONObject s = new JSONObject(json);
|
||||
ContentValues values = new ContentValues();
|
||||
long sessionId = s.getLong("sessionId");
|
||||
values.put("session_id", sessionId);
|
||||
if (s.has("speakerId")) {
|
||||
values.put("speaker_id", s.getLong("speakerId"));
|
||||
}
|
||||
values.put("started_at", s.getLong("startedAt"));
|
||||
values.put("ended_at", s.getLong("endedAt"));
|
||||
if (s.has("audioOffsetMs")) {
|
||||
values.put("audio_offset_ms", s.getLong("audioOffsetMs"));
|
||||
}
|
||||
values.put("text", s.getString("text"));
|
||||
putOptString(values, s, "lang", "lang");
|
||||
putOptString(values, s, "translation", "translation");
|
||||
putOptString(values, s, "translationLang", "translation_lang");
|
||||
if (s.has("sentiment")) {
|
||||
values.put("sentiment", s.getDouble("sentiment"));
|
||||
}
|
||||
putOptString(values, s, "emotion", "emotion");
|
||||
putOptString(values, s, "searchMeta", "search_meta");
|
||||
if (s.has("angle")) {
|
||||
values.put("angle", s.getInt("angle"));
|
||||
}
|
||||
if (s.has("embeddingBase64")) {
|
||||
values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP));
|
||||
}
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
long id = db.insert("segments", null, values);
|
||||
db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?",
|
||||
new Object[]{sessionId});
|
||||
return id;
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "insertSegment failed", t);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column)
|
||||
throws JSONException {
|
||||
if (s.has(jsonKey) && !s.isNull(jsonKey)) {
|
||||
values.put(column, s.getString(jsonKey));
|
||||
}
|
||||
}
|
||||
|
||||
/** Update mutable fields of a segment (same keys as insertSegment). */
|
||||
public void updateSegment(long segmentId, String json) {
|
||||
try {
|
||||
JSONObject s = new JSONObject(json);
|
||||
ContentValues values = new ContentValues();
|
||||
if (s.has("speakerId")) {
|
||||
if (s.isNull("speakerId")) {
|
||||
values.putNull("speaker_id");
|
||||
} else {
|
||||
values.put("speaker_id", s.getLong("speakerId"));
|
||||
}
|
||||
}
|
||||
if (s.has("text")) {
|
||||
values.put("text", s.getString("text"));
|
||||
}
|
||||
putOptString(values, s, "lang", "lang");
|
||||
putOptString(values, s, "translation", "translation");
|
||||
putOptString(values, s, "translationLang", "translation_lang");
|
||||
if (s.has("sentiment")) {
|
||||
values.put("sentiment", s.getDouble("sentiment"));
|
||||
}
|
||||
putOptString(values, s, "emotion", "emotion");
|
||||
putOptString(values, s, "searchMeta", "search_meta");
|
||||
if (values.size() > 0) {
|
||||
getWritableDatabase().update("segments", values, "id=?",
|
||||
new String[]{String.valueOf(segmentId)});
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "updateSegment failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
public String querySegments(long sessionId) {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
|
||||
+ " translation, translation_lang, sentiment, emotion, angle"
|
||||
+ " FROM segments WHERE session_id=? ORDER BY started_at ASC",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
out.put(segmentRowJson(c));
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySegments failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
private JSONObject segmentRowJson(Cursor c) throws JSONException {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1));
|
||||
row.put("startedAt", c.getLong(2));
|
||||
row.put("endedAt", c.getLong(3));
|
||||
row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4));
|
||||
row.put("text", c.getString(5));
|
||||
row.put("lang", c.isNull(6) ? "" : c.getString(6));
|
||||
row.put("translation", c.isNull(7) ? "" : c.getString(7));
|
||||
row.put("translationLang", c.isNull(8) ? "" : c.getString(8));
|
||||
row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9));
|
||||
row.put("emotion", c.isNull(10) ? "" : c.getString(10));
|
||||
row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11));
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search segments across sessions. filterJson: query?, emotion?,
|
||||
* speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and
|
||||
* the sentiment search metadata.
|
||||
*/
|
||||
public String searchSegments(String filterJson) {
|
||||
try {
|
||||
JSONObject filter = filterJson == null || filterJson.isEmpty()
|
||||
? new JSONObject() : new JSONObject(filterJson);
|
||||
StringBuilder sql = new StringBuilder(
|
||||
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
|
||||
+ " translation, translation_lang, sentiment, emotion, angle, session_id"
|
||||
+ " FROM segments");
|
||||
List<String> args = new ArrayList<>();
|
||||
List<String> where = new ArrayList<>();
|
||||
if (filter.has("query") && !filter.getString("query").isEmpty()) {
|
||||
String like = "%" + filter.getString("query") + "%";
|
||||
where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)");
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
}
|
||||
if (filter.has("emotion")) {
|
||||
where.add("emotion = ?");
|
||||
args.add(filter.getString("emotion"));
|
||||
}
|
||||
if (filter.has("speakerId")) {
|
||||
where.add("speaker_id = ?");
|
||||
args.add(String.valueOf(filter.getLong("speakerId")));
|
||||
}
|
||||
if (filter.has("sinceMs")) {
|
||||
where.add("started_at >= ?");
|
||||
args.add(String.valueOf(filter.getLong("sinceMs")));
|
||||
}
|
||||
if (filter.has("untilMs")) {
|
||||
where.add("started_at <= ?");
|
||||
args.add(String.valueOf(filter.getLong("untilMs")));
|
||||
}
|
||||
if (!where.isEmpty()) {
|
||||
sql.append(" WHERE ").append(String.join(" AND ", where));
|
||||
}
|
||||
sql.append(" ORDER BY started_at DESC LIMIT ")
|
||||
.append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000)));
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = segmentRowJson(c);
|
||||
row.put("sessionId", c.getLong(12));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "searchSegments failed", e);
|
||||
return "[]";
|
||||
}
|
||||
}
|
||||
|
||||
/** Per-segment voice-print embeddings of a session (for re-diarization). */
|
||||
public String querySegmentEmbeddings(long sessionId) {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP));
|
||||
out.put(row);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySegmentEmbeddings failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
// ---- speakers ----
|
||||
|
||||
public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("name", name);
|
||||
values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color);
|
||||
values.put("is_wearer", isWearer ? 1 : 0);
|
||||
values.put("created_at", System.currentTimeMillis());
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
if (embeddingBase64 != null && !embeddingBase64.isEmpty()) {
|
||||
values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP));
|
||||
values.put("embedding_count", 1);
|
||||
}
|
||||
return getWritableDatabase().insert("speakers", null, values);
|
||||
}
|
||||
|
||||
public String querySpeakers() {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count,"
|
||||
+ " p.last_heard_at, p.tag,"
|
||||
+ " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id,"
|
||||
+ " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments"
|
||||
+ " FROM speakers p ORDER BY p.last_heard_at DESC", null)) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("name", c.getString(1));
|
||||
row.put("color", c.getString(2));
|
||||
row.put("isWearer", c.getInt(3) != 0);
|
||||
byte[] blob = c.isNull(4) ? null : c.getBlob(4);
|
||||
row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP));
|
||||
row.put("embeddingCount", c.getInt(5));
|
||||
row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6));
|
||||
row.put("tag", c.isNull(7) ? "" : c.getString(7));
|
||||
row.put("lastRecap", c.isNull(8) ? "" : c.getString(8));
|
||||
row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9));
|
||||
row.put("factsJson", c.isNull(10) ? "" : c.getString(10));
|
||||
row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11));
|
||||
row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12));
|
||||
row.put("segmentCount", c.getLong(13));
|
||||
out.put(row);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySpeakers failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
public void renameSpeaker(long speakerId, String name) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("name", name);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerColor(long speakerId, String color) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("color", color);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerTag(long speakerId, String tag) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("tag", tag);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
/**
|
||||
* Store LLM-derived conversation insights for a speaker: a recap of their
|
||||
* most recent conversation, its open action items (JSON string array), and
|
||||
* the accumulated inferred facts about them (JSON string array).
|
||||
*/
|
||||
public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson,
|
||||
String factsJson, long sessionId) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("last_recap", recap == null ? "" : recap);
|
||||
values.put("action_items", actionItemsJson == null ? "" : actionItemsJson);
|
||||
values.put("facts", factsJson == null ? "" : factsJson);
|
||||
values.put("insights_updated_at", System.currentTimeMillis());
|
||||
values.put("insights_session_id", sessionId);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerWearer(long speakerId, boolean isWearer) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
if (isWearer) {
|
||||
// Exactly one wearer profile at a time.
|
||||
db.execSQL("UPDATE speakers SET is_wearer = 0");
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("is_wearer", isWearer ? 1 : 0);
|
||||
db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
/**
|
||||
* Running-mean centroid update for a speaker's voice-print, capped so a
|
||||
* long session cannot drown the enrolled voice.
|
||||
*/
|
||||
public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) {
|
||||
try {
|
||||
byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP);
|
||||
float[] update = blobToFloats(incoming);
|
||||
if (update == null) {
|
||||
return;
|
||||
}
|
||||
float[] current = null;
|
||||
int count = 0;
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT embedding, embedding_count FROM speakers WHERE id=?",
|
||||
new String[]{String.valueOf(speakerId)})) {
|
||||
if (c.moveToFirst()) {
|
||||
current = c.isNull(0) ? null : blobToFloats(c.getBlob(0));
|
||||
count = c.getInt(1);
|
||||
}
|
||||
}
|
||||
float[] merged;
|
||||
if (current == null || current.length != update.length) {
|
||||
merged = update;
|
||||
count = 1;
|
||||
} else {
|
||||
int effective = Math.min(count, Math.max(1, maxCount));
|
||||
merged = new float[current.length];
|
||||
for (int i = 0; i < merged.length; i++) {
|
||||
merged[i] = (current[i] * effective + update[i]) / (effective + 1);
|
||||
}
|
||||
l2NormalizeInPlace(merged);
|
||||
count = count + 1;
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("embedding", floatsToBlob(merged));
|
||||
values.put("embedding_count", count);
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
getWritableDatabase().update("speakers", values, "id=?",
|
||||
new String[]{String.valueOf(speakerId)});
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "updateSpeakerEmbedding failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
/** Move every segment from one speaker onto another and drop the source. */
|
||||
public void mergeSpeakers(long fromId, long intoId) {
|
||||
if (fromId == intoId) {
|
||||
return;
|
||||
}
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
db.beginTransaction();
|
||||
try {
|
||||
db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?",
|
||||
new Object[]{intoId, fromId});
|
||||
// Blend the two centroids weighted by their sample counts.
|
||||
float[] from = null;
|
||||
float[] into = null;
|
||||
int fromCount = 0;
|
||||
int intoCount = 0;
|
||||
boolean fromWearer = false;
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)",
|
||||
new String[]{String.valueOf(fromId), String.valueOf(intoId)})) {
|
||||
while (c.moveToNext()) {
|
||||
long id = c.getLong(0);
|
||||
float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1));
|
||||
int count = c.getInt(2);
|
||||
if (id == fromId) {
|
||||
from = embedding;
|
||||
fromCount = count;
|
||||
fromWearer = c.getInt(3) != 0;
|
||||
} else {
|
||||
into = embedding;
|
||||
intoCount = count;
|
||||
}
|
||||
}
|
||||
}
|
||||
float[] merged = into;
|
||||
if (from != null && into != null && from.length == into.length) {
|
||||
merged = new float[into.length];
|
||||
int total = Math.max(1, fromCount + intoCount);
|
||||
for (int i = 0; i < merged.length; i++) {
|
||||
merged[i] = (into[i] * intoCount + from[i] * fromCount) / total;
|
||||
}
|
||||
l2NormalizeInPlace(merged);
|
||||
} else if (into == null) {
|
||||
merged = from;
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
if (merged != null) {
|
||||
values.put("embedding", floatsToBlob(merged));
|
||||
values.put("embedding_count", fromCount + intoCount);
|
||||
}
|
||||
if (fromWearer) {
|
||||
values.put("is_wearer", 1);
|
||||
}
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
// Keep the target's conversation insights; adopt the source's only
|
||||
// where the target has none (e.g. merging a rich profile into a
|
||||
// freshly auto-created one).
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id"
|
||||
+ " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END",
|
||||
new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) {
|
||||
boolean targetHasInsights = false;
|
||||
while (c.moveToNext()) {
|
||||
boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty();
|
||||
if (c.getPosition() == 0) {
|
||||
targetHasInsights = hasInsights;
|
||||
} else if (!targetHasInsights && hasInsights) {
|
||||
values.put("last_recap", c.getString(0));
|
||||
values.put("action_items", c.isNull(1) ? "" : c.getString(1));
|
||||
values.put("facts", c.isNull(2) ? "" : c.getString(2));
|
||||
values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3));
|
||||
values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4));
|
||||
}
|
||||
}
|
||||
}
|
||||
db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)});
|
||||
db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)});
|
||||
db.setTransactionSuccessful();
|
||||
} finally {
|
||||
db.endTransaction();
|
||||
}
|
||||
}
|
||||
|
||||
public void reassignSegmentSpeaker(long segmentId, long speakerId) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("speaker_id", speakerId);
|
||||
getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)});
|
||||
}
|
||||
|
||||
/** Reassign every segment of a speaker within one session (split support). */
|
||||
public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) {
|
||||
getWritableDatabase().execSQL(
|
||||
"UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?",
|
||||
new Object[]{toSpeakerId, sessionId, fromSpeakerId});
|
||||
}
|
||||
|
||||
public void deleteSpeaker(long speakerId) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId});
|
||||
db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
// ---- retention ----
|
||||
|
||||
/**
|
||||
* Delete caption sessions older than the cutoff (0 disables), and strip
|
||||
* recordings older than their own cutoff while keeping the transcript.
|
||||
* Returns a JSON summary of what was removed.
|
||||
*/
|
||||
public String applyRetention(long captionCutoffMs, long recordingCutoffMs) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
int sessionsDeleted = 0;
|
||||
int recordingsDeleted = 0;
|
||||
if (recordingCutoffMs > 0) {
|
||||
List<String> paths = new ArrayList<>();
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
|
||||
new String[]{String.valueOf(recordingCutoffMs)})) {
|
||||
while (c.moveToNext()) {
|
||||
if (!c.isNull(1)) {
|
||||
paths.add(c.getString(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String path : paths) {
|
||||
if (new File(path).delete()) {
|
||||
recordingsDeleted++;
|
||||
}
|
||||
}
|
||||
db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?",
|
||||
new Object[]{recordingCutoffMs});
|
||||
}
|
||||
if (captionCutoffMs > 0) {
|
||||
List<String> paths = new ArrayList<>();
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
|
||||
new String[]{String.valueOf(captionCutoffMs)})) {
|
||||
while (c.moveToNext()) {
|
||||
paths.add(c.getString(0));
|
||||
}
|
||||
}
|
||||
for (String path : paths) {
|
||||
if (path != null && new File(path).delete()) {
|
||||
recordingsDeleted++;
|
||||
}
|
||||
}
|
||||
db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)",
|
||||
new Object[]{captionCutoffMs});
|
||||
Cursor count = db.rawQuery("SELECT changes()", null);
|
||||
count.close();
|
||||
db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs});
|
||||
try (Cursor c = db.rawQuery("SELECT changes()", null)) {
|
||||
if (c.moveToFirst()) {
|
||||
sessionsDeleted = c.getInt(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
JSONObject summary = new JSONObject();
|
||||
summary.put("sessionsDeleted", sessionsDeleted);
|
||||
summary.put("recordingsDeleted", recordingsDeleted);
|
||||
return summary.toString();
|
||||
} catch (JSONException e) {
|
||||
return "{}";
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteSession(long sessionId) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
String path = getSessionAudioPath(sessionId);
|
||||
if (!path.isEmpty()) {
|
||||
new File(path).delete();
|
||||
}
|
||||
db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)});
|
||||
db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
// ---- blob helpers ----
|
||||
|
||||
private static float[] blobToFloats(byte[] blob) {
|
||||
if (blob == null || blob.length < 4 || blob.length % 4 != 0) {
|
||||
return null;
|
||||
}
|
||||
ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
|
||||
float[] out = new float[blob.length / 4];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
out[i] = buffer.getFloat();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static byte[] floatsToBlob(float[] values) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN);
|
||||
for (float v : values) {
|
||||
buffer.putFloat(v);
|
||||
}
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
private static void l2NormalizeInPlace(float[] v) {
|
||||
double sum = 0;
|
||||
for (float x : v) {
|
||||
sum += (double) x * x;
|
||||
}
|
||||
double norm = Math.sqrt(sum);
|
||||
if (norm <= 0) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < v.length; i++) {
|
||||
v[i] = (float) (v[i] / norm);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,9 +37,6 @@ import java.util.Set;
|
||||
* matching are decided in TypeScript (app/g2/even-advertisement.ts,
|
||||
* app/g2/pairing-candidates.ts) where they can be unit-tested; this filter
|
||||
* must stay a superset of the TS classifier's admission.
|
||||
*
|
||||
* Ported from the discovery path of SybilSight's Bluetooth SDK (G2.didDiscover
|
||||
* and R1's advertisement snapshot), reduced to what Android exposes.
|
||||
*/
|
||||
@SuppressLint("MissingPermission")
|
||||
public class FaceclawDeviceDiscovery {
|
||||
@@ -296,7 +293,7 @@ public class FaceclawDeviceDiscovery {
|
||||
/**
|
||||
* Android strips the 2-byte company identifier from each manufacturer
|
||||
* record and keys the SparseArray by it. Put the bytes back (little-endian)
|
||||
* so TypeScript sees the same "ER"+… layout the iOS SDK and HCI captures
|
||||
* so TypeScript sees the same "ER"+… layout the HCI captures
|
||||
* document. Prefers Even's own id; otherwise returns the first record so an
|
||||
* unexpected company id still reaches the diagnostics log.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FastClusteringConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.DataInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Offline re-diarization of a stored conversation recording (16-bit mono
|
||||
* WAV): pyannote segmentation plus the shared speaker-embedding model with
|
||||
* clustering, via sherpa-onnx. Runs on its own thread; results are handed to
|
||||
* TS as JSON turns for reconciliation against the transcript segments.
|
||||
*/
|
||||
public class FaceclawDiarizer {
|
||||
private static final String TAG = "FaceclawDiarizer";
|
||||
|
||||
public interface Listener {
|
||||
/** progress in [0, 1]. */
|
||||
void onProgress(double progress);
|
||||
|
||||
/** turns: [{startMs, endMs, cluster}], sorted by start time. */
|
||||
void onDone(String turnsJson);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
|
||||
private final String segmentationModelPath;
|
||||
private final String embeddingModelPath;
|
||||
private final String wavPath;
|
||||
private final int numClusters;
|
||||
private final float threshold;
|
||||
private final Listener listener;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private volatile boolean cancelled;
|
||||
|
||||
public FaceclawDiarizer(
|
||||
String segmentationModelPath,
|
||||
String embeddingModelPath,
|
||||
String wavPath,
|
||||
int numClusters,
|
||||
float threshold,
|
||||
Listener listener) {
|
||||
this.segmentationModelPath = segmentationModelPath;
|
||||
this.embeddingModelPath = embeddingModelPath;
|
||||
this.wavPath = wavPath;
|
||||
this.numClusters = numClusters;
|
||||
this.threshold = threshold <= 0 ? 0.5f : threshold;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
Thread thread = new Thread(this::run, "FaceclawDiarizer");
|
||||
thread.setPriority(Thread.MIN_PRIORITY);
|
||||
thread.start();
|
||||
}
|
||||
|
||||
public void cancel() {
|
||||
cancelled = true;
|
||||
}
|
||||
|
||||
private void run() {
|
||||
OfflineSpeakerDiarization diarization = null;
|
||||
try {
|
||||
float[] samples = wavPath.endsWith(".wav")
|
||||
? readWavMono16(wavPath)
|
||||
: decodeCompressedMono(wavPath);
|
||||
OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder()
|
||||
.setSegmentation(OfflineSpeakerSegmentationModelConfig.builder()
|
||||
.setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder()
|
||||
.setModel(segmentationModelPath)
|
||||
.build())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setEmbedding(SpeakerEmbeddingExtractorConfig.builder()
|
||||
.setModel(embeddingModelPath)
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setClustering(FastClusteringConfig.builder()
|
||||
.setNumClusters(numClusters > 0 ? numClusters : -1)
|
||||
.setThreshold(threshold)
|
||||
.build())
|
||||
.setMinDurationOn(0.3f)
|
||||
.setMinDurationOff(0.5f)
|
||||
.build();
|
||||
diarization = new OfflineSpeakerDiarization(config);
|
||||
OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback(
|
||||
samples,
|
||||
(numProcessedChunks, numTotalChunks, arg) -> {
|
||||
emitProgress(numTotalChunks <= 0 ? 0
|
||||
: (double) numProcessedChunks / numTotalChunks);
|
||||
return cancelled ? 1 : 0;
|
||||
});
|
||||
JSONArray turns = new JSONArray();
|
||||
if (segments != null) {
|
||||
for (OfflineSpeakerDiarizationSegment segment : segments) {
|
||||
JSONObject turn = new JSONObject();
|
||||
turn.put("startMs", Math.round(segment.getStart() * 1000));
|
||||
turn.put("endMs", Math.round(segment.getEnd() * 1000));
|
||||
turn.put("cluster", segment.getSpeaker());
|
||||
turns.put(turn);
|
||||
}
|
||||
}
|
||||
String result = turns.toString();
|
||||
mainHandler.post(() -> listener.onDone(result));
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "diarization failed for " + wavPath, t);
|
||||
String message = t.getMessage() == null ? t.toString() : t.getMessage();
|
||||
mainHandler.post(() -> listener.onError(message));
|
||||
} finally {
|
||||
if (diarization != null) {
|
||||
diarization.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void emitProgress(double progress) {
|
||||
mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a compressed recording (M4A/AAC from the transcode sweep) into
|
||||
* [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written
|
||||
* as 16 kHz mono, so no resampling is needed.
|
||||
*/
|
||||
private static float[] decodeCompressedMono(String path) throws IOException {
|
||||
android.media.MediaExtractor extractor = new android.media.MediaExtractor();
|
||||
android.media.MediaCodec codec = null;
|
||||
java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream();
|
||||
try {
|
||||
extractor.setDataSource(path);
|
||||
int trackIndex = -1;
|
||||
android.media.MediaFormat format = null;
|
||||
for (int i = 0; i < extractor.getTrackCount(); i++) {
|
||||
android.media.MediaFormat candidate = extractor.getTrackFormat(i);
|
||||
String mime = candidate.getString(android.media.MediaFormat.KEY_MIME);
|
||||
if (mime != null && mime.startsWith("audio/")) {
|
||||
trackIndex = i;
|
||||
format = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (trackIndex < 0 || format == null) {
|
||||
throw new IOException("no audio track in " + path);
|
||||
}
|
||||
extractor.selectTrack(trackIndex);
|
||||
codec = android.media.MediaCodec.createDecoderByType(
|
||||
format.getString(android.media.MediaFormat.KEY_MIME));
|
||||
codec.configure(format, null, null, 0);
|
||||
codec.start();
|
||||
android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo();
|
||||
boolean inputDone = false;
|
||||
boolean outputDone = false;
|
||||
while (!outputDone) {
|
||||
if (!inputDone) {
|
||||
int inIndex = codec.dequeueInputBuffer(10_000);
|
||||
if (inIndex >= 0) {
|
||||
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
|
||||
int size = extractor.readSampleData(inBuf, 0);
|
||||
if (size < 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, 0,
|
||||
android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM);
|
||||
inputDone = true;
|
||||
} else {
|
||||
codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0);
|
||||
extractor.advance();
|
||||
}
|
||||
}
|
||||
}
|
||||
int outIndex = codec.dequeueOutputBuffer(info, 10_000);
|
||||
if (outIndex >= 0) {
|
||||
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
|
||||
byte[] chunk = new byte[info.size];
|
||||
outBuf.position(info.offset);
|
||||
outBuf.get(chunk);
|
||||
pcm.write(chunk);
|
||||
codec.releaseOutputBuffer(outIndex, false);
|
||||
if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputDone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
extractor.release();
|
||||
if (codec != null) {
|
||||
try {
|
||||
codec.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
codec.release();
|
||||
}
|
||||
}
|
||||
byte[] bytes = pcm.toByteArray();
|
||||
int count = bytes.length / 2;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f;
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
|
||||
/** Load a 16-bit mono PCM WAV into [-1, 1] floats. */
|
||||
private static float[] readWavMono16(String path) throws IOException {
|
||||
File file = new File(path);
|
||||
long fileLength = file.length();
|
||||
try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
|
||||
byte[] header = new byte[44];
|
||||
in.readFully(header);
|
||||
if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') {
|
||||
throw new IOException("not a WAV file");
|
||||
}
|
||||
int channels = (header[22] & 0xff) | (header[23] << 8);
|
||||
int bits = (header[34] & 0xff) | (header[35] << 8);
|
||||
if (channels != 1 || bits != 16) {
|
||||
throw new IOException("only 16-bit mono WAV is supported");
|
||||
}
|
||||
long dataBytes = fileLength - 44;
|
||||
int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE);
|
||||
float[] samples = new float[count];
|
||||
byte[] chunk = new byte[65536];
|
||||
int sampleIndex = 0;
|
||||
int carried = 0;
|
||||
byte carriedByte = 0;
|
||||
int read;
|
||||
while (sampleIndex < count && (read = in.read(chunk)) > 0) {
|
||||
int offset = 0;
|
||||
if (carried == 1) {
|
||||
samples[sampleIndex++] =
|
||||
((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f;
|
||||
offset = 1;
|
||||
carried = 0;
|
||||
}
|
||||
int pairs = (read - offset) / 2;
|
||||
for (int i = 0; i < pairs && sampleIndex < count; i++) {
|
||||
int base = offset + i * 2;
|
||||
samples[sampleIndex++] =
|
||||
((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f;
|
||||
}
|
||||
if (((read - offset) & 1) == 1) {
|
||||
carried = 1;
|
||||
carriedByte = chunk[read - 1];
|
||||
}
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,12 @@ public class FaceclawLc3Decoder {
|
||||
public static final int COUNTER_OFFSET = 204;
|
||||
public static final int SAMPLES_PER_FRAME = 160;
|
||||
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
|
||||
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
|
||||
// glasses DSP appends two signed 16-bit LE values computed on the raw
|
||||
// stereo capture before the mono downmix — a signal-strength ratio and the
|
||||
// asin-derived direction-of-arrival in degrees — then the packet counter.
|
||||
public static final int SSR_OFFSET = 200;
|
||||
public static final int ANGLE_OFFSET = 202;
|
||||
|
||||
static {
|
||||
System.loadLibrary("faceclaw_lc3");
|
||||
@@ -23,6 +29,8 @@ public class FaceclawLc3Decoder {
|
||||
private long duplicatePackets;
|
||||
private long missingPackets;
|
||||
private long decodeErrors;
|
||||
private int lastSsr;
|
||||
private int lastAngleDegrees;
|
||||
|
||||
public FaceclawLc3Decoder() {
|
||||
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
|
||||
@@ -64,9 +72,21 @@ public class FaceclawLc3Decoder {
|
||||
}
|
||||
realPackets++;
|
||||
lastCounter = counter;
|
||||
lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8));
|
||||
lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8));
|
||||
return decoded;
|
||||
}
|
||||
|
||||
/** Firmware signal-strength ratio from the most recent decoded packet. */
|
||||
public synchronized int getLastSsr() {
|
||||
return lastSsr;
|
||||
}
|
||||
|
||||
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
|
||||
public synchronized int getLastAngleDegrees() {
|
||||
return lastAngleDegrees;
|
||||
}
|
||||
|
||||
public synchronized long getRealPackets() {
|
||||
return realPackets;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawMicStatusListener {
|
||||
/**
|
||||
* A CFW mic_control field-104 status record from one temple. body is the
|
||||
* raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is
|
||||
* "L" or "R" for the temple's own BLE link, "?" if unmatched.
|
||||
*/
|
||||
void onMicStatus(byte[] body, String arm);
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.media.audiofx.AcousticEchoCanceler;
|
||||
import android.media.audiofx.AudioEffect;
|
||||
import android.media.audiofx.AutomaticGainControl;
|
||||
import android.media.audiofx.NoiseSuppressor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Phone-side noise suppression for the microphone stream.
|
||||
*
|
||||
* Android's built-in noise cancellation (the {@link NoiseSuppressor} capture
|
||||
* effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio
|
||||
* arrives as raw PCM over BLE from the glasses, which that effect chain never
|
||||
* sees, so this class picks the best engine that can actually reach the
|
||||
* samples:
|
||||
*
|
||||
* - Audio captured by the phone itself: {@link #attachPlatformEffects(int)}
|
||||
* enables the device's built-in NS/AEC/AGC chain (hardware or OEM software,
|
||||
* whatever the Android build ships) on the AudioRecord session.
|
||||
* - Audio from the glasses (the normal case): {@link #process(byte[])} runs a
|
||||
* WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann
|
||||
* frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an
|
||||
* over-subtracted Wiener gain with a residual floor so speech never pumps
|
||||
* in and out of silence.
|
||||
*/
|
||||
public final class FaceclawNoiseSuppressor {
|
||||
|
||||
private static final int FFT_SIZE = 256;
|
||||
private static final int HOP = FFT_SIZE / 2;
|
||||
private static final float OVERSUBTRACTION = 1.6f;
|
||||
/** About -16 dB of residual: keeps the noise bed natural instead of gated. */
|
||||
private static final float GAIN_FLOOR = 0.15f;
|
||||
private static final float GAIN_SMOOTHING = 0.6f;
|
||||
/** Frames of pure noise-floor learning after start/reset (~100 ms). */
|
||||
private static final int NOISE_WARMUP_FRAMES = 12;
|
||||
|
||||
private final int sampleRate;
|
||||
private final float[] window = new float[FFT_SIZE];
|
||||
private final float[] history = new float[FFT_SIZE];
|
||||
private final float[] overlap = new float[HOP];
|
||||
private final float[] noise = new float[FFT_SIZE / 2 + 1];
|
||||
private final float[] gain = new float[FFT_SIZE / 2 + 1];
|
||||
private final float[] re = new float[FFT_SIZE];
|
||||
private final float[] im = new float[FFT_SIZE];
|
||||
private final short[] pending = new short[HOP];
|
||||
private int pendingCount = 0;
|
||||
private int warmupFrames = 0;
|
||||
|
||||
public FaceclawNoiseSuppressor(int sampleRate) {
|
||||
this.sampleRate = sampleRate;
|
||||
// sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap.
|
||||
for (int i = 0; i < FFT_SIZE; i++) {
|
||||
window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE);
|
||||
}
|
||||
reset();
|
||||
}
|
||||
|
||||
public int getSampleRate() {
|
||||
return sampleRate;
|
||||
}
|
||||
|
||||
/** Forget the learned noise floor (e.g. after the suppressor was toggled off). */
|
||||
public void reset() {
|
||||
Arrays.fill(history, 0f);
|
||||
Arrays.fill(overlap, 0f);
|
||||
Arrays.fill(noise, 0f);
|
||||
Arrays.fill(gain, 1f);
|
||||
pendingCount = 0;
|
||||
warmupFrames = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Suppress noise in 16-bit little-endian mono PCM. Streaming: input not
|
||||
* filling a whole 8 ms hop is carried into the next call, so the output
|
||||
* may be up to one hop shorter than the input (and longer on a later
|
||||
* call); total latency is one hop.
|
||||
*/
|
||||
public byte[] process(byte[] pcm16le) {
|
||||
if (pcm16le == null || pcm16le.length < 2) {
|
||||
return pcm16le;
|
||||
}
|
||||
int sampleCount = pcm16le.length / 2;
|
||||
int hops = (pendingCount + sampleCount) / HOP;
|
||||
byte[] out = new byte[hops * HOP * 2];
|
||||
int outPos = 0;
|
||||
int inPos = 0;
|
||||
while (inPos < sampleCount) {
|
||||
int take = Math.min(HOP - pendingCount, sampleCount - inPos);
|
||||
for (int i = 0; i < take; i++) {
|
||||
int lo = pcm16le[(inPos + i) * 2] & 0xff;
|
||||
int hi = pcm16le[(inPos + i) * 2 + 1];
|
||||
pending[pendingCount + i] = (short) ((hi << 8) | lo);
|
||||
}
|
||||
pendingCount += take;
|
||||
inPos += take;
|
||||
if (pendingCount == HOP) {
|
||||
outPos = processHop(out, outPos);
|
||||
pendingCount = 0;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private int processHop(byte[] out, int outPos) {
|
||||
System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP);
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
history[FFT_SIZE - HOP + i] = pending[i] / 32768f;
|
||||
}
|
||||
for (int i = 0; i < FFT_SIZE; i++) {
|
||||
re[i] = history[i] * window[i];
|
||||
im[i] = 0f;
|
||||
}
|
||||
fft(re, im, false);
|
||||
|
||||
boolean warming = warmupFrames < NOISE_WARMUP_FRAMES;
|
||||
if (warming) {
|
||||
warmupFrames++;
|
||||
}
|
||||
for (int b = 0; b <= FFT_SIZE / 2; b++) {
|
||||
float power = re[b] * re[b] + im[b] * im[b];
|
||||
if (warming) {
|
||||
noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames;
|
||||
} else if (power < noise[b]) {
|
||||
// Track the floor down quickly; drift up slowly so speech
|
||||
// energy never gets learned as noise.
|
||||
noise[b] += (power - noise[b]) * 0.2f;
|
||||
} else {
|
||||
noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f);
|
||||
}
|
||||
float clean = power - OVERSUBTRACTION * noise[b];
|
||||
float g = clean > 0f ? clean / power : 0f;
|
||||
if (g < GAIN_FLOOR) {
|
||||
g = GAIN_FLOOR;
|
||||
}
|
||||
g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g;
|
||||
gain[b] = g;
|
||||
re[b] *= g;
|
||||
im[b] *= g;
|
||||
if (b > 0 && b < FFT_SIZE / 2) {
|
||||
re[FFT_SIZE - b] *= g;
|
||||
im[FFT_SIZE - b] *= g;
|
||||
}
|
||||
}
|
||||
|
||||
fft(re, im, true);
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
float value = (overlap[i] + re[i] * window[i]) * 32767f;
|
||||
int sample = Math.round(value);
|
||||
if (sample > 32767) {
|
||||
sample = 32767;
|
||||
}
|
||||
if (sample < -32768) {
|
||||
sample = -32768;
|
||||
}
|
||||
out[outPos + i * 2] = (byte) (sample & 0xff);
|
||||
out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff);
|
||||
}
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
overlap[i] = re[HOP + i] * window[HOP + i];
|
||||
}
|
||||
return outPos + HOP * 2;
|
||||
}
|
||||
|
||||
/** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */
|
||||
private static void fft(float[] re, float[] im, boolean inverse) {
|
||||
int n = re.length;
|
||||
for (int i = 1, j = 0; i < n; i++) {
|
||||
int bit = n >> 1;
|
||||
for (; (j & bit) != 0; bit >>= 1) {
|
||||
j ^= bit;
|
||||
}
|
||||
j ^= bit;
|
||||
if (i < j) {
|
||||
float t = re[i];
|
||||
re[i] = re[j];
|
||||
re[j] = t;
|
||||
t = im[i];
|
||||
im[i] = im[j];
|
||||
im[j] = t;
|
||||
}
|
||||
}
|
||||
for (int len = 2; len <= n; len <<= 1) {
|
||||
double ang = 2 * Math.PI / len * (inverse ? 1 : -1);
|
||||
float wRe = (float) Math.cos(ang);
|
||||
float wIm = (float) Math.sin(ang);
|
||||
for (int i = 0; i < n; i += len) {
|
||||
float curRe = 1f;
|
||||
float curIm = 0f;
|
||||
for (int k = 0; k < len / 2; k++) {
|
||||
int a = i + k;
|
||||
int b = a + len / 2;
|
||||
float xr = re[b] * curRe - im[b] * curIm;
|
||||
float xi = re[b] * curIm + im[b] * curRe;
|
||||
re[b] = re[a] - xr;
|
||||
im[b] = im[a] - xi;
|
||||
re[a] += xr;
|
||||
im[a] += xi;
|
||||
float nextRe = curRe * wRe - curIm * wIm;
|
||||
curIm = curRe * wIm + curIm * wRe;
|
||||
curRe = nextRe;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inverse) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
re[i] /= n;
|
||||
im[i] /= n;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this Android build ships the built-in capture noise suppressor. */
|
||||
public static boolean platformSuppressorAvailable() {
|
||||
try {
|
||||
return NoiseSuppressor.isAvailable();
|
||||
} catch (Throwable t) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Short engine name for the UI. */
|
||||
public static String engineDescription() {
|
||||
return "spectral (phone DSP)";
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach the platform's built-in capture-effect chain (NoiseSuppressor,
|
||||
* AcousticEchoCanceler, AutomaticGainControl — whichever this Android
|
||||
* build provides) to a phone AudioRecord session. Only applicable when the
|
||||
* phone itself captures the audio; the BLE glasses stream must go through
|
||||
* {@link #process(byte[])} instead. The caller keeps the returned effects
|
||||
* for the life of the recording and release()s them with it.
|
||||
*/
|
||||
public static AudioEffect[] attachPlatformEffects(int audioSessionId) {
|
||||
ArrayList<AudioEffect> effects = new ArrayList<>();
|
||||
try {
|
||||
if (NoiseSuppressor.isAvailable()) {
|
||||
NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId);
|
||||
if (ns != null) {
|
||||
ns.setEnabled(true);
|
||||
effects.add(ns);
|
||||
}
|
||||
}
|
||||
if (AcousticEchoCanceler.isAvailable()) {
|
||||
AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId);
|
||||
if (aec != null) {
|
||||
aec.setEnabled(true);
|
||||
effects.add(aec);
|
||||
}
|
||||
}
|
||||
if (AutomaticGainControl.isAvailable()) {
|
||||
AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId);
|
||||
if (agc != null) {
|
||||
agc.setEnabled(true);
|
||||
effects.add(agc);
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
// A missing effect just means this tier is unavailable.
|
||||
}
|
||||
return effects.toArray(new AudioEffect[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.OnlineStream;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
|
||||
|
||||
/**
|
||||
* Speaker voice-print extraction over sherpa-onnx. One embedding per
|
||||
* utterance; matching/enrollment happens on the TS side (cosine similarity
|
||||
* against the stored speaker profiles).
|
||||
*/
|
||||
public class FaceclawSpeakerId {
|
||||
private static final String TAG = "FaceclawSpeakerId";
|
||||
|
||||
private SpeakerEmbeddingExtractor extractor;
|
||||
private final String modelPath;
|
||||
|
||||
public FaceclawSpeakerId(String modelPath) {
|
||||
this.modelPath = modelPath;
|
||||
}
|
||||
|
||||
/** Lazily loads the model; returns false when it cannot be loaded. */
|
||||
public synchronized boolean ensureLoaded() {
|
||||
if (extractor != null) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder()
|
||||
.setModel(modelPath)
|
||||
.setNumThreads(1)
|
||||
.setDebug(false)
|
||||
.setProvider("cpu")
|
||||
.build();
|
||||
extractor = new SpeakerEmbeddingExtractor(config);
|
||||
return true;
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "could not load speaker model at " + modelPath, t);
|
||||
extractor = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized int embeddingDim() {
|
||||
return extractor == null ? 0 : extractor.getDim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute an L2-normalized voice-print for one utterance of 16 kHz mono
|
||||
* S16LE PCM. Returns null when the model is unavailable or the utterance
|
||||
* is too short to embed (under ~0.5 s).
|
||||
*/
|
||||
public synchronized float[] embed(byte[] pcm16le, int sampleRate) {
|
||||
if (pcm16le == null || pcm16le.length < sampleRate) {
|
||||
return null;
|
||||
}
|
||||
if (!ensureLoaded()) {
|
||||
return null;
|
||||
}
|
||||
int count = pcm16le.length / 2;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8));
|
||||
samples[i] = s / 32768.0f;
|
||||
}
|
||||
OnlineStream stream = null;
|
||||
try {
|
||||
stream = extractor.createStream();
|
||||
stream.acceptWaveform(samples, sampleRate);
|
||||
stream.inputFinished();
|
||||
if (!extractor.isReady(stream)) {
|
||||
return null;
|
||||
}
|
||||
float[] embedding = extractor.compute(stream);
|
||||
return l2Normalize(embedding);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "embedding failed", t);
|
||||
return null;
|
||||
} finally {
|
||||
if (stream != null) {
|
||||
stream.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] l2Normalize(float[] v) {
|
||||
if (v == null) {
|
||||
return null;
|
||||
}
|
||||
double sum = 0;
|
||||
for (float x : v) {
|
||||
sum += (double) x * x;
|
||||
}
|
||||
double norm = Math.sqrt(sum);
|
||||
if (norm <= 0) {
|
||||
return v;
|
||||
}
|
||||
for (int i = 0; i < v.length; i++) {
|
||||
v[i] = (float) (v[i] / norm);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
if (extractor != null) {
|
||||
extractor.release();
|
||||
extractor = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -90,6 +90,26 @@ public class FaceclawVoiceController {
|
||||
private volatile boolean endpointing;
|
||||
private final EndpointDetector endpointDetector = new EndpointDetector();
|
||||
private java.io.ByteArrayOutputStream recordingPcm;
|
||||
// Speaker verification against the enrolled wearer voice-print ("my voice
|
||||
// only" command gating). Configured before start(); the utterance PCM is
|
||||
// buffered (capped) and verified once at session end.
|
||||
private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10;
|
||||
private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE;
|
||||
private volatile String verifySpeakerModelPath;
|
||||
private volatile float[] verifyWearerEmbedding;
|
||||
private volatile float verifyThreshold = 0.8f;
|
||||
private short[] verifyBuffer;
|
||||
private int verifyCount;
|
||||
// Global mic processing (Microphones app config): spectral noise
|
||||
// suppression and firmware-DoA beam gating, applied to every capture
|
||||
// session that opts in (assistant push-to-talk, Transcribe, hands-free).
|
||||
// The raw tap opts out — raw means raw, and the Microphones session does
|
||||
// its own beam-compensated processing on that path.
|
||||
private volatile boolean suppressionEnabled;
|
||||
private volatile boolean beamFilterEnabled;
|
||||
private volatile int beamCenterDeg;
|
||||
private volatile int beamHalfWidthDeg = 180;
|
||||
private FaceclawNoiseSuppressor suppressor;
|
||||
private long queuedPackets;
|
||||
private long queueDroppedPackets;
|
||||
private long decodedSamples;
|
||||
@@ -125,6 +145,68 @@ public class FaceclawVoiceController {
|
||||
this.endpointing = endpointing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify this session's speaker against the enrolled wearer voice-print
|
||||
* and report the result via onSpeakerVerified just before the final
|
||||
* transcript. Must be set before {@link #start}; pass a null model path
|
||||
* to disable.
|
||||
*/
|
||||
public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) {
|
||||
this.verifySpeakerModelPath = speakerModelPath;
|
||||
this.verifyWearerEmbedding = wearerEmbedding;
|
||||
this.verifyThreshold = threshold > 0 ? threshold : 0.8f;
|
||||
}
|
||||
|
||||
public void clearSpeakerVerification() {
|
||||
this.verifySpeakerModelPath = null;
|
||||
this.verifyWearerEmbedding = null;
|
||||
}
|
||||
|
||||
/** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */
|
||||
public void setNoiseSuppression(boolean enabled) {
|
||||
this.suppressionEnabled = enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Direction gating from the Sonic Radar beam: packets whose firmware
|
||||
* direction-of-arrival falls outside centerDeg ± halfWidthDeg (device
|
||||
* frame, 0 = straight ahead, positive right) are dropped before any
|
||||
* consumer sees them. Safe to update mid-run.
|
||||
*/
|
||||
public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) {
|
||||
this.beamFilterEnabled = enabled;
|
||||
this.beamCenterDeg = centerDeg;
|
||||
this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg));
|
||||
}
|
||||
|
||||
private boolean withinBeam(int angleDegrees) {
|
||||
int delta = angleDegrees - beamCenterDeg;
|
||||
while (delta > 180) delta -= 360;
|
||||
while (delta < -180) delta += 360;
|
||||
return Math.abs(delta) <= beamHalfWidthDeg;
|
||||
}
|
||||
|
||||
private void applySuppression(short[] pcm, int count) {
|
||||
try {
|
||||
if (suppressor == null) {
|
||||
suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE);
|
||||
}
|
||||
byte[] le = new byte[count * 2];
|
||||
for (int i = 0; i < count; i++) {
|
||||
le[i * 2] = (byte) (pcm[i] & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff);
|
||||
}
|
||||
byte[] cleaned = suppressor.process(le);
|
||||
int cleanedCount = Math.min(count, cleaned.length / 2);
|
||||
for (int i = 0; i < cleanedCount; i++) {
|
||||
pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8));
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "noise suppression failed; passing audio through", t);
|
||||
suppressionEnabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
public void start(String requestedMode) {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
@@ -195,7 +277,13 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
lc3Decoder = new FaceclawLc3Decoder();
|
||||
endpointDetector.reset();
|
||||
if (suppressor != null) {
|
||||
suppressor.reset();
|
||||
}
|
||||
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
|
||||
boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null;
|
||||
verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null;
|
||||
verifyCount = 0;
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.");
|
||||
return;
|
||||
@@ -205,6 +293,10 @@ public class FaceclawVoiceController {
|
||||
? "Listening (cloud)..."
|
||||
: "Listening...");
|
||||
processG2Audio();
|
||||
// Verification result must precede the final transcript so the
|
||||
// TS bridge can suppress a non-wearer command before it is acted
|
||||
// on (the callbacks are posted in order to the main handler).
|
||||
runSpeakerVerification();
|
||||
// Button released / stop requested: emit one final full-utterance
|
||||
// transcript so the UI can freeze it.
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
@@ -374,15 +466,38 @@ public class FaceclawVoiceController {
|
||||
continue;
|
||||
}
|
||||
decodedSamples += count;
|
||||
// Global mic processing from the Microphones app config: the
|
||||
// beam filter drops packets whose firmware direction-of-arrival
|
||||
// falls outside the listening wedge (isolating the aimed talker
|
||||
// for every consumer, recognition included), and the spectral
|
||||
// noise suppressor cleans what remains before it reaches the
|
||||
// recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
int angleDegrees = currentDecoder.getLastAngleDegrees();
|
||||
int ssr = currentDecoder.getLastSsr();
|
||||
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
|
||||
emitFrameMeta(angleDegrees, ssr);
|
||||
maybeEmitAudioStats(false);
|
||||
continue;
|
||||
}
|
||||
if (suppressionEnabled) {
|
||||
applySuppression(pcm, count);
|
||||
}
|
||||
if (recordingPcm != null) {
|
||||
appendRecording(pcm, count);
|
||||
}
|
||||
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
|
||||
int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount);
|
||||
System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied);
|
||||
verifyCount += copied;
|
||||
}
|
||||
if (endpointing && endpointDetector.accept(pcm, count)) {
|
||||
emitSpeechEnd();
|
||||
}
|
||||
if (mode == VoiceInputMode.CLOUD) {
|
||||
emitPcm(pcm, count);
|
||||
} else {
|
||||
// PCM and frame metadata flow in every mode so levels, recording,
|
||||
// and the Microphones radar keep working alongside onboard ASR.
|
||||
emitPcm(pcm, count);
|
||||
emitFrameMeta(angleDegrees, ssr);
|
||||
if (mode != VoiceInputMode.CLOUD) {
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
@@ -582,6 +697,83 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(() -> currentListener.onPcm(le));
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed the session's buffered utterance and compare it to the enrolled
|
||||
* wearer voice-print. Fails open: a session too short to verify, or a
|
||||
* model that will not load, counts as the wearer rather than silencing
|
||||
* every command.
|
||||
*/
|
||||
private void runSpeakerVerification() {
|
||||
short[] buffer = verifyBuffer;
|
||||
float[] wearer = verifyWearerEmbedding;
|
||||
String modelPath = verifySpeakerModelPath;
|
||||
verifyBuffer = null;
|
||||
if (buffer == null || wearer == null || modelPath == null) {
|
||||
return;
|
||||
}
|
||||
if (verifyCount < VERIFY_MIN_SAMPLES) {
|
||||
emitSpeakerVerified(true, 0f);
|
||||
return;
|
||||
}
|
||||
FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath);
|
||||
try {
|
||||
byte[] le = new byte[verifyCount * 2];
|
||||
for (int i = 0; i < verifyCount; i++) {
|
||||
short s = buffer[i];
|
||||
le[i * 2] = (byte) (s & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
|
||||
}
|
||||
float[] embedding = speakerId.embed(le, SAMPLE_RATE);
|
||||
if (embedding == null || embedding.length != wearer.length) {
|
||||
emitSpeakerVerified(true, 0f);
|
||||
return;
|
||||
}
|
||||
double dot = 0;
|
||||
for (int i = 0; i < embedding.length; i++) {
|
||||
dot += (double) embedding[i] * wearer[i];
|
||||
}
|
||||
boolean isWearer = dot >= verifyThreshold;
|
||||
Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot)
|
||||
+ " threshold=" + verifyThreshold + " isWearer=" + isWearer);
|
||||
emitSpeakerVerified(isWearer, (float) dot);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "speaker verification failed", t);
|
||||
emitSpeakerVerified(true, 0f);
|
||||
}
|
||||
}
|
||||
|
||||
// The 28 MB embedding model takes seconds to load; keep one instance
|
||||
// across capture sessions so verification adds only the embed time.
|
||||
private static FaceclawSpeakerId sharedSpeakerId;
|
||||
private static String sharedSpeakerIdPath;
|
||||
|
||||
private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) {
|
||||
if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) {
|
||||
if (sharedSpeakerId != null) {
|
||||
sharedSpeakerId.close();
|
||||
}
|
||||
sharedSpeakerId = new FaceclawSpeakerId(modelPath);
|
||||
sharedSpeakerIdPath = modelPath;
|
||||
}
|
||||
return sharedSpeakerId;
|
||||
}
|
||||
|
||||
private void emitSpeakerVerified(boolean isWearer, float similarity) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity));
|
||||
}
|
||||
|
||||
private void emitFrameMeta(int angleDegrees, int ssr) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr));
|
||||
}
|
||||
|
||||
private void emitSpeechEnd() {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
|
||||
+21
-2
@@ -11,15 +11,34 @@ public interface FaceclawVoiceControllerListener {
|
||||
void onTranscript(String text, boolean isFinal);
|
||||
|
||||
/**
|
||||
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
|
||||
* little-endian PCM. Empty/absent in onboard mode.
|
||||
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
|
||||
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
|
||||
* keep working while the onboard recognizer owns the transcript.
|
||||
*/
|
||||
void onPcm(byte[] pcm16le);
|
||||
|
||||
/**
|
||||
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
|
||||
* packet, computed on the raw stereo capture before its mono downmix: the
|
||||
* direction-of-arrival in signed degrees (0 = straight ahead, positive to
|
||||
* the wearer's right) and a signal-strength ratio that serves as a
|
||||
* confidence/energy proxy for the angle.
|
||||
*/
|
||||
void onFrameMeta(int angleDegrees, int ssr);
|
||||
|
||||
/**
|
||||
* The speaker stopped, in a hands-free session that has no button release
|
||||
* to end it. Only fires when endpointing was enabled for the session; see
|
||||
* FaceclawVoiceController.setEndpointing.
|
||||
*/
|
||||
void onSpeechEnd();
|
||||
|
||||
/**
|
||||
* Result of speaker verification against the enrolled wearer voice-print,
|
||||
* emitted once per capture session just before the final transcript.
|
||||
* Only fires when verification was configured for the session; fails
|
||||
* open (isWearer=true) when the utterance was too short to verify or the
|
||||
* speaker model could not be loaded.
|
||||
*/
|
||||
void onSpeakerVerified(boolean isWearer, float similarity);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* Streaming WAV writer for conversation recordings: the header is written up
|
||||
* front with zero sizes and patched on finish, so a crash mid-session leaves
|
||||
* a recoverable file (data length can be derived from the file size).
|
||||
*/
|
||||
public class FaceclawWavRecorder {
|
||||
private static final String TAG = "FaceclawWavRecorder";
|
||||
|
||||
private final String path;
|
||||
private final int sampleRate;
|
||||
private final int channels;
|
||||
private RandomAccessFile file;
|
||||
private long dataBytes;
|
||||
|
||||
public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException {
|
||||
this.path = path;
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
File parent = new File(path).getParentFile();
|
||||
if (parent != null && !parent.exists() && !parent.mkdirs()) {
|
||||
throw new IOException("could not create " + parent);
|
||||
}
|
||||
file = new RandomAccessFile(path, "rw");
|
||||
file.setLength(0);
|
||||
file.write(buildHeader(0));
|
||||
}
|
||||
|
||||
public synchronized void append(byte[] pcm16le) {
|
||||
if (file == null || pcm16le == null || pcm16le.length == 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
file.write(pcm16le);
|
||||
dataBytes += pcm16le.length;
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "append failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized long getDurationMs() {
|
||||
return dataBytes * 1000L / (sampleRate * channels * 2L);
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Patch the header and close. Returns the PCM byte count (0 = empty). */
|
||||
public synchronized long finish() {
|
||||
RandomAccessFile current = file;
|
||||
file = null;
|
||||
if (current == null) {
|
||||
return dataBytes;
|
||||
}
|
||||
try {
|
||||
current.seek(0);
|
||||
current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE)));
|
||||
current.close();
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "finish failed", e);
|
||||
}
|
||||
return dataBytes;
|
||||
}
|
||||
|
||||
/** Delete the (finished or abandoned) recording. */
|
||||
public synchronized void discard() {
|
||||
finish();
|
||||
new File(path).delete();
|
||||
}
|
||||
|
||||
private byte[] buildHeader(int pcmBytes) {
|
||||
int byteRate = sampleRate * channels * 2;
|
||||
ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN);
|
||||
b.put("RIFF".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(36 + pcmBytes);
|
||||
b.put("WAVE".getBytes(StandardCharsets.US_ASCII));
|
||||
b.put("fmt ".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(16);
|
||||
b.putShort((short) 1);
|
||||
b.putShort((short) channels);
|
||||
b.putInt(sampleRate);
|
||||
b.putInt(byteRate);
|
||||
b.putShort((short) (channels * 2));
|
||||
b.putShort((short) 16);
|
||||
b.put("data".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(pcmBytes);
|
||||
return b.array();
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,14 @@ public class BleProtocol {
|
||||
// protobuf fields to stock implementations, so they are safely ignored.
|
||||
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
|
||||
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
|
||||
/**
|
||||
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
|
||||
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
|
||||
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
|
||||
* to settings read responses and as standalone commandId=3 pushes.
|
||||
*/
|
||||
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
|
||||
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
|
||||
public static final int FACECLAW_WAKE_OP_ACQUIRE = 1;
|
||||
public static final int FACECLAW_WAKE_OP_RELEASE = 2;
|
||||
public static final int FACECLAW_WAKE_OP_CLAIM = 3;
|
||||
@@ -405,6 +413,40 @@ public class BleProtocol {
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a
|
||||
* settings write carrying CFW field 103. Magic zero makes it
|
||||
* fire-and-forget like the wake lease: the firmware consumes the field
|
||||
* before its stock nanopb decoder discards the unknown tag, and the
|
||||
* "ack" is the field-104 status notify that follows.
|
||||
*/
|
||||
public static byte[] buildFaceclawMicControl(byte[] record) {
|
||||
return concat(CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1),
|
||||
encodeVarintField(2, 0),
|
||||
encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the CFW mic status record (field 104) from a settings frame, or
|
||||
* null when absent. The 21-byte body layout is decoded on the TS side;
|
||||
* here we only validate the 'M','C',version prelude.
|
||||
*/
|
||||
public static byte[] parseFaceclawMicStatus(byte[] pb) {
|
||||
if (pb == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD);
|
||||
if (body == null
|
||||
|| body.length < 3
|
||||
|| body[0] != 'M'
|
||||
|| body[1] != 'C') {
|
||||
return null;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
|
||||
* when this is an ordinary settings frame.
|
||||
|
||||
@@ -267,6 +267,21 @@ public class MessageBuilder {
|
||||
);
|
||||
}
|
||||
|
||||
/** CFW mic_control write (settings field 103) targeted at one temple. */
|
||||
public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) {
|
||||
return new OutboundMessage(
|
||||
"mic-control",
|
||||
"mic control " + label + (leftArm ? " L" : " R"),
|
||||
BleProtocol.SID_UI_SETTING,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
0,
|
||||
BleProtocol.buildFaceclawMicControl(record),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
leftArm
|
||||
);
|
||||
}
|
||||
|
||||
public OutboundMessage faceclawWearQuery(boolean leftArm) {
|
||||
return new OutboundMessage(
|
||||
"wear-query-control",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class FastClusteringConfig {
|
||||
private final int numClusters;
|
||||
private final float threshold;
|
||||
|
||||
private FastClusteringConfig(Builder builder) {
|
||||
this.numClusters = builder.numClusters;
|
||||
this.threshold = builder.threshold;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public int getNumClusters() {
|
||||
return numClusters;
|
||||
}
|
||||
|
||||
public float getThreshold() {
|
||||
return threshold;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private int numClusters = -1;
|
||||
private float threshold = 0.5f;
|
||||
|
||||
public FastClusteringConfig build() {
|
||||
return new FastClusteringConfig(this);
|
||||
}
|
||||
|
||||
public Builder setNumClusters(int numClusters) {
|
||||
this.numClusters = numClusters;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setThreshold(float threshold) {
|
||||
this.threshold = threshold;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarization {
|
||||
private long ptr = 0;
|
||||
|
||||
public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization");
|
||||
}
|
||||
}
|
||||
|
||||
public int getSampleRate() {
|
||||
return getSampleRate(ptr);
|
||||
}
|
||||
|
||||
// Only config.clustering is used. All other fields are ignored
|
||||
public void setConfig(OfflineSpeakerDiarizationConfig config) {
|
||||
setConfig(ptr, config);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] process(float[] samples) {
|
||||
return process(ptr, samples);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) {
|
||||
return processWithCallback(ptr, samples, callback, 0);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) {
|
||||
return processWithCallback(ptr, samples, callback, arg);
|
||||
}
|
||||
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
// You'd better call it manually if it is not used anymore
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
private native int getSampleRate(long ptr);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long newFromFile(OfflineSpeakerDiarizationConfig config);
|
||||
|
||||
private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config);
|
||||
|
||||
private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples);
|
||||
|
||||
private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg);
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface OfflineSpeakerDiarizationCallback {
|
||||
Integer invoke(int numProcessedChunks, int numTotalCunks, long arg);
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarizationConfig {
|
||||
private final OfflineSpeakerSegmentationModelConfig segmentation;
|
||||
private final SpeakerEmbeddingExtractorConfig embedding;
|
||||
private final FastClusteringConfig clustering;
|
||||
private final float minDurationOn;
|
||||
private final float minDurationOff;
|
||||
|
||||
private OfflineSpeakerDiarizationConfig(Builder builder) {
|
||||
this.segmentation = builder.segmentation;
|
||||
this.embedding = builder.embedding;
|
||||
this.clustering = builder.clustering;
|
||||
this.minDurationOff = builder.minDurationOff;
|
||||
this.minDurationOn = builder.minDurationOn;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public OfflineSpeakerSegmentationModelConfig getSegmentation() {
|
||||
return segmentation;
|
||||
}
|
||||
|
||||
public SpeakerEmbeddingExtractorConfig getEmbedding() {
|
||||
return embedding;
|
||||
}
|
||||
|
||||
public FastClusteringConfig getClustering() {
|
||||
return clustering;
|
||||
}
|
||||
|
||||
public float getMinDurationOff() {
|
||||
return minDurationOff;
|
||||
}
|
||||
|
||||
public float getMinDurationOn() {
|
||||
return minDurationOn;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build();
|
||||
private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build();
|
||||
private FastClusteringConfig clustering = FastClusteringConfig.builder().build();
|
||||
private float minDurationOn = 0.2f;
|
||||
private float minDurationOff = 0.5f;
|
||||
|
||||
public OfflineSpeakerDiarizationConfig build() {
|
||||
return new OfflineSpeakerDiarizationConfig(this);
|
||||
}
|
||||
|
||||
public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) {
|
||||
this.segmentation = segmentation;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) {
|
||||
this.embedding = embedding;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setClustering(FastClusteringConfig clustering) {
|
||||
this.clustering = clustering;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setMinDurationOff(float minDurationOff) {
|
||||
this.minDurationOff = minDurationOff;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setMinDurationOn(float minDurationOn) {
|
||||
this.minDurationOn = minDurationOn;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarizationSegment {
|
||||
private final float start;
|
||||
private final float end;
|
||||
private final int speaker;
|
||||
|
||||
public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.speaker = speaker;
|
||||
}
|
||||
|
||||
public float getStart() {
|
||||
return start;
|
||||
}
|
||||
|
||||
public float getEnd() {
|
||||
return end;
|
||||
}
|
||||
|
||||
public int getSpeaker() {
|
||||
return speaker;
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerSegmentationModelConfig {
|
||||
private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
|
||||
private OfflineSpeakerSegmentationModelConfig(Builder builder) {
|
||||
this.pyannote = builder.pyannote;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build();
|
||||
private int numThreads = 1;
|
||||
private boolean debug = true;
|
||||
private String provider = "cpu";
|
||||
|
||||
public OfflineSpeakerSegmentationModelConfig build() {
|
||||
return new OfflineSpeakerSegmentationModelConfig(this);
|
||||
}
|
||||
|
||||
public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) {
|
||||
this.pyannote = pyannote;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDebug(boolean debug) {
|
||||
this.debug = debug;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerSegmentationPyannoteModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public OfflineSpeakerSegmentationPyannoteModelConfig build() {
|
||||
return new OfflineSpeakerSegmentationPyannoteModelConfig(this);
|
||||
}
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright 2022-2023 by zhaoming
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineStream {
|
||||
private long ptr = 0;
|
||||
|
||||
public OnlineStream() {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public OnlineStream(long ptr) {
|
||||
this.ptr = ptr;
|
||||
}
|
||||
|
||||
public long getPtr() {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
public void setPtr(long ptr) {
|
||||
this.ptr = ptr;
|
||||
}
|
||||
|
||||
public void acceptWaveform(float[] samples, int sampleRate) {
|
||||
acceptWaveform(this.ptr, samples, sampleRate);
|
||||
}
|
||||
|
||||
public void inputFinished() {
|
||||
inputFinished(this.ptr);
|
||||
}
|
||||
|
||||
public void setOption(String key, String value) {
|
||||
setOption(this.ptr, key, value);
|
||||
}
|
||||
|
||||
public String getOption(String key) {
|
||||
return getOption(this.ptr, key);
|
||||
}
|
||||
|
||||
public boolean hasOption(String key) {
|
||||
return hasOption(this.ptr, key);
|
||||
}
|
||||
|
||||
public void release() {
|
||||
close();
|
||||
}
|
||||
|
||||
public void close() {
|
||||
// stream object must be release after used
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
close();
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
|
||||
|
||||
private native void inputFinished(long ptr);
|
||||
|
||||
private native void setOption(long ptr, String key, String value);
|
||||
|
||||
private native String getOption(long ptr, String key);
|
||||
|
||||
private native boolean hasOption(long ptr, String key);
|
||||
|
||||
private native void delete(long ptr);
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingExtractor {
|
||||
private long ptr = 0;
|
||||
|
||||
public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public OnlineStream createStream() {
|
||||
long p = createStream(ptr);
|
||||
return new OnlineStream(p);
|
||||
}
|
||||
|
||||
public boolean isReady(OnlineStream s) {
|
||||
return isReady(ptr, s.getPtr());
|
||||
}
|
||||
|
||||
public float[] compute(OnlineStream s) {
|
||||
return compute(ptr, s.getPtr());
|
||||
}
|
||||
|
||||
public int getDim() {
|
||||
return dim(ptr);
|
||||
}
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long newFromFile(SpeakerEmbeddingExtractorConfig config);
|
||||
|
||||
private native long createStream(long ptr);
|
||||
|
||||
private native boolean isReady(long ptr, long streamPtr);
|
||||
|
||||
private native float[] compute(long ptr, long streamPtr);
|
||||
|
||||
private native int dim(long ptr);
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingExtractorConfig {
|
||||
private final String model;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
|
||||
private SpeakerEmbeddingExtractorConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
private int numThreads = 1;
|
||||
private boolean debug = true;
|
||||
private String provider = "cpu";
|
||||
|
||||
public SpeakerEmbeddingExtractorConfig build() {
|
||||
return new SpeakerEmbeddingExtractorConfig(this);
|
||||
}
|
||||
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDebug(boolean debug) {
|
||||
this.debug = debug;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingManager {
|
||||
private long ptr = 0;
|
||||
|
||||
public SpeakerEmbeddingManager(int dim) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = create(dim);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public boolean add(String name, float[] embedding) {
|
||||
return add(ptr, name, embedding);
|
||||
}
|
||||
|
||||
public boolean add(String name, float[][] embedding) {
|
||||
return addList(ptr, name, embedding);
|
||||
}
|
||||
|
||||
public boolean remove(String name) {
|
||||
return remove(ptr, name);
|
||||
}
|
||||
|
||||
public String search(float[] embedding, float threshold) {
|
||||
return search(ptr, embedding, threshold);
|
||||
}
|
||||
|
||||
public boolean verify(String name, float[] embedding, float threshold) {
|
||||
return verify(ptr, name, embedding, threshold);
|
||||
}
|
||||
|
||||
public boolean contains(String name) {
|
||||
return contains(ptr, name);
|
||||
}
|
||||
|
||||
public int getNumSpeakers() {
|
||||
return numSpeakers(ptr);
|
||||
}
|
||||
|
||||
public String[] getAllSpeakerNames() {
|
||||
return allSpeakerNames(ptr);
|
||||
}
|
||||
|
||||
private native long create(int dim);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native boolean add(long ptr, String name, float[] embedding);
|
||||
|
||||
private native boolean addList(long ptr, String name, float[][] embedding);
|
||||
|
||||
private native boolean remove(long ptr, String name);
|
||||
|
||||
private native String search(long ptr, float[] embedding, float threshold);
|
||||
|
||||
private native boolean verify(long ptr, String name, float[] embedding, float threshold);
|
||||
|
||||
private native boolean contains(long ptr, String name);
|
||||
|
||||
private native int numSpeakers(long ptr);
|
||||
|
||||
private native String[] allSpeakerNames(long ptr);
|
||||
}
|
||||
Reference in new issue
Block a user