forked from Blendman974/faceclaw
Merge branch 'main' into ble-bandwidth-test
This commit is contained in:
commit
0114c521ad
316 files changed
+32980
-882
No files matched your search
@@ -0,0 +1,242 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodecInfo;
|
||||
import android.media.MediaFormat;
|
||||
import android.media.MediaMuxer;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
|
||||
/**
|
||||
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
|
||||
* container, cutting speech recordings to roughly a tenth of their WAV size.
|
||||
* Runs on its own thread; the source WAV is deleted on success when requested.
|
||||
*/
|
||||
public class FaceclawAudioTranscoder {
|
||||
private static final String TAG = "FaceclawTranscode";
|
||||
private static final long CODEC_TIMEOUT_US = 10_000;
|
||||
|
||||
public interface Listener {
|
||||
void onDone(String outputPath, long outputBytes);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
|
||||
private final String wavPath;
|
||||
private final String outputPath;
|
||||
private final int bitrateBps;
|
||||
private final boolean deleteSourceOnSuccess;
|
||||
private final Listener listener;
|
||||
|
||||
public FaceclawAudioTranscoder(
|
||||
String wavPath,
|
||||
String outputPath,
|
||||
int bitrateBps,
|
||||
boolean deleteSourceOnSuccess,
|
||||
Listener listener) {
|
||||
this.wavPath = wavPath;
|
||||
this.outputPath = outputPath;
|
||||
this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000;
|
||||
this.deleteSourceOnSuccess = deleteSourceOnSuccess;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
Thread thread = new Thread(this::run, "FaceclawTranscode");
|
||||
thread.setPriority(Thread.MIN_PRIORITY);
|
||||
thread.start();
|
||||
}
|
||||
|
||||
private void run() {
|
||||
try {
|
||||
transcode();
|
||||
long size = new File(outputPath).length();
|
||||
if (deleteSourceOnSuccess) {
|
||||
new File(wavPath).delete();
|
||||
}
|
||||
if (listener != null) {
|
||||
listener.onDone(outputPath, size);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "transcode failed for " + wavPath, t);
|
||||
new File(outputPath).delete();
|
||||
if (listener != null) {
|
||||
listener.onError(t.getMessage() == null ? t.toString() : t.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void transcode() throws IOException {
|
||||
WavInfo wav = readWavHeader(wavPath);
|
||||
MediaFormat format = MediaFormat.createAudioFormat(
|
||||
MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels);
|
||||
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
|
||||
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps);
|
||||
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
|
||||
|
||||
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
|
||||
MediaMuxer muxer = null;
|
||||
FileInputStream input = null;
|
||||
try {
|
||||
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
|
||||
codec.start();
|
||||
muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
|
||||
input = new FileInputStream(wavPath);
|
||||
long skipped = 0;
|
||||
while (skipped < wav.dataOffset) {
|
||||
long step = input.skip(wav.dataOffset - skipped);
|
||||
if (step <= 0) {
|
||||
throw new IOException("could not seek past WAV header");
|
||||
}
|
||||
skipped += step;
|
||||
}
|
||||
|
||||
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
|
||||
byte[] chunk = new byte[8192];
|
||||
int track = -1;
|
||||
boolean muxerStarted = false;
|
||||
boolean inputDone = false;
|
||||
boolean outputDone = false;
|
||||
long remaining = wav.dataBytes;
|
||||
long presentationUs = 0;
|
||||
long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2;
|
||||
|
||||
while (!outputDone) {
|
||||
if (!inputDone) {
|
||||
int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US);
|
||||
if (inIndex >= 0) {
|
||||
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
|
||||
int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity()));
|
||||
int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1;
|
||||
if (read <= 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, presentationUs,
|
||||
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
|
||||
inputDone = true;
|
||||
} else {
|
||||
inBuf.clear();
|
||||
inBuf.put(chunk, 0, read);
|
||||
codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0);
|
||||
remaining -= read;
|
||||
presentationUs += read * 1_000_000L / bytesPerSecond;
|
||||
}
|
||||
}
|
||||
}
|
||||
int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US);
|
||||
if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
track = muxer.addTrack(codec.getOutputFormat());
|
||||
muxer.start();
|
||||
muxerStarted = true;
|
||||
} else if (outIndex >= 0) {
|
||||
if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
|
||||
if (!muxerStarted) {
|
||||
throw new IOException("encoder produced data before format");
|
||||
}
|
||||
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
|
||||
outBuf.position(info.offset);
|
||||
outBuf.limit(info.offset + info.size);
|
||||
muxer.writeSampleData(track, outBuf, info);
|
||||
}
|
||||
codec.releaseOutputBuffer(outIndex, false);
|
||||
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputDone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (input != null) {
|
||||
try {
|
||||
input.close();
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
try {
|
||||
codec.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
codec.release();
|
||||
if (muxer != null) {
|
||||
try {
|
||||
muxer.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
muxer.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final class WavInfo {
|
||||
int sampleRate;
|
||||
int channels;
|
||||
long dataOffset;
|
||||
long dataBytes;
|
||||
}
|
||||
|
||||
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
|
||||
private static WavInfo readWavHeader(String path) throws IOException {
|
||||
try (FileInputStream in = new FileInputStream(path)) {
|
||||
byte[] head = new byte[12];
|
||||
if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') {
|
||||
throw new IOException("not a WAV file");
|
||||
}
|
||||
WavInfo info = new WavInfo();
|
||||
long offset = 12;
|
||||
byte[] chunkHead = new byte[8];
|
||||
while (in.read(chunkHead) == 8) {
|
||||
ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN);
|
||||
int id = b.getInt();
|
||||
long size = b.getInt() & 0xffffffffL;
|
||||
offset += 8;
|
||||
if (id == 0x20746d66) { // "fmt "
|
||||
byte[] fmt = new byte[(int) Math.min(size, 16)];
|
||||
if (in.read(fmt) != fmt.length) {
|
||||
throw new IOException("truncated fmt chunk");
|
||||
}
|
||||
ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN);
|
||||
int audioFormat = f.getShort();
|
||||
info.channels = f.getShort();
|
||||
info.sampleRate = f.getInt();
|
||||
f.getInt();
|
||||
f.getShort();
|
||||
int bits = f.getShort();
|
||||
if (audioFormat != 1 || bits != 16) {
|
||||
throw new IOException("only 16-bit PCM WAV is supported");
|
||||
}
|
||||
long skip = size - fmt.length;
|
||||
while (skip > 0) {
|
||||
long step = in.skip(skip);
|
||||
if (step <= 0) {
|
||||
break;
|
||||
}
|
||||
skip -= step;
|
||||
}
|
||||
offset += size;
|
||||
} else if (id == 0x61746164) { // "data"
|
||||
info.dataOffset = offset;
|
||||
// A zero size means the header was never patched (crash
|
||||
// mid-recording); recover the length from the file size.
|
||||
info.dataBytes = size > 0 ? size : new File(path).length() - offset;
|
||||
break;
|
||||
} else {
|
||||
long skip = size;
|
||||
while (skip > 0) {
|
||||
long step = in.skip(skip);
|
||||
if (step <= 0) {
|
||||
break;
|
||||
}
|
||||
skip -= step;
|
||||
}
|
||||
offset += size;
|
||||
}
|
||||
}
|
||||
if (info.sampleRate == 0 || info.dataOffset == 0) {
|
||||
throw new IOException("missing fmt or data chunk");
|
||||
}
|
||||
return info;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private final java.util.List<FaceclawCompassListener> compassListeners =
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private volatile String lastFirmwareCapabilities = "";
|
||||
private volatile Thread workerThread;
|
||||
private volatile boolean running;
|
||||
private volatile boolean userDisconnectRequested;
|
||||
@@ -389,6 +392,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the glasses mic is enabled right now. The enable lives in the
|
||||
* current EvenHub session, so it dies with a transport drop, the charging
|
||||
* case, or a suspend — silently, from the phone's point of view. Callers
|
||||
* that track a capture across those events must check this rather than
|
||||
* assume their earlier enable still holds.
|
||||
*/
|
||||
public boolean isAudioCaptureActive() {
|
||||
synchronized (lock) {
|
||||
return running && sessionReady && !shutdownRequested && audioCaptureActive;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire/renew or release CFW's volatile wake-takeover lease on both
|
||||
* arms. Delivery (not a protocol ACK) is awaited so a caller can ensure
|
||||
@@ -743,6 +759,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
|
||||
public String getFirmwareCapabilities() {
|
||||
return lastFirmwareCapabilities;
|
||||
}
|
||||
|
||||
public void addMicStatusListener(FaceclawMicStatusListener listener) {
|
||||
if (listener != null) {
|
||||
micStatusListeners.add(listener);
|
||||
}
|
||||
}
|
||||
|
||||
public void removeMicStatusListener(FaceclawMicStatusListener listener) {
|
||||
if (listener != null) {
|
||||
micStatusListeners.remove(listener);
|
||||
}
|
||||
}
|
||||
|
||||
private void emitMicStatus(byte[] body, String address) {
|
||||
if (micStatusListeners.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String arm = address.equalsIgnoreCase(leftAddress) ? "L"
|
||||
: address.equalsIgnoreCase(rightAddress) ? "R" : "?";
|
||||
byte[] copy = java.util.Arrays.copyOf(body, body.length);
|
||||
mainHandler.post(() -> {
|
||||
for (FaceclawMicStatusListener micListener : micStatusListeners) {
|
||||
try {
|
||||
micListener.onMicStatus(copy, arm);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "mic status listener failed", t);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
|
||||
* field-103 write to both temples, or to a single one. Fire-and-forget:
|
||||
* the firmware answers with a field-104 status notify per temple, which
|
||||
* arrives through addMicStatusListener.
|
||||
*/
|
||||
public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) {
|
||||
if (record == null || record.length < 4) {
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (!running || !sessionReady) {
|
||||
logLine("skip mic control (" + label + "); session not ready");
|
||||
return;
|
||||
}
|
||||
if (rightTemple) {
|
||||
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false));
|
||||
}
|
||||
if (leftTemple) {
|
||||
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true));
|
||||
}
|
||||
logLine("queue mic control " + label);
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward render-characteristic audio packets to the listener WITHOUT
|
||||
* sending the stock EvenHub audio-control enable. Used for the CFW
|
||||
* mic_control streaming path, where capture is armed through settings
|
||||
* field 103 and the temples emit 'SM' frames on the same characteristic
|
||||
* that stock mono LC3 uses. Returns false when no session is up.
|
||||
*/
|
||||
public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) {
|
||||
if (listener == null) {
|
||||
throw new IllegalArgumentException("listener is required");
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (!running || !sessionReady || shutdownRequested) {
|
||||
logLine("skip G2 audio forwarding; session not ready");
|
||||
return false;
|
||||
}
|
||||
audioPacketListener = listener;
|
||||
audioCaptureActive = true;
|
||||
logLine("G2 audio forwarding enabled");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public void stopG2AudioForwarding() {
|
||||
synchronized (lock) {
|
||||
audioPacketListener = null;
|
||||
audioCaptureActive = false;
|
||||
logLine("G2 audio forwarding disabled");
|
||||
}
|
||||
}
|
||||
|
||||
public void addCompassListener(FaceclawCompassListener listener) {
|
||||
if (listener != null) {
|
||||
compassListeners.add(listener);
|
||||
@@ -777,12 +885,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
* foreground), including worker-app frames the TS side never sees.
|
||||
*/
|
||||
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) {
|
||||
return getCompositePreviewBitmap(brightenGamma, false);
|
||||
}
|
||||
|
||||
/** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */
|
||||
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) {
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return null;
|
||||
}
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma);
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green);
|
||||
}
|
||||
|
||||
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
|
||||
@@ -1433,6 +1546,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
0
|
||||
);
|
||||
}
|
||||
if (!faceclawWakeNotification
|
||||
&& frame.ok
|
||||
&& frame.sid == BleProtocol.SID_UI_SETTING) {
|
||||
// CFW mic status (field 104) rides both standalone pushes and
|
||||
// settings read acks, from each temple on its own link.
|
||||
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
|
||||
if (micStatus != null) {
|
||||
emitMicStatus(micStatus, address);
|
||||
}
|
||||
}
|
||||
if (!faceclawWakeNotification
|
||||
&& decodedWearState < 0
|
||||
&& frame.ok
|
||||
@@ -2716,6 +2839,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
|
||||
if (firmwareInfo != null) {
|
||||
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
|
||||
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
|
||||
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
|
||||
&& hasCapability(firmwareInfo.capabilities, "texstr14");
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FeatureConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineStream;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Continuous captioning over the decoded mic PCM: segments speech into
|
||||
* utterances with an adaptive energy gate, transcribes each utterance with
|
||||
* the on-device Moonshine model, and attaches a speaker voice-print per
|
||||
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
|
||||
* and any beam-direction gating), so this engine has no BLE dependencies.
|
||||
*
|
||||
* Utterance boundaries are measured on the sample clock, not the wall clock:
|
||||
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
|
||||
* audio heard. The TS side converts startMs/endMs to wall-clock times using
|
||||
* the engine start timestamp.
|
||||
*/
|
||||
public class FaceclawCaptionEngine {
|
||||
private static final String TAG = "FaceclawCaptions";
|
||||
private static final int SAMPLE_RATE = 16000;
|
||||
private static final int FEATURE_DIM = 80;
|
||||
private static final int MAX_QUEUE_PACKETS = 200;
|
||||
|
||||
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
|
||||
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
|
||||
private static final double ONSET_FACTOR = 3.0;
|
||||
private static final double RELEASE_FACTOR = 1.8;
|
||||
private static final double MIN_RMS = 220.0;
|
||||
private static final double NOISE_EMA_ALPHA = 0.05;
|
||||
private static final int PRE_ROLL_MS = 400;
|
||||
private static final int DEFAULT_SILENCE_MS = 800;
|
||||
private static final int MIN_UTTERANCE_MS = 350;
|
||||
private static final int MAX_UTTERANCE_MS = 15000;
|
||||
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
|
||||
// segments cut at the quietest window (mirrors FaceclawVoiceController).
|
||||
private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
|
||||
private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2;
|
||||
private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000;
|
||||
private static final float NORMALIZE_TARGET_PEAK = 0.9f;
|
||||
private static final float NORMALIZE_MAX_GAIN = 30f;
|
||||
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
|
||||
private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10;
|
||||
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private final Object lock = new Object();
|
||||
private final Object queueLock = new Object();
|
||||
private final ArrayDeque<byte[]> queue = new ArrayDeque<>();
|
||||
|
||||
private volatile FaceclawCaptionEngineListener listener;
|
||||
private volatile String asrModelDir;
|
||||
private volatile String speakerModelPath;
|
||||
private volatile int silenceMs = DEFAULT_SILENCE_MS;
|
||||
private Thread workerThread;
|
||||
private volatile boolean started;
|
||||
|
||||
private OfflineRecognizer recognizer;
|
||||
private FaceclawSpeakerId speakerId;
|
||||
|
||||
// Segmentation state (worker thread only).
|
||||
private long totalSamples;
|
||||
private double noiseFloor;
|
||||
private boolean inUtterance;
|
||||
private long utteranceStartSample;
|
||||
private long silenceRunSamples;
|
||||
private double utterancePeakRms;
|
||||
private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)];
|
||||
private int utteranceLength;
|
||||
private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000];
|
||||
private int preRollLength;
|
||||
|
||||
public void setListener(FaceclawCaptionEngineListener listener) {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
/** Directory holding the Moonshine model files, or null to disable ASR. */
|
||||
public void setAsrModelDir(String dir) {
|
||||
this.asrModelDir = dir;
|
||||
}
|
||||
|
||||
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
|
||||
public void setSpeakerModelPath(String path) {
|
||||
this.speakerModelPath = path;
|
||||
}
|
||||
|
||||
public void setSilenceMs(int ms) {
|
||||
this.silenceMs = Math.max(200, Math.min(3000, ms));
|
||||
}
|
||||
|
||||
public void start() {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
Thread threadToJoin;
|
||||
synchronized (lock) {
|
||||
if (!started) {
|
||||
return;
|
||||
}
|
||||
started = false;
|
||||
threadToJoin = workerThread;
|
||||
workerThread = null;
|
||||
}
|
||||
synchronized (queueLock) {
|
||||
queueLock.notifyAll();
|
||||
}
|
||||
if (threadToJoin != null) {
|
||||
threadToJoin.interrupt();
|
||||
if (Thread.currentThread() != threadToJoin) {
|
||||
try {
|
||||
threadToJoin.join(2000);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
|
||||
public void acceptPcm(byte[] pcm16le) {
|
||||
if (!started || pcm16le == null || pcm16le.length < 2) {
|
||||
return;
|
||||
}
|
||||
synchronized (queueLock) {
|
||||
if (queue.size() >= MAX_QUEUE_PACKETS) {
|
||||
queue.removeFirst();
|
||||
}
|
||||
queue.addLast(pcm16le);
|
||||
queueLock.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
try {
|
||||
loadModels();
|
||||
resetSegmentation();
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
byte[] chunk = takeChunk();
|
||||
if (chunk == null) {
|
||||
continue;
|
||||
}
|
||||
processChunk(chunk);
|
||||
}
|
||||
// Flush a trailing utterance so its text isn't lost on stop.
|
||||
if (inUtterance && utteranceLength > 0) {
|
||||
finalizeUtterance();
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.e(TAG, "caption engine failed", t);
|
||||
emitStatus("Captions failed: " + t.getMessage());
|
||||
} finally {
|
||||
releaseModels();
|
||||
}
|
||||
}
|
||||
|
||||
private void loadModels() {
|
||||
String modelDir = asrModelDir;
|
||||
if (modelDir != null && new File(modelDir, "tokens.txt").exists()) {
|
||||
emitStatus("Loading caption model...");
|
||||
recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setModelConfig(OfflineModelConfig.builder()
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.build());
|
||||
} else {
|
||||
recognizer = null;
|
||||
}
|
||||
String embedModel = speakerModelPath;
|
||||
if (embedModel != null && new File(embedModel).exists()) {
|
||||
speakerId = new FaceclawSpeakerId(embedModel);
|
||||
speakerId.ensureLoaded();
|
||||
} else {
|
||||
speakerId = null;
|
||||
}
|
||||
emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)...");
|
||||
}
|
||||
|
||||
private void releaseModels() {
|
||||
if (recognizer != null) {
|
||||
recognizer.release();
|
||||
recognizer = null;
|
||||
}
|
||||
if (speakerId != null) {
|
||||
speakerId.close();
|
||||
speakerId = null;
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] takeChunk() {
|
||||
synchronized (queueLock) {
|
||||
while (started && queue.isEmpty()) {
|
||||
try {
|
||||
queueLock.wait(250);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return queue.pollFirst();
|
||||
}
|
||||
}
|
||||
|
||||
private void resetSegmentation() {
|
||||
totalSamples = 0;
|
||||
noiseFloor = 0;
|
||||
inUtterance = false;
|
||||
utteranceLength = 0;
|
||||
preRollLength = 0;
|
||||
silenceRunSamples = 0;
|
||||
utterancePeakRms = 0;
|
||||
}
|
||||
|
||||
private void processChunk(byte[] chunk) {
|
||||
int count = chunk.length / 2;
|
||||
short[] pcm = new short[count];
|
||||
double sumSquares = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8));
|
||||
pcm[i] = s;
|
||||
sumSquares += (double) s * s;
|
||||
}
|
||||
double rms = Math.sqrt(sumSquares / Math.max(1, count));
|
||||
totalSamples += count;
|
||||
|
||||
if (!inUtterance) {
|
||||
// Track the noise floor only while idle so speech doesn't raise it.
|
||||
noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA;
|
||||
double threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
if (rms >= threshold * ONSET_FACTOR) {
|
||||
inUtterance = true;
|
||||
utteranceLength = 0;
|
||||
silenceRunSamples = 0;
|
||||
utterancePeakRms = rms;
|
||||
appendUtterance(preRoll, preRollLength);
|
||||
utteranceStartSample = Math.max(0, totalSamples - count - preRollLength);
|
||||
appendUtterance(pcm, count);
|
||||
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE);
|
||||
} else {
|
||||
appendPreRoll(pcm, count);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
appendUtterance(pcm, count);
|
||||
utterancePeakRms = Math.max(utterancePeakRms, rms);
|
||||
double threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
if (rms < threshold * RELEASE_FACTOR) {
|
||||
silenceRunSamples += count;
|
||||
} else {
|
||||
silenceRunSamples = 0;
|
||||
}
|
||||
long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE;
|
||||
boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs;
|
||||
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
|
||||
finalizeUtterance();
|
||||
inUtterance = false;
|
||||
preRollLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private void appendPreRoll(short[] pcm, int count) {
|
||||
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
|
||||
if (count >= preRoll.length) {
|
||||
System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length);
|
||||
preRollLength = preRoll.length;
|
||||
return;
|
||||
}
|
||||
int keep = Math.min(preRollLength, preRoll.length - count);
|
||||
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep);
|
||||
System.arraycopy(pcm, 0, preRoll, keep, count);
|
||||
preRollLength = keep + count;
|
||||
}
|
||||
|
||||
private void appendUtterance(short[] pcm, int count) {
|
||||
int room = utterance.length - utteranceLength;
|
||||
int copied = Math.min(room, count);
|
||||
if (copied > 0) {
|
||||
System.arraycopy(pcm, 0, utterance, utteranceLength, copied);
|
||||
utteranceLength += copied;
|
||||
}
|
||||
}
|
||||
|
||||
private void finalizeUtterance() {
|
||||
int length = utteranceLength;
|
||||
long startMs = utteranceStartSample * 1000 / SAMPLE_RATE;
|
||||
long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE;
|
||||
if (endMs - startMs < MIN_UTTERANCE_MS) {
|
||||
return;
|
||||
}
|
||||
String text = recognizeUtterance(length);
|
||||
float[] embedding = embedUtterance(length);
|
||||
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms);
|
||||
}
|
||||
|
||||
private String recognizeUtterance(int length) {
|
||||
if (recognizer == null || length <= 0) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder joined = new StringBuilder();
|
||||
int offset = 0;
|
||||
while (offset < length) {
|
||||
int remaining = length - offset;
|
||||
int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES);
|
||||
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
|
||||
segment = findQuietCut(offset, segment);
|
||||
}
|
||||
String part = recognizeRange(offset, segment);
|
||||
if (part.length() > 0) {
|
||||
if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) {
|
||||
joined.append(' ');
|
||||
}
|
||||
joined.append(part);
|
||||
}
|
||||
offset += segment;
|
||||
}
|
||||
return joined.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* End a decode segment at the center of the quietest window near its end
|
||||
* so the cut lands between words (same approach as the PTT controller).
|
||||
*/
|
||||
private int findQuietCut(int offset, int segment) {
|
||||
int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES);
|
||||
int win = CUT_WINDOW_SAMPLES;
|
||||
if (segment - searchStart <= win) {
|
||||
return segment;
|
||||
}
|
||||
double sum = 0;
|
||||
for (int i = searchStart; i < searchStart + win; i++) {
|
||||
double s = utterance[offset + i];
|
||||
sum += s * s;
|
||||
}
|
||||
double best = sum;
|
||||
int bestStart = searchStart;
|
||||
for (int start = searchStart + 1; start + win <= segment; start++) {
|
||||
double dropped = utterance[offset + start - 1];
|
||||
double added = utterance[offset + start + win - 1];
|
||||
sum += added * added - dropped * dropped;
|
||||
if (sum < best) {
|
||||
best = sum;
|
||||
bestStart = start;
|
||||
}
|
||||
}
|
||||
return bestStart + win / 2;
|
||||
}
|
||||
|
||||
private String recognizeRange(int offset, int count) {
|
||||
float[] samples = new float[count];
|
||||
float peak = 0f;
|
||||
for (int i = 0; i < count; i++) {
|
||||
float v = utterance[offset + i] / 32768.0f;
|
||||
samples[i] = v;
|
||||
float a = Math.abs(v);
|
||||
if (a > peak) {
|
||||
peak = a;
|
||||
}
|
||||
}
|
||||
if (peak > 0f) {
|
||||
float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN);
|
||||
if (gain > 1f) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] *= gain;
|
||||
}
|
||||
}
|
||||
}
|
||||
OfflineStream stream = recognizer.createStream();
|
||||
try {
|
||||
stream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
recognizer.decode(stream);
|
||||
OfflineRecognizerResult result = recognizer.getResult(stream);
|
||||
String raw = result == null ? "" : result.getText();
|
||||
return raw == null ? "" : raw.trim();
|
||||
} finally {
|
||||
stream.release();
|
||||
}
|
||||
}
|
||||
|
||||
private float[] embedUtterance(int length) {
|
||||
FaceclawSpeakerId currentSpeakerId = speakerId;
|
||||
if (currentSpeakerId == null || length <= 0) {
|
||||
return null;
|
||||
}
|
||||
int count = Math.min(length, EMBED_MAX_SAMPLES);
|
||||
byte[] le = new byte[count * 2];
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = utterance[i];
|
||||
le[i * 2] = (byte) (s & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
|
||||
}
|
||||
return currentSpeakerId.embed(le, SAMPLE_RATE);
|
||||
}
|
||||
|
||||
private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length);
|
||||
mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms));
|
||||
}
|
||||
|
||||
private void emitSpeechStart(long startMs) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onSpeechStart(startMs));
|
||||
}
|
||||
|
||||
private void emitStatus(String status) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onStatus(status));
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawCaptionEngineListener {
|
||||
/**
|
||||
* One finished utterance. text may be empty when no ASR model is loaded
|
||||
* or recognition produced nothing; embedding is the L2-normalized
|
||||
* voice-print or null when the speaker model is unavailable or the
|
||||
* utterance was too short. startMs/endMs are on the engine's sample
|
||||
* clock (milliseconds of audio since start()).
|
||||
*/
|
||||
void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms);
|
||||
|
||||
/** A speech onset was detected; the utterance is now being collected. */
|
||||
void onSpeechStart(long startMs);
|
||||
|
||||
void onStatus(String status);
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.ContentValues;
|
||||
import android.content.Context;
|
||||
import android.database.Cursor;
|
||||
import android.database.sqlite.SQLiteDatabase;
|
||||
import android.database.sqlite.SQLiteOpenHelper;
|
||||
import android.util.Base64;
|
||||
import android.util.Log;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Persistent store for caption sessions, transcript segments, and speaker
|
||||
* voice profiles, backing the Microphones app and its phone-side review UI.
|
||||
* All complex values cross the JS bridge as JSON strings; voice-print
|
||||
* embeddings are float32 LE blobs, exposed as base64.
|
||||
*/
|
||||
public class FaceclawConversationStore extends SQLiteOpenHelper {
|
||||
private static final String TAG = "FaceclawConvStore";
|
||||
private static final String DB_NAME = "faceclaw-conversations.db";
|
||||
private static final int DB_VERSION = 2;
|
||||
|
||||
private static FaceclawConversationStore instance;
|
||||
|
||||
public static synchronized FaceclawConversationStore getInstance(Context context) {
|
||||
if (instance == null) {
|
||||
instance = new FaceclawConversationStore(context.getApplicationContext());
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
private FaceclawConversationStore(Context context) {
|
||||
super(context, DB_NAME, null, DB_VERSION);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCreate(SQLiteDatabase db) {
|
||||
db.execSQL("CREATE TABLE speakers ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "name TEXT NOT NULL,"
|
||||
+ "color TEXT NOT NULL DEFAULT '#4FC3F7',"
|
||||
+ "is_wearer INTEGER NOT NULL DEFAULT 0,"
|
||||
+ "embedding BLOB,"
|
||||
+ "embedding_count INTEGER NOT NULL DEFAULT 0,"
|
||||
+ "created_at INTEGER NOT NULL,"
|
||||
+ "last_heard_at INTEGER,"
|
||||
+ "tag TEXT,"
|
||||
+ "last_recap TEXT,"
|
||||
+ "action_items TEXT,"
|
||||
+ "facts TEXT,"
|
||||
+ "insights_updated_at INTEGER,"
|
||||
+ "insights_session_id INTEGER)");
|
||||
db.execSQL("CREATE TABLE sessions ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "started_at INTEGER NOT NULL,"
|
||||
+ "ended_at INTEGER,"
|
||||
+ "title TEXT,"
|
||||
+ "audio_path TEXT,"
|
||||
+ "audio_codec TEXT,"
|
||||
+ "avg_sentiment REAL,"
|
||||
+ "segment_count INTEGER NOT NULL DEFAULT 0)");
|
||||
db.execSQL("CREATE TABLE segments ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "session_id INTEGER NOT NULL,"
|
||||
+ "speaker_id INTEGER,"
|
||||
+ "started_at INTEGER NOT NULL,"
|
||||
+ "ended_at INTEGER NOT NULL,"
|
||||
+ "audio_offset_ms INTEGER,"
|
||||
+ "text TEXT NOT NULL,"
|
||||
+ "lang TEXT,"
|
||||
+ "translation TEXT,"
|
||||
+ "translation_lang TEXT,"
|
||||
+ "sentiment REAL,"
|
||||
+ "emotion TEXT,"
|
||||
+ "search_meta TEXT,"
|
||||
+ "angle INTEGER,"
|
||||
+ "embedding BLOB)");
|
||||
db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)");
|
||||
db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)");
|
||||
db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)");
|
||||
db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
|
||||
if (oldVersion < 2) {
|
||||
// v2: per-speaker conversation insights (recap of the most recent
|
||||
// conversation, its action items, and accumulated inferred facts).
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER");
|
||||
}
|
||||
}
|
||||
|
||||
// ---- sessions ----
|
||||
|
||||
public long startSession(long startedAtMs, String title) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("started_at", startedAtMs);
|
||||
values.put("title", title == null || title.isEmpty() ? null : title);
|
||||
return getWritableDatabase().insert("sessions", null, values);
|
||||
}
|
||||
|
||||
public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("ended_at", endedAtMs);
|
||||
if (audioPath != null && !audioPath.isEmpty()) {
|
||||
values.put("audio_path", audioPath);
|
||||
values.put("audio_codec", audioCodec);
|
||||
}
|
||||
values.put("avg_sentiment", avgSentiment);
|
||||
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
public void setSessionAudio(long sessionId, String audioPath, String audioCodec) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("audio_path", audioPath);
|
||||
values.put("audio_codec", audioCodec);
|
||||
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
public String getSessionAudioPath(long sessionId) {
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) {
|
||||
if (c.moveToFirst()) {
|
||||
return c.isNull(0) ? "" : c.getString(0);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs,
|
||||
* speakerId, emotion (matches any segment emotion in the session), query
|
||||
* (LIKE over title and segment text/search metadata), limit.
|
||||
*/
|
||||
public String querySessions(String filterJson) {
|
||||
try {
|
||||
JSONObject filter = filterJson == null || filterJson.isEmpty()
|
||||
? new JSONObject() : new JSONObject(filterJson);
|
||||
StringBuilder sql = new StringBuilder(
|
||||
"SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec,"
|
||||
+ " s.avg_sentiment, s.segment_count FROM sessions s");
|
||||
List<String> args = new ArrayList<>();
|
||||
List<String> where = new ArrayList<>();
|
||||
boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query");
|
||||
if (joinSegments) {
|
||||
sql.append(" LEFT JOIN segments g ON g.session_id = s.id");
|
||||
}
|
||||
if (filter.has("sinceMs")) {
|
||||
where.add("s.started_at >= ?");
|
||||
args.add(String.valueOf(filter.getLong("sinceMs")));
|
||||
}
|
||||
if (filter.has("untilMs")) {
|
||||
where.add("s.started_at <= ?");
|
||||
args.add(String.valueOf(filter.getLong("untilMs")));
|
||||
}
|
||||
if (filter.has("speakerId")) {
|
||||
where.add("g.speaker_id = ?");
|
||||
args.add(String.valueOf(filter.getLong("speakerId")));
|
||||
}
|
||||
if (filter.has("emotion")) {
|
||||
where.add("g.emotion = ?");
|
||||
args.add(filter.getString("emotion"));
|
||||
}
|
||||
if (filter.has("query")) {
|
||||
String like = "%" + filter.getString("query") + "%";
|
||||
where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)");
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
}
|
||||
if (!where.isEmpty()) {
|
||||
sql.append(" WHERE ").append(String.join(" AND ", where));
|
||||
}
|
||||
sql.append(" ORDER BY s.started_at DESC");
|
||||
int limit = filter.optInt("limit", 200);
|
||||
sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000)));
|
||||
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
long id = c.getLong(0);
|
||||
row.put("id", id);
|
||||
row.put("startedAt", c.getLong(1));
|
||||
row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2));
|
||||
row.put("title", c.isNull(3) ? "" : c.getString(3));
|
||||
row.put("audioPath", c.isNull(4) ? "" : c.getString(4));
|
||||
row.put("audioCodec", c.isNull(5) ? "" : c.getString(5));
|
||||
row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6));
|
||||
row.put("segmentCount", c.getLong(7));
|
||||
row.put("speakers", sessionSpeakersJson(id));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySessions failed", e);
|
||||
return "[]";
|
||||
}
|
||||
}
|
||||
|
||||
private JSONArray sessionSpeakersJson(long sessionId) throws JSONException {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT DISTINCT p.id, p.name, p.color FROM segments g"
|
||||
+ " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("name", c.getString(1));
|
||||
row.put("color", c.getString(2));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- segments ----
|
||||
|
||||
/**
|
||||
* Insert one transcript segment. json fields: sessionId, speakerId?,
|
||||
* startedAt, endedAt, audioOffsetMs?, text, lang?, translation?,
|
||||
* translationLang?, sentiment?, emotion?, searchMeta?, angle?,
|
||||
* embeddingBase64?. Returns the new row id, or -1.
|
||||
*/
|
||||
public long insertSegment(String json) {
|
||||
try {
|
||||
JSONObject s = new JSONObject(json);
|
||||
ContentValues values = new ContentValues();
|
||||
long sessionId = s.getLong("sessionId");
|
||||
values.put("session_id", sessionId);
|
||||
if (s.has("speakerId")) {
|
||||
values.put("speaker_id", s.getLong("speakerId"));
|
||||
}
|
||||
values.put("started_at", s.getLong("startedAt"));
|
||||
values.put("ended_at", s.getLong("endedAt"));
|
||||
if (s.has("audioOffsetMs")) {
|
||||
values.put("audio_offset_ms", s.getLong("audioOffsetMs"));
|
||||
}
|
||||
values.put("text", s.getString("text"));
|
||||
putOptString(values, s, "lang", "lang");
|
||||
putOptString(values, s, "translation", "translation");
|
||||
putOptString(values, s, "translationLang", "translation_lang");
|
||||
if (s.has("sentiment")) {
|
||||
values.put("sentiment", s.getDouble("sentiment"));
|
||||
}
|
||||
putOptString(values, s, "emotion", "emotion");
|
||||
putOptString(values, s, "searchMeta", "search_meta");
|
||||
if (s.has("angle")) {
|
||||
values.put("angle", s.getInt("angle"));
|
||||
}
|
||||
if (s.has("embeddingBase64")) {
|
||||
values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP));
|
||||
}
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
long id = db.insert("segments", null, values);
|
||||
db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?",
|
||||
new Object[]{sessionId});
|
||||
return id;
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "insertSegment failed", t);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column)
|
||||
throws JSONException {
|
||||
if (s.has(jsonKey) && !s.isNull(jsonKey)) {
|
||||
values.put(column, s.getString(jsonKey));
|
||||
}
|
||||
}
|
||||
|
||||
/** Update mutable fields of a segment (same keys as insertSegment). */
|
||||
public void updateSegment(long segmentId, String json) {
|
||||
try {
|
||||
JSONObject s = new JSONObject(json);
|
||||
ContentValues values = new ContentValues();
|
||||
if (s.has("speakerId")) {
|
||||
if (s.isNull("speakerId")) {
|
||||
values.putNull("speaker_id");
|
||||
} else {
|
||||
values.put("speaker_id", s.getLong("speakerId"));
|
||||
}
|
||||
}
|
||||
if (s.has("text")) {
|
||||
values.put("text", s.getString("text"));
|
||||
}
|
||||
putOptString(values, s, "lang", "lang");
|
||||
putOptString(values, s, "translation", "translation");
|
||||
putOptString(values, s, "translationLang", "translation_lang");
|
||||
if (s.has("sentiment")) {
|
||||
values.put("sentiment", s.getDouble("sentiment"));
|
||||
}
|
||||
putOptString(values, s, "emotion", "emotion");
|
||||
putOptString(values, s, "searchMeta", "search_meta");
|
||||
if (values.size() > 0) {
|
||||
getWritableDatabase().update("segments", values, "id=?",
|
||||
new String[]{String.valueOf(segmentId)});
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "updateSegment failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
public String querySegments(long sessionId) {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
|
||||
+ " translation, translation_lang, sentiment, emotion, angle"
|
||||
+ " FROM segments WHERE session_id=? ORDER BY started_at ASC",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
out.put(segmentRowJson(c));
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySegments failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
private JSONObject segmentRowJson(Cursor c) throws JSONException {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1));
|
||||
row.put("startedAt", c.getLong(2));
|
||||
row.put("endedAt", c.getLong(3));
|
||||
row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4));
|
||||
row.put("text", c.getString(5));
|
||||
row.put("lang", c.isNull(6) ? "" : c.getString(6));
|
||||
row.put("translation", c.isNull(7) ? "" : c.getString(7));
|
||||
row.put("translationLang", c.isNull(8) ? "" : c.getString(8));
|
||||
row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9));
|
||||
row.put("emotion", c.isNull(10) ? "" : c.getString(10));
|
||||
row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11));
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search segments across sessions. filterJson: query?, emotion?,
|
||||
* speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and
|
||||
* the sentiment search metadata.
|
||||
*/
|
||||
public String searchSegments(String filterJson) {
|
||||
try {
|
||||
JSONObject filter = filterJson == null || filterJson.isEmpty()
|
||||
? new JSONObject() : new JSONObject(filterJson);
|
||||
StringBuilder sql = new StringBuilder(
|
||||
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
|
||||
+ " translation, translation_lang, sentiment, emotion, angle, session_id"
|
||||
+ " FROM segments");
|
||||
List<String> args = new ArrayList<>();
|
||||
List<String> where = new ArrayList<>();
|
||||
if (filter.has("query") && !filter.getString("query").isEmpty()) {
|
||||
String like = "%" + filter.getString("query") + "%";
|
||||
where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)");
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
}
|
||||
if (filter.has("emotion")) {
|
||||
where.add("emotion = ?");
|
||||
args.add(filter.getString("emotion"));
|
||||
}
|
||||
if (filter.has("speakerId")) {
|
||||
where.add("speaker_id = ?");
|
||||
args.add(String.valueOf(filter.getLong("speakerId")));
|
||||
}
|
||||
if (filter.has("sinceMs")) {
|
||||
where.add("started_at >= ?");
|
||||
args.add(String.valueOf(filter.getLong("sinceMs")));
|
||||
}
|
||||
if (filter.has("untilMs")) {
|
||||
where.add("started_at <= ?");
|
||||
args.add(String.valueOf(filter.getLong("untilMs")));
|
||||
}
|
||||
if (!where.isEmpty()) {
|
||||
sql.append(" WHERE ").append(String.join(" AND ", where));
|
||||
}
|
||||
sql.append(" ORDER BY started_at DESC LIMIT ")
|
||||
.append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000)));
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = segmentRowJson(c);
|
||||
row.put("sessionId", c.getLong(12));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "searchSegments failed", e);
|
||||
return "[]";
|
||||
}
|
||||
}
|
||||
|
||||
/** Per-segment voice-print embeddings of a session (for re-diarization). */
|
||||
public String querySegmentEmbeddings(long sessionId) {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP));
|
||||
out.put(row);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySegmentEmbeddings failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
// ---- speakers ----
|
||||
|
||||
public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("name", name);
|
||||
values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color);
|
||||
values.put("is_wearer", isWearer ? 1 : 0);
|
||||
values.put("created_at", System.currentTimeMillis());
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
if (embeddingBase64 != null && !embeddingBase64.isEmpty()) {
|
||||
values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP));
|
||||
values.put("embedding_count", 1);
|
||||
}
|
||||
return getWritableDatabase().insert("speakers", null, values);
|
||||
}
|
||||
|
||||
public String querySpeakers() {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count,"
|
||||
+ " p.last_heard_at, p.tag,"
|
||||
+ " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id,"
|
||||
+ " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments"
|
||||
+ " FROM speakers p ORDER BY p.last_heard_at DESC", null)) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("name", c.getString(1));
|
||||
row.put("color", c.getString(2));
|
||||
row.put("isWearer", c.getInt(3) != 0);
|
||||
byte[] blob = c.isNull(4) ? null : c.getBlob(4);
|
||||
row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP));
|
||||
row.put("embeddingCount", c.getInt(5));
|
||||
row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6));
|
||||
row.put("tag", c.isNull(7) ? "" : c.getString(7));
|
||||
row.put("lastRecap", c.isNull(8) ? "" : c.getString(8));
|
||||
row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9));
|
||||
row.put("factsJson", c.isNull(10) ? "" : c.getString(10));
|
||||
row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11));
|
||||
row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12));
|
||||
row.put("segmentCount", c.getLong(13));
|
||||
out.put(row);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySpeakers failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
public void renameSpeaker(long speakerId, String name) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("name", name);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerColor(long speakerId, String color) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("color", color);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerTag(long speakerId, String tag) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("tag", tag);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
/**
|
||||
* Store LLM-derived conversation insights for a speaker: a recap of their
|
||||
* most recent conversation, its open action items (JSON string array), and
|
||||
* the accumulated inferred facts about them (JSON string array).
|
||||
*/
|
||||
public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson,
|
||||
String factsJson, long sessionId) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("last_recap", recap == null ? "" : recap);
|
||||
values.put("action_items", actionItemsJson == null ? "" : actionItemsJson);
|
||||
values.put("facts", factsJson == null ? "" : factsJson);
|
||||
values.put("insights_updated_at", System.currentTimeMillis());
|
||||
values.put("insights_session_id", sessionId);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerWearer(long speakerId, boolean isWearer) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
if (isWearer) {
|
||||
// Exactly one wearer profile at a time.
|
||||
db.execSQL("UPDATE speakers SET is_wearer = 0");
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("is_wearer", isWearer ? 1 : 0);
|
||||
db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
/**
|
||||
* Running-mean centroid update for a speaker's voice-print, capped so a
|
||||
* long session cannot drown the enrolled voice.
|
||||
*/
|
||||
public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) {
|
||||
try {
|
||||
byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP);
|
||||
float[] update = blobToFloats(incoming);
|
||||
if (update == null) {
|
||||
return;
|
||||
}
|
||||
float[] current = null;
|
||||
int count = 0;
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT embedding, embedding_count FROM speakers WHERE id=?",
|
||||
new String[]{String.valueOf(speakerId)})) {
|
||||
if (c.moveToFirst()) {
|
||||
current = c.isNull(0) ? null : blobToFloats(c.getBlob(0));
|
||||
count = c.getInt(1);
|
||||
}
|
||||
}
|
||||
float[] merged;
|
||||
if (current == null || current.length != update.length) {
|
||||
merged = update;
|
||||
count = 1;
|
||||
} else {
|
||||
int effective = Math.min(count, Math.max(1, maxCount));
|
||||
merged = new float[current.length];
|
||||
for (int i = 0; i < merged.length; i++) {
|
||||
merged[i] = (current[i] * effective + update[i]) / (effective + 1);
|
||||
}
|
||||
l2NormalizeInPlace(merged);
|
||||
count = count + 1;
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("embedding", floatsToBlob(merged));
|
||||
values.put("embedding_count", count);
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
getWritableDatabase().update("speakers", values, "id=?",
|
||||
new String[]{String.valueOf(speakerId)});
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "updateSpeakerEmbedding failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
/** Move every segment from one speaker onto another and drop the source. */
|
||||
public void mergeSpeakers(long fromId, long intoId) {
|
||||
if (fromId == intoId) {
|
||||
return;
|
||||
}
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
db.beginTransaction();
|
||||
try {
|
||||
db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?",
|
||||
new Object[]{intoId, fromId});
|
||||
// Blend the two centroids weighted by their sample counts.
|
||||
float[] from = null;
|
||||
float[] into = null;
|
||||
int fromCount = 0;
|
||||
int intoCount = 0;
|
||||
boolean fromWearer = false;
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)",
|
||||
new String[]{String.valueOf(fromId), String.valueOf(intoId)})) {
|
||||
while (c.moveToNext()) {
|
||||
long id = c.getLong(0);
|
||||
float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1));
|
||||
int count = c.getInt(2);
|
||||
if (id == fromId) {
|
||||
from = embedding;
|
||||
fromCount = count;
|
||||
fromWearer = c.getInt(3) != 0;
|
||||
} else {
|
||||
into = embedding;
|
||||
intoCount = count;
|
||||
}
|
||||
}
|
||||
}
|
||||
float[] merged = into;
|
||||
if (from != null && into != null && from.length == into.length) {
|
||||
merged = new float[into.length];
|
||||
int total = Math.max(1, fromCount + intoCount);
|
||||
for (int i = 0; i < merged.length; i++) {
|
||||
merged[i] = (into[i] * intoCount + from[i] * fromCount) / total;
|
||||
}
|
||||
l2NormalizeInPlace(merged);
|
||||
} else if (into == null) {
|
||||
merged = from;
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
if (merged != null) {
|
||||
values.put("embedding", floatsToBlob(merged));
|
||||
values.put("embedding_count", fromCount + intoCount);
|
||||
}
|
||||
if (fromWearer) {
|
||||
values.put("is_wearer", 1);
|
||||
}
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
// Keep the target's conversation insights; adopt the source's only
|
||||
// where the target has none (e.g. merging a rich profile into a
|
||||
// freshly auto-created one).
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id"
|
||||
+ " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END",
|
||||
new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) {
|
||||
boolean targetHasInsights = false;
|
||||
while (c.moveToNext()) {
|
||||
boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty();
|
||||
if (c.getPosition() == 0) {
|
||||
targetHasInsights = hasInsights;
|
||||
} else if (!targetHasInsights && hasInsights) {
|
||||
values.put("last_recap", c.getString(0));
|
||||
values.put("action_items", c.isNull(1) ? "" : c.getString(1));
|
||||
values.put("facts", c.isNull(2) ? "" : c.getString(2));
|
||||
values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3));
|
||||
values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4));
|
||||
}
|
||||
}
|
||||
}
|
||||
db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)});
|
||||
db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)});
|
||||
db.setTransactionSuccessful();
|
||||
} finally {
|
||||
db.endTransaction();
|
||||
}
|
||||
}
|
||||
|
||||
public void reassignSegmentSpeaker(long segmentId, long speakerId) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("speaker_id", speakerId);
|
||||
getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)});
|
||||
}
|
||||
|
||||
/** Reassign every segment of a speaker within one session (split support). */
|
||||
public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) {
|
||||
getWritableDatabase().execSQL(
|
||||
"UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?",
|
||||
new Object[]{toSpeakerId, sessionId, fromSpeakerId});
|
||||
}
|
||||
|
||||
public void deleteSpeaker(long speakerId) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId});
|
||||
db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
// ---- retention ----
|
||||
|
||||
/**
|
||||
* Delete caption sessions older than the cutoff (0 disables), and strip
|
||||
* recordings older than their own cutoff while keeping the transcript.
|
||||
* Returns a JSON summary of what was removed.
|
||||
*/
|
||||
public String applyRetention(long captionCutoffMs, long recordingCutoffMs) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
int sessionsDeleted = 0;
|
||||
int recordingsDeleted = 0;
|
||||
if (recordingCutoffMs > 0) {
|
||||
List<String> paths = new ArrayList<>();
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
|
||||
new String[]{String.valueOf(recordingCutoffMs)})) {
|
||||
while (c.moveToNext()) {
|
||||
if (!c.isNull(1)) {
|
||||
paths.add(c.getString(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String path : paths) {
|
||||
if (new File(path).delete()) {
|
||||
recordingsDeleted++;
|
||||
}
|
||||
}
|
||||
db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?",
|
||||
new Object[]{recordingCutoffMs});
|
||||
}
|
||||
if (captionCutoffMs > 0) {
|
||||
List<String> paths = new ArrayList<>();
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
|
||||
new String[]{String.valueOf(captionCutoffMs)})) {
|
||||
while (c.moveToNext()) {
|
||||
paths.add(c.getString(0));
|
||||
}
|
||||
}
|
||||
for (String path : paths) {
|
||||
if (path != null && new File(path).delete()) {
|
||||
recordingsDeleted++;
|
||||
}
|
||||
}
|
||||
db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)",
|
||||
new Object[]{captionCutoffMs});
|
||||
Cursor count = db.rawQuery("SELECT changes()", null);
|
||||
count.close();
|
||||
db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs});
|
||||
try (Cursor c = db.rawQuery("SELECT changes()", null)) {
|
||||
if (c.moveToFirst()) {
|
||||
sessionsDeleted = c.getInt(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
JSONObject summary = new JSONObject();
|
||||
summary.put("sessionsDeleted", sessionsDeleted);
|
||||
summary.put("recordingsDeleted", recordingsDeleted);
|
||||
return summary.toString();
|
||||
} catch (JSONException e) {
|
||||
return "{}";
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteSession(long sessionId) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
String path = getSessionAudioPath(sessionId);
|
||||
if (!path.isEmpty()) {
|
||||
new File(path).delete();
|
||||
}
|
||||
db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)});
|
||||
db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
// ---- blob helpers ----
|
||||
|
||||
private static float[] blobToFloats(byte[] blob) {
|
||||
if (blob == null || blob.length < 4 || blob.length % 4 != 0) {
|
||||
return null;
|
||||
}
|
||||
ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
|
||||
float[] out = new float[blob.length / 4];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
out[i] = buffer.getFloat();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static byte[] floatsToBlob(float[] values) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN);
|
||||
for (float v : values) {
|
||||
buffer.putFloat(v);
|
||||
}
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
private static void l2NormalizeInPlace(float[] v) {
|
||||
double sum = 0;
|
||||
for (float x : v) {
|
||||
sum += (double) x * x;
|
||||
}
|
||||
double norm = Math.sqrt(sum);
|
||||
if (norm <= 0) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < v.length; i++) {
|
||||
v[i] = (float) (v[i] / norm);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,9 +37,6 @@ import java.util.Set;
|
||||
* matching are decided in TypeScript (app/g2/even-advertisement.ts,
|
||||
* app/g2/pairing-candidates.ts) where they can be unit-tested; this filter
|
||||
* must stay a superset of the TS classifier's admission.
|
||||
*
|
||||
* Ported from the discovery path of SybilSight's Bluetooth SDK (G2.didDiscover
|
||||
* and R1's advertisement snapshot), reduced to what Android exposes.
|
||||
*/
|
||||
@SuppressLint("MissingPermission")
|
||||
public class FaceclawDeviceDiscovery {
|
||||
@@ -296,7 +293,7 @@ public class FaceclawDeviceDiscovery {
|
||||
/**
|
||||
* Android strips the 2-byte company identifier from each manufacturer
|
||||
* record and keys the SparseArray by it. Put the bytes back (little-endian)
|
||||
* so TypeScript sees the same "ER"+… layout the iOS SDK and HCI captures
|
||||
* so TypeScript sees the same "ER"+… layout the HCI captures
|
||||
* document. Prefers Even's own id; otherwise returns the first record so an
|
||||
* unexpected company id still reaches the diagnostics log.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FastClusteringConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.DataInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Offline re-diarization of a stored conversation recording (16-bit mono
|
||||
* WAV): pyannote segmentation plus the shared speaker-embedding model with
|
||||
* clustering, via sherpa-onnx. Runs on its own thread; results are handed to
|
||||
* TS as JSON turns for reconciliation against the transcript segments.
|
||||
*/
|
||||
public class FaceclawDiarizer {
|
||||
private static final String TAG = "FaceclawDiarizer";
|
||||
|
||||
public interface Listener {
|
||||
/** progress in [0, 1]. */
|
||||
void onProgress(double progress);
|
||||
|
||||
/** turns: [{startMs, endMs, cluster}], sorted by start time. */
|
||||
void onDone(String turnsJson);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
|
||||
private final String segmentationModelPath;
|
||||
private final String embeddingModelPath;
|
||||
private final String wavPath;
|
||||
private final int numClusters;
|
||||
private final float threshold;
|
||||
private final Listener listener;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private volatile boolean cancelled;
|
||||
|
||||
public FaceclawDiarizer(
|
||||
String segmentationModelPath,
|
||||
String embeddingModelPath,
|
||||
String wavPath,
|
||||
int numClusters,
|
||||
float threshold,
|
||||
Listener listener) {
|
||||
this.segmentationModelPath = segmentationModelPath;
|
||||
this.embeddingModelPath = embeddingModelPath;
|
||||
this.wavPath = wavPath;
|
||||
this.numClusters = numClusters;
|
||||
this.threshold = threshold <= 0 ? 0.5f : threshold;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
Thread thread = new Thread(this::run, "FaceclawDiarizer");
|
||||
thread.setPriority(Thread.MIN_PRIORITY);
|
||||
thread.start();
|
||||
}
|
||||
|
||||
public void cancel() {
|
||||
cancelled = true;
|
||||
}
|
||||
|
||||
private void run() {
|
||||
OfflineSpeakerDiarization diarization = null;
|
||||
try {
|
||||
float[] samples = wavPath.endsWith(".wav")
|
||||
? readWavMono16(wavPath)
|
||||
: decodeCompressedMono(wavPath);
|
||||
OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder()
|
||||
.setSegmentation(OfflineSpeakerSegmentationModelConfig.builder()
|
||||
.setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder()
|
||||
.setModel(segmentationModelPath)
|
||||
.build())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setEmbedding(SpeakerEmbeddingExtractorConfig.builder()
|
||||
.setModel(embeddingModelPath)
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setClustering(FastClusteringConfig.builder()
|
||||
.setNumClusters(numClusters > 0 ? numClusters : -1)
|
||||
.setThreshold(threshold)
|
||||
.build())
|
||||
.setMinDurationOn(0.3f)
|
||||
.setMinDurationOff(0.5f)
|
||||
.build();
|
||||
diarization = new OfflineSpeakerDiarization(config);
|
||||
OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback(
|
||||
samples,
|
||||
(numProcessedChunks, numTotalChunks, arg) -> {
|
||||
emitProgress(numTotalChunks <= 0 ? 0
|
||||
: (double) numProcessedChunks / numTotalChunks);
|
||||
return cancelled ? 1 : 0;
|
||||
});
|
||||
JSONArray turns = new JSONArray();
|
||||
if (segments != null) {
|
||||
for (OfflineSpeakerDiarizationSegment segment : segments) {
|
||||
JSONObject turn = new JSONObject();
|
||||
turn.put("startMs", Math.round(segment.getStart() * 1000));
|
||||
turn.put("endMs", Math.round(segment.getEnd() * 1000));
|
||||
turn.put("cluster", segment.getSpeaker());
|
||||
turns.put(turn);
|
||||
}
|
||||
}
|
||||
String result = turns.toString();
|
||||
mainHandler.post(() -> listener.onDone(result));
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "diarization failed for " + wavPath, t);
|
||||
String message = t.getMessage() == null ? t.toString() : t.getMessage();
|
||||
mainHandler.post(() -> listener.onError(message));
|
||||
} finally {
|
||||
if (diarization != null) {
|
||||
diarization.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void emitProgress(double progress) {
|
||||
mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a compressed recording (M4A/AAC from the transcode sweep) into
|
||||
* [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written
|
||||
* as 16 kHz mono, so no resampling is needed.
|
||||
*/
|
||||
private static float[] decodeCompressedMono(String path) throws IOException {
|
||||
android.media.MediaExtractor extractor = new android.media.MediaExtractor();
|
||||
android.media.MediaCodec codec = null;
|
||||
java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream();
|
||||
try {
|
||||
extractor.setDataSource(path);
|
||||
int trackIndex = -1;
|
||||
android.media.MediaFormat format = null;
|
||||
for (int i = 0; i < extractor.getTrackCount(); i++) {
|
||||
android.media.MediaFormat candidate = extractor.getTrackFormat(i);
|
||||
String mime = candidate.getString(android.media.MediaFormat.KEY_MIME);
|
||||
if (mime != null && mime.startsWith("audio/")) {
|
||||
trackIndex = i;
|
||||
format = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (trackIndex < 0 || format == null) {
|
||||
throw new IOException("no audio track in " + path);
|
||||
}
|
||||
extractor.selectTrack(trackIndex);
|
||||
codec = android.media.MediaCodec.createDecoderByType(
|
||||
format.getString(android.media.MediaFormat.KEY_MIME));
|
||||
codec.configure(format, null, null, 0);
|
||||
codec.start();
|
||||
android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo();
|
||||
boolean inputDone = false;
|
||||
boolean outputDone = false;
|
||||
while (!outputDone) {
|
||||
if (!inputDone) {
|
||||
int inIndex = codec.dequeueInputBuffer(10_000);
|
||||
if (inIndex >= 0) {
|
||||
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
|
||||
int size = extractor.readSampleData(inBuf, 0);
|
||||
if (size < 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, 0,
|
||||
android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM);
|
||||
inputDone = true;
|
||||
} else {
|
||||
codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0);
|
||||
extractor.advance();
|
||||
}
|
||||
}
|
||||
}
|
||||
int outIndex = codec.dequeueOutputBuffer(info, 10_000);
|
||||
if (outIndex >= 0) {
|
||||
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
|
||||
byte[] chunk = new byte[info.size];
|
||||
outBuf.position(info.offset);
|
||||
outBuf.get(chunk);
|
||||
pcm.write(chunk);
|
||||
codec.releaseOutputBuffer(outIndex, false);
|
||||
if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputDone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
extractor.release();
|
||||
if (codec != null) {
|
||||
try {
|
||||
codec.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
codec.release();
|
||||
}
|
||||
}
|
||||
byte[] bytes = pcm.toByteArray();
|
||||
int count = bytes.length / 2;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f;
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
|
||||
/** Load a 16-bit mono PCM WAV into [-1, 1] floats. */
|
||||
private static float[] readWavMono16(String path) throws IOException {
|
||||
File file = new File(path);
|
||||
long fileLength = file.length();
|
||||
try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
|
||||
byte[] header = new byte[44];
|
||||
in.readFully(header);
|
||||
if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') {
|
||||
throw new IOException("not a WAV file");
|
||||
}
|
||||
int channels = (header[22] & 0xff) | (header[23] << 8);
|
||||
int bits = (header[34] & 0xff) | (header[35] << 8);
|
||||
if (channels != 1 || bits != 16) {
|
||||
throw new IOException("only 16-bit mono WAV is supported");
|
||||
}
|
||||
long dataBytes = fileLength - 44;
|
||||
int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE);
|
||||
float[] samples = new float[count];
|
||||
byte[] chunk = new byte[65536];
|
||||
int sampleIndex = 0;
|
||||
int carried = 0;
|
||||
byte carriedByte = 0;
|
||||
int read;
|
||||
while (sampleIndex < count && (read = in.read(chunk)) > 0) {
|
||||
int offset = 0;
|
||||
if (carried == 1) {
|
||||
samples[sampleIndex++] =
|
||||
((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f;
|
||||
offset = 1;
|
||||
carried = 0;
|
||||
}
|
||||
int pairs = (read - offset) / 2;
|
||||
for (int i = 0; i < pairs && sampleIndex < count; i++) {
|
||||
int base = offset + i * 2;
|
||||
samples[sampleIndex++] =
|
||||
((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f;
|
||||
}
|
||||
if (((read - offset) & 1) == 1) {
|
||||
carried = 1;
|
||||
carriedByte = chunk[read - 1];
|
||||
}
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
}
|
||||
}
|
||||
+11
-40
@@ -26,6 +26,10 @@ import java.util.concurrent.TimeUnit;
|
||||
* protobuf helpers (BleProtocol), but owns its own GATT connection and runs its
|
||||
* whole lifecycle on a single worker thread. It expects the main app to be
|
||||
* disconnected while it runs (onboarding, before flashing).
|
||||
*
|
||||
* Only the right arm is used: the lenses relay messages to each other, so
|
||||
* anything sent to the right arm reaches both, and acks/events only ever come
|
||||
* from the right arm anyway.
|
||||
*/
|
||||
public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
private static final String TAG = "FaceclawFlashPrompt";
|
||||
@@ -43,7 +47,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
|
||||
private final Context context;
|
||||
private final String rightAddress;
|
||||
private final String leftAddress;
|
||||
private final String warningText;
|
||||
private final FaceclawBleManager bleManager;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
@@ -58,7 +61,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
private volatile boolean cancelled = false;
|
||||
private volatile boolean finished = false;
|
||||
private volatile boolean rightConnected = false;
|
||||
private volatile boolean leftConnected = false;
|
||||
private volatile boolean rightLost = false;
|
||||
private volatile Boolean approved = null;
|
||||
private volatile ScheduledExecutorService heartbeatExecutor;
|
||||
@@ -66,10 +68,9 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
private int nextMagic = 100;
|
||||
private int nextSeq = 0x40;
|
||||
|
||||
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String leftAddress, String warningText) {
|
||||
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String warningText) {
|
||||
this.context = context.getApplicationContext();
|
||||
this.rightAddress = rightAddress == null ? "" : rightAddress;
|
||||
this.leftAddress = leftAddress == null ? "" : leftAddress;
|
||||
this.warningText = warningText == null ? "" : warningText;
|
||||
this.bleManager = new FaceclawBleManager(this.context);
|
||||
this.bleManager.setListener(this);
|
||||
@@ -116,18 +117,8 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
}
|
||||
|
||||
emitState("connecting", "");
|
||||
connectArm(rightAddress, "right");
|
||||
connectArm(rightAddress);
|
||||
rightConnected = true;
|
||||
|
||||
boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress);
|
||||
if (haveLeft) {
|
||||
try {
|
||||
connectArm(leftAddress, "left");
|
||||
leftConnected = true;
|
||||
} catch (Exception e) {
|
||||
emitLog("left arm connect failed (continuing on right only): " + e.getMessage());
|
||||
}
|
||||
}
|
||||
if (cancelled) {
|
||||
teardown();
|
||||
return;
|
||||
@@ -135,14 +126,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
|
||||
emitState("connected", "");
|
||||
bringUpArm(rightAddress);
|
||||
if (leftConnected) {
|
||||
try {
|
||||
bringUpArm(leftAddress);
|
||||
} catch (Exception e) {
|
||||
emitLog("left arm prompt failed (continuing on right only): " + e.getMessage());
|
||||
leftConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
startHeartbeat();
|
||||
emitState("prompting", "");
|
||||
@@ -158,7 +141,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
if (approved != null) {
|
||||
boolean result = Boolean.TRUE.equals(approved);
|
||||
try {
|
||||
sendShutdownBoth();
|
||||
sendShutdown();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
emitResult(result);
|
||||
@@ -182,7 +165,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
}
|
||||
}
|
||||
|
||||
private void connectArm(String address, String labelForLog) {
|
||||
private void connectArm(String address) {
|
||||
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
|
||||
throw new IllegalStateException("connect failed: " + address);
|
||||
}
|
||||
@@ -194,7 +177,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
|
||||
throw new IllegalStateException("enableNotifications failed: " + address);
|
||||
}
|
||||
emitLog("connected " + labelForLog + " arm");
|
||||
emitLog("connected right arm");
|
||||
}
|
||||
|
||||
private void bringUpArm(String address) throws InterruptedException {
|
||||
@@ -301,9 +284,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
if (rightConnected) {
|
||||
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
|
||||
}
|
||||
if (leftConnected) {
|
||||
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.w(TAG, "heartbeat failed: " + e.getMessage());
|
||||
}
|
||||
@@ -317,14 +297,11 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
}
|
||||
}
|
||||
|
||||
private void sendShutdownBoth() {
|
||||
private void sendShutdown() {
|
||||
byte[] shutdown = BleProtocol.buildShutdown(allocMagic(), 0);
|
||||
if (rightConnected) {
|
||||
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
|
||||
}
|
||||
if (leftConnected) {
|
||||
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
|
||||
}
|
||||
}
|
||||
|
||||
private void teardown() {
|
||||
@@ -335,7 +312,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
rightConnected = false;
|
||||
leftConnected = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -355,10 +331,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
return;
|
||||
}
|
||||
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
|
||||
// Async events (taps/selections) only ever come from the right arm.
|
||||
if (address.equalsIgnoreCase(rightAddress)) {
|
||||
handleEvent(frame);
|
||||
}
|
||||
handleEvent(frame);
|
||||
return;
|
||||
}
|
||||
if (frame.msgSeq >= 0) {
|
||||
@@ -407,8 +380,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
selectionLatch.countDown();
|
||||
}
|
||||
}
|
||||
} else if (address.equalsIgnoreCase(leftAddress)) {
|
||||
leftConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,12 @@ public class FaceclawLc3Decoder {
|
||||
public static final int COUNTER_OFFSET = 204;
|
||||
public static final int SAMPLES_PER_FRAME = 160;
|
||||
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
|
||||
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
|
||||
// glasses DSP appends two signed 16-bit LE values computed on the raw
|
||||
// stereo capture before the mono downmix — a signal-strength ratio and the
|
||||
// asin-derived direction-of-arrival in degrees — then the packet counter.
|
||||
public static final int SSR_OFFSET = 200;
|
||||
public static final int ANGLE_OFFSET = 202;
|
||||
|
||||
static {
|
||||
System.loadLibrary("faceclaw_lc3");
|
||||
@@ -23,6 +29,8 @@ public class FaceclawLc3Decoder {
|
||||
private long duplicatePackets;
|
||||
private long missingPackets;
|
||||
private long decodeErrors;
|
||||
private int lastSsr;
|
||||
private int lastAngleDegrees;
|
||||
|
||||
public FaceclawLc3Decoder() {
|
||||
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
|
||||
@@ -64,9 +72,21 @@ public class FaceclawLc3Decoder {
|
||||
}
|
||||
realPackets++;
|
||||
lastCounter = counter;
|
||||
lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8));
|
||||
lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8));
|
||||
return decoded;
|
||||
}
|
||||
|
||||
/** Firmware signal-strength ratio from the most recent decoded packet. */
|
||||
public synchronized int getLastSsr() {
|
||||
return lastSsr;
|
||||
}
|
||||
|
||||
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
|
||||
public synchronized int getLastAngleDegrees() {
|
||||
return lastAngleDegrees;
|
||||
}
|
||||
|
||||
public synchronized long getRealPackets() {
|
||||
return realPackets;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawMicStatusListener {
|
||||
/**
|
||||
* A CFW mic_control field-104 status record from one temple. body is the
|
||||
* raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is
|
||||
* "L" or "R" for the temple's own BLE link, "?" if unmatched.
|
||||
*/
|
||||
void onMicStatus(byte[] body, String arm);
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.media.audiofx.AcousticEchoCanceler;
|
||||
import android.media.audiofx.AudioEffect;
|
||||
import android.media.audiofx.AutomaticGainControl;
|
||||
import android.media.audiofx.NoiseSuppressor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Phone-side noise suppression for the microphone stream.
|
||||
*
|
||||
* Android's built-in noise cancellation (the {@link NoiseSuppressor} capture
|
||||
* effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio
|
||||
* arrives as raw PCM over BLE from the glasses, which that effect chain never
|
||||
* sees, so this class picks the best engine that can actually reach the
|
||||
* samples:
|
||||
*
|
||||
* - Audio captured by the phone itself: {@link #attachPlatformEffects(int)}
|
||||
* enables the device's built-in NS/AEC/AGC chain (hardware or OEM software,
|
||||
* whatever the Android build ships) on the AudioRecord session.
|
||||
* - Audio from the glasses (the normal case): {@link #process(byte[])} runs a
|
||||
* WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann
|
||||
* frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an
|
||||
* over-subtracted Wiener gain with a residual floor so speech never pumps
|
||||
* in and out of silence.
|
||||
*/
|
||||
public final class FaceclawNoiseSuppressor {
|
||||
|
||||
private static final int FFT_SIZE = 256;
|
||||
private static final int HOP = FFT_SIZE / 2;
|
||||
private static final float OVERSUBTRACTION = 1.6f;
|
||||
/** About -16 dB of residual: keeps the noise bed natural instead of gated. */
|
||||
private static final float GAIN_FLOOR = 0.15f;
|
||||
private static final float GAIN_SMOOTHING = 0.6f;
|
||||
/** Frames of pure noise-floor learning after start/reset (~100 ms). */
|
||||
private static final int NOISE_WARMUP_FRAMES = 12;
|
||||
|
||||
private final int sampleRate;
|
||||
private final float[] window = new float[FFT_SIZE];
|
||||
private final float[] history = new float[FFT_SIZE];
|
||||
private final float[] overlap = new float[HOP];
|
||||
private final float[] noise = new float[FFT_SIZE / 2 + 1];
|
||||
private final float[] gain = new float[FFT_SIZE / 2 + 1];
|
||||
private final float[] re = new float[FFT_SIZE];
|
||||
private final float[] im = new float[FFT_SIZE];
|
||||
private final short[] pending = new short[HOP];
|
||||
private int pendingCount = 0;
|
||||
private int warmupFrames = 0;
|
||||
|
||||
public FaceclawNoiseSuppressor(int sampleRate) {
|
||||
this.sampleRate = sampleRate;
|
||||
// sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap.
|
||||
for (int i = 0; i < FFT_SIZE; i++) {
|
||||
window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE);
|
||||
}
|
||||
reset();
|
||||
}
|
||||
|
||||
public int getSampleRate() {
|
||||
return sampleRate;
|
||||
}
|
||||
|
||||
/** Forget the learned noise floor (e.g. after the suppressor was toggled off). */
|
||||
public void reset() {
|
||||
Arrays.fill(history, 0f);
|
||||
Arrays.fill(overlap, 0f);
|
||||
Arrays.fill(noise, 0f);
|
||||
Arrays.fill(gain, 1f);
|
||||
pendingCount = 0;
|
||||
warmupFrames = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Suppress noise in 16-bit little-endian mono PCM. Streaming: input not
|
||||
* filling a whole 8 ms hop is carried into the next call, so the output
|
||||
* may be up to one hop shorter than the input (and longer on a later
|
||||
* call); total latency is one hop.
|
||||
*/
|
||||
public byte[] process(byte[] pcm16le) {
|
||||
if (pcm16le == null || pcm16le.length < 2) {
|
||||
return pcm16le;
|
||||
}
|
||||
int sampleCount = pcm16le.length / 2;
|
||||
int hops = (pendingCount + sampleCount) / HOP;
|
||||
byte[] out = new byte[hops * HOP * 2];
|
||||
int outPos = 0;
|
||||
int inPos = 0;
|
||||
while (inPos < sampleCount) {
|
||||
int take = Math.min(HOP - pendingCount, sampleCount - inPos);
|
||||
for (int i = 0; i < take; i++) {
|
||||
int lo = pcm16le[(inPos + i) * 2] & 0xff;
|
||||
int hi = pcm16le[(inPos + i) * 2 + 1];
|
||||
pending[pendingCount + i] = (short) ((hi << 8) | lo);
|
||||
}
|
||||
pendingCount += take;
|
||||
inPos += take;
|
||||
if (pendingCount == HOP) {
|
||||
outPos = processHop(out, outPos);
|
||||
pendingCount = 0;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private int processHop(byte[] out, int outPos) {
|
||||
System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP);
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
history[FFT_SIZE - HOP + i] = pending[i] / 32768f;
|
||||
}
|
||||
for (int i = 0; i < FFT_SIZE; i++) {
|
||||
re[i] = history[i] * window[i];
|
||||
im[i] = 0f;
|
||||
}
|
||||
fft(re, im, false);
|
||||
|
||||
boolean warming = warmupFrames < NOISE_WARMUP_FRAMES;
|
||||
if (warming) {
|
||||
warmupFrames++;
|
||||
}
|
||||
for (int b = 0; b <= FFT_SIZE / 2; b++) {
|
||||
float power = re[b] * re[b] + im[b] * im[b];
|
||||
if (warming) {
|
||||
noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames;
|
||||
} else if (power < noise[b]) {
|
||||
// Track the floor down quickly; drift up slowly so speech
|
||||
// energy never gets learned as noise.
|
||||
noise[b] += (power - noise[b]) * 0.2f;
|
||||
} else {
|
||||
noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f);
|
||||
}
|
||||
float clean = power - OVERSUBTRACTION * noise[b];
|
||||
float g = clean > 0f ? clean / power : 0f;
|
||||
if (g < GAIN_FLOOR) {
|
||||
g = GAIN_FLOOR;
|
||||
}
|
||||
g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g;
|
||||
gain[b] = g;
|
||||
re[b] *= g;
|
||||
im[b] *= g;
|
||||
if (b > 0 && b < FFT_SIZE / 2) {
|
||||
re[FFT_SIZE - b] *= g;
|
||||
im[FFT_SIZE - b] *= g;
|
||||
}
|
||||
}
|
||||
|
||||
fft(re, im, true);
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
float value = (overlap[i] + re[i] * window[i]) * 32767f;
|
||||
int sample = Math.round(value);
|
||||
if (sample > 32767) {
|
||||
sample = 32767;
|
||||
}
|
||||
if (sample < -32768) {
|
||||
sample = -32768;
|
||||
}
|
||||
out[outPos + i * 2] = (byte) (sample & 0xff);
|
||||
out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff);
|
||||
}
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
overlap[i] = re[HOP + i] * window[HOP + i];
|
||||
}
|
||||
return outPos + HOP * 2;
|
||||
}
|
||||
|
||||
/** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */
|
||||
private static void fft(float[] re, float[] im, boolean inverse) {
|
||||
int n = re.length;
|
||||
for (int i = 1, j = 0; i < n; i++) {
|
||||
int bit = n >> 1;
|
||||
for (; (j & bit) != 0; bit >>= 1) {
|
||||
j ^= bit;
|
||||
}
|
||||
j ^= bit;
|
||||
if (i < j) {
|
||||
float t = re[i];
|
||||
re[i] = re[j];
|
||||
re[j] = t;
|
||||
t = im[i];
|
||||
im[i] = im[j];
|
||||
im[j] = t;
|
||||
}
|
||||
}
|
||||
for (int len = 2; len <= n; len <<= 1) {
|
||||
double ang = 2 * Math.PI / len * (inverse ? 1 : -1);
|
||||
float wRe = (float) Math.cos(ang);
|
||||
float wIm = (float) Math.sin(ang);
|
||||
for (int i = 0; i < n; i += len) {
|
||||
float curRe = 1f;
|
||||
float curIm = 0f;
|
||||
for (int k = 0; k < len / 2; k++) {
|
||||
int a = i + k;
|
||||
int b = a + len / 2;
|
||||
float xr = re[b] * curRe - im[b] * curIm;
|
||||
float xi = re[b] * curIm + im[b] * curRe;
|
||||
re[b] = re[a] - xr;
|
||||
im[b] = im[a] - xi;
|
||||
re[a] += xr;
|
||||
im[a] += xi;
|
||||
float nextRe = curRe * wRe - curIm * wIm;
|
||||
curIm = curRe * wIm + curIm * wRe;
|
||||
curRe = nextRe;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inverse) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
re[i] /= n;
|
||||
im[i] /= n;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this Android build ships the built-in capture noise suppressor. */
|
||||
public static boolean platformSuppressorAvailable() {
|
||||
try {
|
||||
return NoiseSuppressor.isAvailable();
|
||||
} catch (Throwable t) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Short engine name for the UI. */
|
||||
public static String engineDescription() {
|
||||
return "spectral (phone DSP)";
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach the platform's built-in capture-effect chain (NoiseSuppressor,
|
||||
* AcousticEchoCanceler, AutomaticGainControl — whichever this Android
|
||||
* build provides) to a phone AudioRecord session. Only applicable when the
|
||||
* phone itself captures the audio; the BLE glasses stream must go through
|
||||
* {@link #process(byte[])} instead. The caller keeps the returned effects
|
||||
* for the life of the recording and release()s them with it.
|
||||
*/
|
||||
public static AudioEffect[] attachPlatformEffects(int audioSessionId) {
|
||||
ArrayList<AudioEffect> effects = new ArrayList<>();
|
||||
try {
|
||||
if (NoiseSuppressor.isAvailable()) {
|
||||
NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId);
|
||||
if (ns != null) {
|
||||
ns.setEnabled(true);
|
||||
effects.add(ns);
|
||||
}
|
||||
}
|
||||
if (AcousticEchoCanceler.isAvailable()) {
|
||||
AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId);
|
||||
if (aec != null) {
|
||||
aec.setEnabled(true);
|
||||
effects.add(aec);
|
||||
}
|
||||
}
|
||||
if (AutomaticGainControl.isAvailable()) {
|
||||
AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId);
|
||||
if (agc != null) {
|
||||
agc.setEnabled(true);
|
||||
effects.add(agc);
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
// A missing effect just means this tier is unavailable.
|
||||
}
|
||||
return effects.toArray(new AudioEffect[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.Context;
|
||||
|
||||
/**
|
||||
* Headless display target for preview-only mode (no glasses paired). Hosts
|
||||
* the same SurfaceCompositor the BLE communicator uses, so the TS side can
|
||||
* run its full surface/render pipeline unchanged; composited frames simply
|
||||
* have nowhere to go, and the phone-mirror preview, screenshots, and GIF
|
||||
* recordings read the retained composite back exactly as they do on a live
|
||||
* connection.
|
||||
*
|
||||
* The method signatures mirror the display subset of FaceclawBleCommunicator
|
||||
* (the TS DisplayTarget interface is typed against that subset).
|
||||
*/
|
||||
public final class FaceclawPreviewCompositor {
|
||||
private final Context appContext;
|
||||
private final SurfaceCompositor compositor = new SurfaceCompositor();
|
||||
private final android.os.Handler mainHandler = new android.os.Handler(android.os.Looper.getMainLooper());
|
||||
|
||||
// Notified (on the main looper, where the phone UI's JS runs) after each
|
||||
// applied surface frame, standing in for the BLE path's per-frame metrics
|
||||
// callback so the phone mirror refreshes promptly instead of waiting for
|
||||
// its safety-net poll.
|
||||
private volatile Runnable frameListener;
|
||||
|
||||
// Active animated-GIF screen recording, or null when idle. Frames are
|
||||
// pushed by recordScreenFrame(), which the TS side calls at each
|
||||
// phone-preview flush.
|
||||
private volatile GifScreenRecorder screenRecorder;
|
||||
|
||||
// Worker-app isolates locate their frame target through a static, like
|
||||
// FaceclawBleCommunicator.getActive(); Java statics are shared across
|
||||
// isolates, unlike anything on the JS side.
|
||||
private static volatile FaceclawPreviewCompositor activeInstance;
|
||||
|
||||
public static FaceclawPreviewCompositor getActive() {
|
||||
return activeInstance;
|
||||
}
|
||||
|
||||
public FaceclawPreviewCompositor(Context context) {
|
||||
this.appContext = context.getApplicationContext();
|
||||
}
|
||||
|
||||
/** Publish this compositor as the workers' frame target. */
|
||||
public void makeActive() {
|
||||
activeInstance = this;
|
||||
}
|
||||
|
||||
/** Withdraw from workers (a real connection is taking over). */
|
||||
public void release() {
|
||||
if (activeInstance == this) {
|
||||
activeInstance = null;
|
||||
}
|
||||
frameListener = null;
|
||||
}
|
||||
|
||||
public void setFrameListener(Runnable listener) {
|
||||
this.frameListener = listener;
|
||||
}
|
||||
|
||||
/** Set the compositor's output frame size. Call before configuring surfaces. */
|
||||
public void configureCompositorScreen(int width, int height) {
|
||||
compositor.configureScreen(width, height);
|
||||
}
|
||||
|
||||
public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) {
|
||||
compositor.configureSurface(id, x, y, width, height, zOrder, transparency);
|
||||
}
|
||||
|
||||
public void removeSurface(String id) {
|
||||
compositor.removeSurface(id);
|
||||
}
|
||||
|
||||
public void setSurfaceVisible(String id, boolean visible) {
|
||||
compositor.setSurfaceVisible(id, visible);
|
||||
}
|
||||
|
||||
/** Blank (screen off) or unblank the output; retained surface state survives. */
|
||||
public void setScreenBlanked(boolean blanked) {
|
||||
compositor.setBlanked(blanked);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply an update to one compositor surface. Unlike the connected path
|
||||
* there is no desired-frame store or transmit pipeline downstream; the
|
||||
* preview/screenshot calls recomposite the retained state on demand, so
|
||||
* applying the update is the whole job and the frame finishes here.
|
||||
*/
|
||||
public void submitSurfaceFrame(
|
||||
java.nio.ByteBuffer pixels8bpp,
|
||||
String surfaceId,
|
||||
int rectX,
|
||||
int rectY,
|
||||
int rectWidth,
|
||||
int rectHeight,
|
||||
String contentFingerprint,
|
||||
int paintMs,
|
||||
int frameId,
|
||||
java.nio.ByteBuffer glyphs
|
||||
) {
|
||||
compositor.applyAndComposite(
|
||||
surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs);
|
||||
FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)");
|
||||
Runnable listener = frameListener;
|
||||
if (listener != null) {
|
||||
mainHandler.post(listener);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The current composited screen as a phone-UI preview bitmap, or null
|
||||
* before any surface has been configured.
|
||||
*/
|
||||
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) {
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return null;
|
||||
}
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green);
|
||||
}
|
||||
|
||||
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
|
||||
public String saveCompositePngScreenshot() throws java.io.IOException {
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return "";
|
||||
}
|
||||
return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the current composite cropped to the given screen rect (the region
|
||||
* the shell says is actually occupied). The rect is clamped to the screen;
|
||||
* a degenerate rect falls back to the full screen.
|
||||
*/
|
||||
public String saveCompositePngScreenshot(int cropX, int cropY, int cropWidth, int cropHeight)
|
||||
throws java.io.IOException {
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return "";
|
||||
}
|
||||
int x = Math.max(0, cropX);
|
||||
int y = Math.max(0, cropY);
|
||||
int width = Math.min(composite.width - x, cropWidth - (x - cropX));
|
||||
int height = Math.min(composite.height - y, cropHeight - (y - cropY));
|
||||
if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) {
|
||||
return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height);
|
||||
}
|
||||
byte[] cropped = new byte[width * height];
|
||||
for (int row = 0; row < height; row++) {
|
||||
System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width);
|
||||
}
|
||||
return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height);
|
||||
}
|
||||
|
||||
/** Begin collecting composite frames for an animated-GIF screen recording. */
|
||||
public void startScreenRecording() {
|
||||
screenRecorder = new GifScreenRecorder();
|
||||
}
|
||||
|
||||
/** Capture the current composite into the active recording; no-op when idle. */
|
||||
public void recordScreenFrame() {
|
||||
GifScreenRecorder recorder = screenRecorder;
|
||||
if (recorder == null) {
|
||||
return;
|
||||
}
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return;
|
||||
}
|
||||
recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/** Finish the recording and save it as an animated GIF; returns the path or "". */
|
||||
public String stopScreenRecording() throws java.io.IOException {
|
||||
GifScreenRecorder recorder = screenRecorder;
|
||||
screenRecorder = null;
|
||||
if (recorder == null) {
|
||||
return "";
|
||||
}
|
||||
return recorder.save(appContext);
|
||||
}
|
||||
}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.BroadcastReceiver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
import android.util.Xml;
|
||||
|
||||
import org.xmlpull.v1.XmlPullParser;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
|
||||
/**
|
||||
* adb-triggered export/import of the faceclaw_settings SharedPreferences
|
||||
* file, so configuration (API keys, device addresses, ...) can be moved off a
|
||||
* release build, where run-as is unavailable. See scripts/pull_config.sh and
|
||||
* scripts/push_config.sh, which use run-as on debuggable builds and fall back
|
||||
* to this receiver otherwise.
|
||||
*
|
||||
* adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \
|
||||
* -a com.faceclaw.app.SETTINGS_EXPORT
|
||||
* adb pull /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-export.xml
|
||||
*
|
||||
* adb push settings.xml /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-import.xml
|
||||
* adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \
|
||||
* -a com.faceclaw.app.SETTINGS_IMPORT
|
||||
*
|
||||
* Security: the settings file contains API tokens, so the manifest guards
|
||||
* this receiver with android.permission.DUMP — a development permission the
|
||||
* adb shell holds but ordinary apps cannot obtain without adb's help, keeping
|
||||
* the trigger reachable from adb only. The transfer file lives in
|
||||
* getExternalFilesDir(), which other apps cannot read on Android 11+ (and
|
||||
* which the scripts delete as soon as the transfer completes); on the
|
||||
* Android 7-10 devices minSdk still admits, apps holding READ_EXTERNAL_STORAGE
|
||||
* could read it during that window.
|
||||
*/
|
||||
public class FaceclawSettingsPortReceiver extends BroadcastReceiver {
|
||||
private static final String TAG = "FaceclawSettingsPort";
|
||||
public static final String ACTION_EXPORT = "com.faceclaw.app.SETTINGS_EXPORT";
|
||||
public static final String ACTION_IMPORT = "com.faceclaw.app.SETTINGS_IMPORT";
|
||||
private static final String PREFS_FILE = "faceclaw_settings.xml";
|
||||
private static final String EXPORT_NAME = "faceclaw-settings-export.xml";
|
||||
private static final String IMPORT_NAME = "faceclaw-settings-import.xml";
|
||||
|
||||
@Override
|
||||
public void onReceive(Context context, Intent intent) {
|
||||
String action = intent == null ? null : intent.getAction();
|
||||
try {
|
||||
if (ACTION_EXPORT.equals(action)) {
|
||||
setResultData(exportSettings(context));
|
||||
} else if (ACTION_IMPORT.equals(action)) {
|
||||
setResultData(importSettings(context));
|
||||
// Exit once the broadcast result has been delivered: this
|
||||
// process may hold stale in-memory SharedPreferences that a
|
||||
// later apply() would flush over the imported file, and the
|
||||
// next launch must re-read from disk.
|
||||
new Handler(Looper.getMainLooper()).postDelayed(() -> {
|
||||
Log.i(TAG, "Exiting after settings import");
|
||||
System.exit(0);
|
||||
}, 500);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Settings " + action + " failed", e);
|
||||
setResultData("error: " + e);
|
||||
}
|
||||
}
|
||||
|
||||
private File prefsFile(Context context) {
|
||||
return new File(context.getApplicationInfo().dataDir, "shared_prefs/" + PREFS_FILE);
|
||||
}
|
||||
|
||||
private File externalFile(Context context, String name) throws IOException {
|
||||
File dir = context.getExternalFilesDir(null);
|
||||
if (dir == null) {
|
||||
throw new IOException("external files dir unavailable");
|
||||
}
|
||||
return new File(dir, name);
|
||||
}
|
||||
|
||||
private String exportSettings(Context context) throws IOException {
|
||||
File prefs = prefsFile(context);
|
||||
if (!prefs.exists()) {
|
||||
return "error: no settings file yet (" + prefs + ")";
|
||||
}
|
||||
File out = externalFile(context, EXPORT_NAME);
|
||||
copy(prefs, out);
|
||||
return "exported: " + out;
|
||||
}
|
||||
|
||||
private String importSettings(Context context) throws Exception {
|
||||
File staged = externalFile(context, IMPORT_NAME);
|
||||
if (!staged.exists()) {
|
||||
return "error: nothing staged at " + staged;
|
||||
}
|
||||
validatePrefsXml(staged);
|
||||
File prefs = prefsFile(context);
|
||||
prefs.getParentFile().mkdirs();
|
||||
if (prefs.exists()) {
|
||||
copy(prefs, new File(prefs.getPath() + ".bak"));
|
||||
}
|
||||
copy(staged, prefs);
|
||||
staged.delete();
|
||||
return "imported: " + prefs + " (previous settings in " + PREFS_FILE + ".bak)";
|
||||
}
|
||||
|
||||
/** Reject files that are not a SharedPreferences <map> document. */
|
||||
private void validatePrefsXml(File file) throws Exception {
|
||||
try (InputStream in = new FileInputStream(file)) {
|
||||
XmlPullParser parser = Xml.newPullParser();
|
||||
parser.setInput(in, null);
|
||||
int event = parser.next();
|
||||
while (event != XmlPullParser.START_TAG && event != XmlPullParser.END_DOCUMENT) {
|
||||
event = parser.next();
|
||||
}
|
||||
if (event != XmlPullParser.START_TAG || !"map".equals(parser.getName())) {
|
||||
throw new IOException("not a SharedPreferences <map> document");
|
||||
}
|
||||
// Walk the rest so malformed XML is caught before it replaces
|
||||
// the live settings file.
|
||||
while (parser.next() != XmlPullParser.END_DOCUMENT) { }
|
||||
}
|
||||
}
|
||||
|
||||
private void copy(File from, File to) throws IOException {
|
||||
try (InputStream in = new FileInputStream(from);
|
||||
OutputStream out = new FileOutputStream(to)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int n;
|
||||
while ((n = in.read(buffer)) > 0) {
|
||||
out.write(buffer, 0, n);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.OnlineStream;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
|
||||
|
||||
/**
|
||||
* Speaker voice-print extraction over sherpa-onnx. One embedding per
|
||||
* utterance; matching/enrollment happens on the TS side (cosine similarity
|
||||
* against the stored speaker profiles).
|
||||
*/
|
||||
public class FaceclawSpeakerId {
|
||||
private static final String TAG = "FaceclawSpeakerId";
|
||||
|
||||
private SpeakerEmbeddingExtractor extractor;
|
||||
private final String modelPath;
|
||||
|
||||
public FaceclawSpeakerId(String modelPath) {
|
||||
this.modelPath = modelPath;
|
||||
}
|
||||
|
||||
/** Lazily loads the model; returns false when it cannot be loaded. */
|
||||
public synchronized boolean ensureLoaded() {
|
||||
if (extractor != null) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder()
|
||||
.setModel(modelPath)
|
||||
.setNumThreads(1)
|
||||
.setDebug(false)
|
||||
.setProvider("cpu")
|
||||
.build();
|
||||
extractor = new SpeakerEmbeddingExtractor(config);
|
||||
return true;
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "could not load speaker model at " + modelPath, t);
|
||||
extractor = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized int embeddingDim() {
|
||||
return extractor == null ? 0 : extractor.getDim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute an L2-normalized voice-print for one utterance of 16 kHz mono
|
||||
* S16LE PCM. Returns null when the model is unavailable or the utterance
|
||||
* is too short to embed (under ~0.5 s).
|
||||
*/
|
||||
public synchronized float[] embed(byte[] pcm16le, int sampleRate) {
|
||||
if (pcm16le == null || pcm16le.length < sampleRate) {
|
||||
return null;
|
||||
}
|
||||
if (!ensureLoaded()) {
|
||||
return null;
|
||||
}
|
||||
int count = pcm16le.length / 2;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8));
|
||||
samples[i] = s / 32768.0f;
|
||||
}
|
||||
OnlineStream stream = null;
|
||||
try {
|
||||
stream = extractor.createStream();
|
||||
stream.acceptWaveform(samples, sampleRate);
|
||||
stream.inputFinished();
|
||||
if (!extractor.isReady(stream)) {
|
||||
return null;
|
||||
}
|
||||
float[] embedding = extractor.compute(stream);
|
||||
return l2Normalize(embedding);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "embedding failed", t);
|
||||
return null;
|
||||
} finally {
|
||||
if (stream != null) {
|
||||
stream.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] l2Normalize(float[] v) {
|
||||
if (v == null) {
|
||||
return null;
|
||||
}
|
||||
double sum = 0;
|
||||
for (float x : v) {
|
||||
sum += (double) x * x;
|
||||
}
|
||||
double norm = Math.sqrt(sum);
|
||||
if (norm <= 0) {
|
||||
return v;
|
||||
}
|
||||
for (int i = 0; i < v.length; i++) {
|
||||
v[i] = (float) (v[i] / norm);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
if (extractor != null) {
|
||||
extractor.release();
|
||||
extractor = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+365
-25
@@ -76,6 +76,15 @@ public class FaceclawVoiceController {
|
||||
private volatile FaceclawBleCommunicator communicator;
|
||||
private Thread workerThread;
|
||||
private volatile boolean started;
|
||||
// Set once the worker has the glasses mic enabled for this session.
|
||||
// Read and written under `lock`, so it flips with `started` atomically.
|
||||
private boolean audioStarted;
|
||||
// Capture from the phone's own microphone instead of the G2 over BLE
|
||||
// (preview-only mode, where no glasses are connected). Latched into
|
||||
// activePhoneMic at start() (under `lock`) so a mid-session setter call
|
||||
// can't switch pipelines underneath the worker.
|
||||
private volatile boolean usePhoneMic;
|
||||
private boolean activePhoneMic;
|
||||
private VoiceInputMode mode = VoiceInputMode.CLOUD;
|
||||
private OfflineRecognizer recognizer;
|
||||
private FaceclawLc3Decoder lc3Decoder;
|
||||
@@ -90,6 +99,26 @@ public class FaceclawVoiceController {
|
||||
private volatile boolean endpointing;
|
||||
private final EndpointDetector endpointDetector = new EndpointDetector();
|
||||
private java.io.ByteArrayOutputStream recordingPcm;
|
||||
// Speaker verification against the enrolled wearer voice-print ("my voice
|
||||
// only" command gating). Configured before start(); the utterance PCM is
|
||||
// buffered (capped) and verified once at session end.
|
||||
private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10;
|
||||
private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE;
|
||||
private volatile String verifySpeakerModelPath;
|
||||
private volatile float[] verifyWearerEmbedding;
|
||||
private volatile float verifyThreshold = 0.8f;
|
||||
private short[] verifyBuffer;
|
||||
private int verifyCount;
|
||||
// Global mic processing (Microphones app config): spectral noise
|
||||
// suppression and firmware-DoA beam gating, applied to every capture
|
||||
// session that opts in (assistant push-to-talk, Transcribe, hands-free).
|
||||
// The raw tap opts out — raw means raw, and the Microphones session does
|
||||
// its own beam-compensated processing on that path.
|
||||
private volatile boolean suppressionEnabled;
|
||||
private volatile boolean beamFilterEnabled;
|
||||
private volatile int beamCenterDeg;
|
||||
private volatile int beamHalfWidthDeg = 180;
|
||||
private FaceclawNoiseSuppressor suppressor;
|
||||
private long queuedPackets;
|
||||
private long queueDroppedPackets;
|
||||
private long decodedSamples;
|
||||
@@ -111,6 +140,11 @@ public class FaceclawVoiceController {
|
||||
this.communicator = communicator;
|
||||
}
|
||||
|
||||
/** Source the next capture from the phone microphone (no glasses paired). */
|
||||
public void setUsePhoneMic(boolean usePhoneMic) {
|
||||
this.usePhoneMic = usePhoneMic;
|
||||
}
|
||||
|
||||
/** When true, the decoded mic PCM for each session is saved as a WAV. */
|
||||
public void setSaveRecordings(boolean saveRecordings) {
|
||||
this.saveRecordings = saveRecordings;
|
||||
@@ -125,23 +159,117 @@ public class FaceclawVoiceController {
|
||||
this.endpointing = endpointing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify this session's speaker against the enrolled wearer voice-print
|
||||
* and report the result via onSpeakerVerified just before the final
|
||||
* transcript. Must be set before {@link #start}; pass a null model path
|
||||
* to disable.
|
||||
*/
|
||||
public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) {
|
||||
this.verifySpeakerModelPath = speakerModelPath;
|
||||
this.verifyWearerEmbedding = wearerEmbedding;
|
||||
this.verifyThreshold = threshold > 0 ? threshold : 0.8f;
|
||||
}
|
||||
|
||||
public void clearSpeakerVerification() {
|
||||
this.verifySpeakerModelPath = null;
|
||||
this.verifyWearerEmbedding = null;
|
||||
}
|
||||
|
||||
/** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */
|
||||
public void setNoiseSuppression(boolean enabled) {
|
||||
this.suppressionEnabled = enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Direction gating from the Sonic Radar beam: packets whose firmware
|
||||
* direction-of-arrival falls outside centerDeg ± halfWidthDeg (device
|
||||
* frame, 0 = straight ahead, positive right) are dropped before any
|
||||
* consumer sees them. Safe to update mid-run.
|
||||
*/
|
||||
public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) {
|
||||
this.beamFilterEnabled = enabled;
|
||||
this.beamCenterDeg = centerDeg;
|
||||
this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg));
|
||||
}
|
||||
|
||||
private boolean withinBeam(int angleDegrees) {
|
||||
int delta = angleDegrees - beamCenterDeg;
|
||||
while (delta > 180) delta -= 360;
|
||||
while (delta < -180) delta += 360;
|
||||
return Math.abs(delta) <= beamHalfWidthDeg;
|
||||
}
|
||||
|
||||
private void applySuppression(short[] pcm, int count) {
|
||||
try {
|
||||
if (suppressor == null) {
|
||||
suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE);
|
||||
}
|
||||
byte[] le = new byte[count * 2];
|
||||
for (int i = 0; i < count; i++) {
|
||||
le[i * 2] = (byte) (pcm[i] & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff);
|
||||
}
|
||||
byte[] cleaned = suppressor.process(le);
|
||||
int cleanedCount = Math.min(count, cleaned.length / 2);
|
||||
for (int i = 0; i < cleanedCount; i++) {
|
||||
pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8));
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "noise suppression failed; passing audio through", t);
|
||||
suppressionEnabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
public void start(String requestedMode) {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
emitStatus("Voice control is already listening.");
|
||||
return;
|
||||
}
|
||||
if (communicator == null || !communicator.isSessionReady()) {
|
||||
if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) {
|
||||
emitStatus("Voice control needs an active G2 connection.");
|
||||
return;
|
||||
}
|
||||
mode = parseMode(requestedMode);
|
||||
activePhoneMic = usePhoneMic;
|
||||
started = true;
|
||||
audioStarted = false;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether mic audio is actually flowing. {@link #start} only records
|
||||
* intent: the enable lives in the glasses' EvenHub session, so a transport
|
||||
* drop or a session suspend can leave this controller started with a
|
||||
* worker that will never see another packet. Anything deciding whether to
|
||||
* (re)start capture must ask this rather than assume its own bookkeeping.
|
||||
*/
|
||||
public boolean isCapturing() {
|
||||
boolean audioUp;
|
||||
boolean phoneMic;
|
||||
synchronized (lock) {
|
||||
if (!started) {
|
||||
return false;
|
||||
}
|
||||
audioUp = audioStarted;
|
||||
phoneMic = activePhoneMic;
|
||||
}
|
||||
if (!audioUp) {
|
||||
// The worker is still bringing the mic up; report it as running so
|
||||
// a concurrent request shares it instead of restarting it.
|
||||
return true;
|
||||
}
|
||||
if (phoneMic) {
|
||||
// AudioRecord has no session to lose the enable to; it runs until stop().
|
||||
return true;
|
||||
}
|
||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||
return currentCommunicator != null && currentCommunicator.isAudioCaptureActive();
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
Thread threadToJoin;
|
||||
synchronized (lock) {
|
||||
@@ -193,18 +321,54 @@ public class FaceclawVoiceController {
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
}
|
||||
lc3Decoder = new FaceclawLc3Decoder();
|
||||
endpointDetector.reset();
|
||||
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.");
|
||||
return;
|
||||
if (suppressor != null) {
|
||||
suppressor.reset();
|
||||
}
|
||||
|
||||
emitStatus(currentMode == VoiceInputMode.CLOUD
|
||||
? "Listening (cloud)..."
|
||||
: "Listening...");
|
||||
processG2Audio();
|
||||
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
|
||||
boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null;
|
||||
verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null;
|
||||
verifyCount = 0;
|
||||
if (activePhoneMic) {
|
||||
android.media.AudioRecord record = openPhoneMic();
|
||||
if (record == null) {
|
||||
emitStatus("Could not start the phone microphone.");
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
audioStarted = true;
|
||||
}
|
||||
emitStatus(currentMode == VoiceInputMode.CLOUD
|
||||
? "Listening (cloud)..."
|
||||
: "Listening...");
|
||||
try {
|
||||
processPhoneAudio(record);
|
||||
} finally {
|
||||
try {
|
||||
record.stop();
|
||||
} catch (Throwable ignored) {
|
||||
// Already stopped or never recording; release below either way.
|
||||
}
|
||||
record.release();
|
||||
}
|
||||
} else {
|
||||
lc3Decoder = new FaceclawLc3Decoder();
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.");
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
audioStarted = true;
|
||||
}
|
||||
emitStatus(currentMode == VoiceInputMode.CLOUD
|
||||
? "Listening (cloud)..."
|
||||
: "Listening...");
|
||||
processG2Audio();
|
||||
}
|
||||
// Verification result must precede the final transcript so the
|
||||
// TS bridge can suppress a non-wearer command before it is acted
|
||||
// on (the callbacks are posted in order to the main handler).
|
||||
runSpeakerVerification();
|
||||
// Button released / stop requested: emit one final full-utterance
|
||||
// transcript so the UI can freeze it.
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
@@ -220,6 +384,7 @@ public class FaceclawVoiceController {
|
||||
releaseLc3();
|
||||
synchronized (lock) {
|
||||
started = false;
|
||||
audioStarted = false;
|
||||
workerThread = null;
|
||||
}
|
||||
}
|
||||
@@ -374,25 +539,123 @@ public class FaceclawVoiceController {
|
||||
continue;
|
||||
}
|
||||
decodedSamples += count;
|
||||
if (recordingPcm != null) {
|
||||
appendRecording(pcm, count);
|
||||
}
|
||||
if (endpointing && endpointDetector.accept(pcm, count)) {
|
||||
emitSpeechEnd();
|
||||
}
|
||||
if (mode == VoiceInputMode.CLOUD) {
|
||||
emitPcm(pcm, count);
|
||||
} else {
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
processRecognizer(samples);
|
||||
// Global mic processing from the Microphones app config: the
|
||||
// beam filter drops packets whose firmware direction-of-arrival
|
||||
// falls outside the listening wedge (isolating the aimed talker
|
||||
// for every consumer, recognition included), and the spectral
|
||||
// noise suppressor cleans what remains before it reaches the
|
||||
// recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
int angleDegrees = currentDecoder.getLastAngleDegrees();
|
||||
int ssr = currentDecoder.getLastSsr();
|
||||
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
|
||||
emitFrameMeta(angleDegrees, ssr);
|
||||
maybeEmitAudioStats(false);
|
||||
continue;
|
||||
}
|
||||
processPcmChunk(pcm, count, angleDegrees, ssr, true);
|
||||
maybeEmitAudioStats(false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
|
||||
* decode and the beam filter. hasFrameMeta is false for the phone mic,
|
||||
* which has no firmware DSP metadata to report.
|
||||
*/
|
||||
private void processPcmChunk(short[] pcm, int count, int angleDegrees, int ssr, boolean hasFrameMeta) {
|
||||
if (suppressionEnabled) {
|
||||
applySuppression(pcm, count);
|
||||
}
|
||||
if (recordingPcm != null) {
|
||||
appendRecording(pcm, count);
|
||||
}
|
||||
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
|
||||
int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount);
|
||||
System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied);
|
||||
verifyCount += copied;
|
||||
}
|
||||
if (endpointing && endpointDetector.accept(pcm, count)) {
|
||||
emitSpeechEnd();
|
||||
}
|
||||
// PCM and frame metadata flow in every mode so levels, recording,
|
||||
// and the Microphones radar keep working alongside onboard ASR.
|
||||
emitPcm(pcm, count);
|
||||
if (hasFrameMeta) {
|
||||
emitFrameMeta(angleDegrees, ssr);
|
||||
}
|
||||
if (mode != VoiceInputMode.CLOUD) {
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
processRecognizer(samples);
|
||||
}
|
||||
}
|
||||
|
||||
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
||||
// (endpointing, transcript pacing) is tuned for.
|
||||
private static final int PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20;
|
||||
|
||||
/**
|
||||
* Open the phone's own microphone at the pipeline's native format
|
||||
* (16 kHz mono PCM16), or null when it cannot start — the permission is
|
||||
* missing (SecurityException) or the device refuses the configuration.
|
||||
*/
|
||||
private android.media.AudioRecord openPhoneMic() {
|
||||
android.media.AudioRecord record = null;
|
||||
try {
|
||||
int minBytes = android.media.AudioRecord.getMinBufferSize(
|
||||
SAMPLE_RATE,
|
||||
android.media.AudioFormat.CHANNEL_IN_MONO,
|
||||
android.media.AudioFormat.ENCODING_PCM_16BIT);
|
||||
int bufferBytes = Math.max(minBytes, PHONE_MIC_CHUNK_SAMPLES * 2 * 4);
|
||||
record = new android.media.AudioRecord(
|
||||
android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION,
|
||||
SAMPLE_RATE,
|
||||
android.media.AudioFormat.CHANNEL_IN_MONO,
|
||||
android.media.AudioFormat.ENCODING_PCM_16BIT,
|
||||
bufferBytes);
|
||||
if (record.getState() != android.media.AudioRecord.STATE_INITIALIZED) {
|
||||
record.release();
|
||||
return null;
|
||||
}
|
||||
record.startRecording();
|
||||
if (record.getRecordingState() != android.media.AudioRecord.RECORDSTATE_RECORDING) {
|
||||
record.release();
|
||||
return null;
|
||||
}
|
||||
return record;
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "phone mic open failed", t);
|
||||
if (record != null) {
|
||||
record.release();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame
|
||||
* metadata — AudioRecord already delivers the pipeline's PCM format. The
|
||||
* blocking read returns every chunk (50 ms), which bounds how long a
|
||||
* stop() waits for the loop to notice `started` dropped.
|
||||
*/
|
||||
private void processPhoneAudio(android.media.AudioRecord record) {
|
||||
short[] pcm = new short[PHONE_MIC_CHUNK_SAMPLES];
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
int read = record.read(pcm, 0, pcm.length);
|
||||
if (read < 0) {
|
||||
Log.w(TAG, "phone mic read failed: " + read);
|
||||
return;
|
||||
}
|
||||
if (read == 0) {
|
||||
continue;
|
||||
}
|
||||
decodedSamples += read;
|
||||
processPcmChunk(pcm, read, 0, 0, false);
|
||||
}
|
||||
}
|
||||
|
||||
private void processRecognizer(float[] samples) {
|
||||
appendTranscriptSamples(samples);
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
@@ -582,6 +845,83 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(() -> currentListener.onPcm(le));
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed the session's buffered utterance and compare it to the enrolled
|
||||
* wearer voice-print. Fails open: a session too short to verify, or a
|
||||
* model that will not load, counts as the wearer rather than silencing
|
||||
* every command.
|
||||
*/
|
||||
private void runSpeakerVerification() {
|
||||
short[] buffer = verifyBuffer;
|
||||
float[] wearer = verifyWearerEmbedding;
|
||||
String modelPath = verifySpeakerModelPath;
|
||||
verifyBuffer = null;
|
||||
if (buffer == null || wearer == null || modelPath == null) {
|
||||
return;
|
||||
}
|
||||
if (verifyCount < VERIFY_MIN_SAMPLES) {
|
||||
emitSpeakerVerified(true, 0f);
|
||||
return;
|
||||
}
|
||||
FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath);
|
||||
try {
|
||||
byte[] le = new byte[verifyCount * 2];
|
||||
for (int i = 0; i < verifyCount; i++) {
|
||||
short s = buffer[i];
|
||||
le[i * 2] = (byte) (s & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
|
||||
}
|
||||
float[] embedding = speakerId.embed(le, SAMPLE_RATE);
|
||||
if (embedding == null || embedding.length != wearer.length) {
|
||||
emitSpeakerVerified(true, 0f);
|
||||
return;
|
||||
}
|
||||
double dot = 0;
|
||||
for (int i = 0; i < embedding.length; i++) {
|
||||
dot += (double) embedding[i] * wearer[i];
|
||||
}
|
||||
boolean isWearer = dot >= verifyThreshold;
|
||||
Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot)
|
||||
+ " threshold=" + verifyThreshold + " isWearer=" + isWearer);
|
||||
emitSpeakerVerified(isWearer, (float) dot);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "speaker verification failed", t);
|
||||
emitSpeakerVerified(true, 0f);
|
||||
}
|
||||
}
|
||||
|
||||
// The 28 MB embedding model takes seconds to load; keep one instance
|
||||
// across capture sessions so verification adds only the embed time.
|
||||
private static FaceclawSpeakerId sharedSpeakerId;
|
||||
private static String sharedSpeakerIdPath;
|
||||
|
||||
private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) {
|
||||
if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) {
|
||||
if (sharedSpeakerId != null) {
|
||||
sharedSpeakerId.close();
|
||||
}
|
||||
sharedSpeakerId = new FaceclawSpeakerId(modelPath);
|
||||
sharedSpeakerIdPath = modelPath;
|
||||
}
|
||||
return sharedSpeakerId;
|
||||
}
|
||||
|
||||
private void emitSpeakerVerified(boolean isWearer, float similarity) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity));
|
||||
}
|
||||
|
||||
private void emitFrameMeta(int angleDegrees, int ssr) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr));
|
||||
}
|
||||
|
||||
private void emitSpeechEnd() {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
|
||||
+21
-2
@@ -11,15 +11,34 @@ public interface FaceclawVoiceControllerListener {
|
||||
void onTranscript(String text, boolean isFinal);
|
||||
|
||||
/**
|
||||
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
|
||||
* little-endian PCM. Empty/absent in onboard mode.
|
||||
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
|
||||
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
|
||||
* keep working while the onboard recognizer owns the transcript.
|
||||
*/
|
||||
void onPcm(byte[] pcm16le);
|
||||
|
||||
/**
|
||||
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
|
||||
* packet, computed on the raw stereo capture before its mono downmix: the
|
||||
* direction-of-arrival in signed degrees (0 = straight ahead, positive to
|
||||
* the wearer's right) and a signal-strength ratio that serves as a
|
||||
* confidence/energy proxy for the angle.
|
||||
*/
|
||||
void onFrameMeta(int angleDegrees, int ssr);
|
||||
|
||||
/**
|
||||
* The speaker stopped, in a hands-free session that has no button release
|
||||
* to end it. Only fires when endpointing was enabled for the session; see
|
||||
* FaceclawVoiceController.setEndpointing.
|
||||
*/
|
||||
void onSpeechEnd();
|
||||
|
||||
/**
|
||||
* Result of speaker verification against the enrolled wearer voice-print,
|
||||
* emitted once per capture session just before the final transcript.
|
||||
* Only fires when verification was configured for the session; fails
|
||||
* open (isWearer=true) when the utterance was too short to verify or the
|
||||
* speaker model could not be loaded.
|
||||
*/
|
||||
void onSpeakerVerified(boolean isWearer, float similarity);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* Streaming WAV writer for conversation recordings: the header is written up
|
||||
* front with zero sizes and patched on finish, so a crash mid-session leaves
|
||||
* a recoverable file (data length can be derived from the file size).
|
||||
*/
|
||||
public class FaceclawWavRecorder {
|
||||
private static final String TAG = "FaceclawWavRecorder";
|
||||
|
||||
private final String path;
|
||||
private final int sampleRate;
|
||||
private final int channels;
|
||||
private RandomAccessFile file;
|
||||
private long dataBytes;
|
||||
|
||||
public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException {
|
||||
this.path = path;
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
File parent = new File(path).getParentFile();
|
||||
if (parent != null && !parent.exists() && !parent.mkdirs()) {
|
||||
throw new IOException("could not create " + parent);
|
||||
}
|
||||
file = new RandomAccessFile(path, "rw");
|
||||
file.setLength(0);
|
||||
file.write(buildHeader(0));
|
||||
}
|
||||
|
||||
public synchronized void append(byte[] pcm16le) {
|
||||
if (file == null || pcm16le == null || pcm16le.length == 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
file.write(pcm16le);
|
||||
dataBytes += pcm16le.length;
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "append failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized long getDurationMs() {
|
||||
return dataBytes * 1000L / (sampleRate * channels * 2L);
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Patch the header and close. Returns the PCM byte count (0 = empty). */
|
||||
public synchronized long finish() {
|
||||
RandomAccessFile current = file;
|
||||
file = null;
|
||||
if (current == null) {
|
||||
return dataBytes;
|
||||
}
|
||||
try {
|
||||
current.seek(0);
|
||||
current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE)));
|
||||
current.close();
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "finish failed", e);
|
||||
}
|
||||
return dataBytes;
|
||||
}
|
||||
|
||||
/** Delete the (finished or abandoned) recording. */
|
||||
public synchronized void discard() {
|
||||
finish();
|
||||
new File(path).delete();
|
||||
}
|
||||
|
||||
private byte[] buildHeader(int pcmBytes) {
|
||||
int byteRate = sampleRate * channels * 2;
|
||||
ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN);
|
||||
b.put("RIFF".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(36 + pcmBytes);
|
||||
b.put("WAVE".getBytes(StandardCharsets.US_ASCII));
|
||||
b.put("fmt ".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(16);
|
||||
b.putShort((short) 1);
|
||||
b.putShort((short) channels);
|
||||
b.putInt(sampleRate);
|
||||
b.putInt(byteRate);
|
||||
b.putShort((short) (channels * 2));
|
||||
b.putShort((short) 16);
|
||||
b.put("data".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(pcmBytes);
|
||||
return b.array();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.Context;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.google.android.gms.common.ConnectionResult;
|
||||
import com.google.android.gms.common.GoogleApiAvailability;
|
||||
import com.google.android.gms.wearable.CapabilityClient;
|
||||
import com.google.android.gms.wearable.CapabilityInfo;
|
||||
import com.google.android.gms.wearable.DataMap;
|
||||
import com.google.android.gms.wearable.MessageEvent;
|
||||
import com.google.android.gms.wearable.Node;
|
||||
import com.google.android.gms.wearable.PutDataMapRequest;
|
||||
import com.google.android.gms.wearable.PutDataRequest;
|
||||
import com.google.android.gms.wearable.Wearable;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Phone side of the Wear OS integration: the single place that talks to the
|
||||
* Wearable Data Layer (Google Play services), so the JS side only ever sees
|
||||
* JSON strings and never touches Play services types.
|
||||
*
|
||||
* Inbound: FaceclawWearListenerService hands every watch message to
|
||||
* {@link #handleMessage}; it is forwarded to the JS listener on the thread
|
||||
* that registered it (the main isolate), mirroring FaceclawSettings. Messages
|
||||
* that arrive before a listener exists (the watch poked a phone whose JS
|
||||
* dashboard is not up yet) are acked with jsReady=false so the watch can say
|
||||
* "open Faceclaw on your phone"; only idempotent state requests are kept for
|
||||
* replay on registration — the watch already reported gestures and commands
|
||||
* as failed, and executing them tens of seconds later would act on whatever
|
||||
* the glasses happen to show by then.
|
||||
*
|
||||
* Outbound: {@link #publishState} mirrors the dashboard state into a Data
|
||||
* Layer item (delivered even if the watch is out of range right now, and
|
||||
* available to the watch app the moment it opens), while {@link #sendToWatch}
|
||||
* sends fire-and-forget messages (acks, assistant text) to every reachable
|
||||
* watch that advertises the Faceclaw watch-app capability.
|
||||
*
|
||||
* Everything is a no-op on phones without Google Play services.
|
||||
*/
|
||||
public final class FaceclawWearBridge {
|
||||
private static final String TAG = "FaceclawWear";
|
||||
|
||||
/** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */
|
||||
public static final String CAPABILITY_WATCH = "faceclaw_watch";
|
||||
/** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */
|
||||
public static final String CAPABILITY_PHONE = "faceclaw_phone";
|
||||
|
||||
public static final String PATH_PREFIX = "/faceclaw";
|
||||
public static final String PATH_STATE = "/faceclaw/state";
|
||||
public static final String PATH_STATE_REQUEST = "/faceclaw/state/request";
|
||||
public static final String PATH_ACK = "/faceclaw/ack";
|
||||
public static final String PATH_EVENT = "/faceclaw/event";
|
||||
|
||||
private static final String STATE_KEY_JSON = "json";
|
||||
private static final String STATE_KEY_UPDATED_AT = "updatedAt";
|
||||
|
||||
/** Messages kept for a listener that is not registered yet. */
|
||||
private static final int PENDING_LIMIT = 16;
|
||||
private static final long PENDING_MAX_AGE_MS = 30_000;
|
||||
|
||||
private static volatile FaceclawWearBridge instance;
|
||||
|
||||
private final Context context;
|
||||
private final boolean available;
|
||||
private final Object lock = new Object();
|
||||
private FaceclawWearListener listener;
|
||||
private Handler listenerHandler;
|
||||
private final ArrayDeque<PendingMessage> pending = new ArrayDeque<>();
|
||||
private final List<Node> watchNodes = new ArrayList<>();
|
||||
private String lastPublishedState = null;
|
||||
|
||||
private static final class PendingMessage {
|
||||
final String path;
|
||||
final String json;
|
||||
final String nodeId;
|
||||
final long receivedAt;
|
||||
|
||||
PendingMessage(String path, String json, String nodeId, long receivedAt) {
|
||||
this.path = path;
|
||||
this.json = json;
|
||||
this.nodeId = nodeId;
|
||||
this.receivedAt = receivedAt;
|
||||
}
|
||||
}
|
||||
|
||||
private FaceclawWearBridge(Context context) {
|
||||
this.context = context.getApplicationContext();
|
||||
boolean playServices;
|
||||
try {
|
||||
playServices = GoogleApiAvailability.getInstance()
|
||||
.isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS;
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "Play services availability check failed", error);
|
||||
playServices = false;
|
||||
}
|
||||
this.available = playServices;
|
||||
if (!available) {
|
||||
Log.i(TAG, "Google Play services unavailable; watch integration disabled");
|
||||
}
|
||||
}
|
||||
|
||||
/** Initialize (idempotent) and return the singleton. */
|
||||
public static FaceclawWearBridge getInstance(Context context) {
|
||||
if (instance == null) {
|
||||
synchronized (FaceclawWearBridge.class) {
|
||||
if (instance == null) {
|
||||
instance = new FaceclawWearBridge(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
/** Whether the Wearable Data Layer can be used on this phone at all. */
|
||||
public boolean isAvailable() {
|
||||
return available;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the JS listener. Must be called from the thread whose isolate
|
||||
* owns it; that thread's Looper is captured for dispatch. Replays any
|
||||
* recently queued watch messages and refreshes the reachable-watch set.
|
||||
*/
|
||||
public void setListener(FaceclawWearListener newListener) {
|
||||
Looper looper = Looper.myLooper();
|
||||
if (looper == null) {
|
||||
Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified");
|
||||
}
|
||||
List<PendingMessage> replay;
|
||||
synchronized (lock) {
|
||||
listener = newListener;
|
||||
listenerHandler = looper != null ? new Handler(looper) : null;
|
||||
replay = new ArrayList<>(pending);
|
||||
pending.clear();
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
for (PendingMessage message : replay) {
|
||||
if (now - message.receivedAt <= PENDING_MAX_AGE_MS) {
|
||||
dispatchMessage(message.path, message.json, message.nodeId);
|
||||
}
|
||||
}
|
||||
refreshWatchNodes();
|
||||
}
|
||||
|
||||
public void clearListener() {
|
||||
synchronized (lock) {
|
||||
listener = null;
|
||||
listenerHandler = null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isWatchReachable() {
|
||||
synchronized (lock) {
|
||||
return !watchNodes.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
/** Display name of a reachable watch ("" if none). */
|
||||
public String getWatchName() {
|
||||
synchronized (lock) {
|
||||
return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-query which watches advertise the Faceclaw watch-app capability. */
|
||||
public void refreshWatchNodes() {
|
||||
if (!available) return;
|
||||
try {
|
||||
Wearable.getCapabilityClient(context)
|
||||
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
|
||||
.addOnSuccessListener(this::updateWatchNodes)
|
||||
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "capability query failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror the dashboard state to the watch. Identical consecutive states
|
||||
* are not re-sent (the Data Layer would drop them anyway).
|
||||
*/
|
||||
public void publishState(String json) {
|
||||
publishState(json, false);
|
||||
}
|
||||
|
||||
/** As {@link #publishState(String)}; `force` re-sends an unchanged state. */
|
||||
public void publishState(String json, boolean force) {
|
||||
if (!available || json == null) return;
|
||||
synchronized (lock) {
|
||||
if (!force && json.equals(lastPublishedState)) return;
|
||||
lastPublishedState = json;
|
||||
}
|
||||
try {
|
||||
PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE);
|
||||
DataMap map = request.getDataMap();
|
||||
map.putString(STATE_KEY_JSON, json);
|
||||
map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis());
|
||||
PutDataRequest put = request.asPutDataRequest().setUrgent();
|
||||
Wearable.getDataClient(context)
|
||||
.putDataItem(put)
|
||||
.addOnFailureListener(error -> Log.w(TAG, "state publish failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "state publish failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
/** Send a message to every reachable watch running the Faceclaw watch app. */
|
||||
public void sendToWatch(String path, String json) {
|
||||
if (!available) return;
|
||||
List<Node> targets;
|
||||
synchronized (lock) {
|
||||
targets = new ArrayList<>(watchNodes);
|
||||
}
|
||||
if (targets.isEmpty()) {
|
||||
// Nothing known yet (first message after boot): look the watch up, then send.
|
||||
try {
|
||||
Wearable.getCapabilityClient(context)
|
||||
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
|
||||
.addOnSuccessListener(info -> {
|
||||
updateWatchNodes(info);
|
||||
for (Node node : info.getNodes()) {
|
||||
sendToNode(node.getId(), path, json);
|
||||
}
|
||||
})
|
||||
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "capability query failed", error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (Node node : targets) {
|
||||
sendToNode(node.getId(), path, json);
|
||||
}
|
||||
}
|
||||
|
||||
/** Send a message to one specific node (e.g. an ack back to the sender). */
|
||||
public void sendToNode(String nodeId, String path, String json) {
|
||||
if (!available || nodeId == null || nodeId.isEmpty()) return;
|
||||
try {
|
||||
byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8);
|
||||
Wearable.getMessageClient(context)
|
||||
.sendMessage(nodeId, path, payload)
|
||||
.addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "send " + path + " failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reply to a watch message; `seq` echoes the watch's sequence number. */
|
||||
public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) {
|
||||
JSONObject ack = new JSONObject();
|
||||
try {
|
||||
ack.put("seq", seq);
|
||||
ack.put("ok", ok);
|
||||
ack.put("jsReady", jsReady);
|
||||
ack.put("message", message != null ? message : "");
|
||||
} catch (JSONException ignored) {
|
||||
// The keys are constants; this cannot happen.
|
||||
}
|
||||
sendToNode(nodeId, PATH_ACK, ack.toString());
|
||||
}
|
||||
|
||||
/** Entry point for FaceclawWearListenerService; may run on any thread. */
|
||||
void handleMessage(MessageEvent event) {
|
||||
if (event == null) return;
|
||||
String path = event.getPath();
|
||||
if (path == null || !path.startsWith(PATH_PREFIX)) return;
|
||||
byte[] data = event.getData();
|
||||
String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}";
|
||||
String nodeId = event.getSourceNodeId();
|
||||
Log.d(TAG, "recv " + path + " seq=" + readSeq(json));
|
||||
|
||||
boolean delivered = dispatchMessage(path, json, nodeId);
|
||||
if (delivered) return;
|
||||
|
||||
// No JS listener yet: tell the watch so it can explain why nothing is
|
||||
// happening. Only state requests are kept for replay (see class doc);
|
||||
// everything else is dropped after the failure ack.
|
||||
if (PATH_STATE_REQUEST.equals(path)) {
|
||||
synchronized (lock) {
|
||||
long now = System.currentTimeMillis();
|
||||
while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) {
|
||||
pending.pollFirst();
|
||||
}
|
||||
if (pending.size() >= PENDING_LIMIT) {
|
||||
pending.pollFirst();
|
||||
}
|
||||
pending.addLast(new PendingMessage(path, json, nodeId, now));
|
||||
}
|
||||
}
|
||||
sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses.");
|
||||
}
|
||||
|
||||
/** Entry point for FaceclawWearListenerService's capability callback. */
|
||||
void handleCapabilityChanged(CapabilityInfo info) {
|
||||
if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return;
|
||||
updateWatchNodes(info);
|
||||
}
|
||||
|
||||
private void updateWatchNodes(CapabilityInfo info) {
|
||||
Set<Node> nodes = info != null ? info.getNodes() : Collections.<Node>emptySet();
|
||||
// Every reachable watch gets messages; a nearby (Bluetooth-linked)
|
||||
// one is listed first so it is the name the phone UI shows.
|
||||
List<Node> reachable = new ArrayList<>();
|
||||
for (Node node : nodes) {
|
||||
if (node.isNearby()) reachable.add(0, node); else reachable.add(node);
|
||||
}
|
||||
boolean changed;
|
||||
String name;
|
||||
synchronized (lock) {
|
||||
changed = !sameNodeIds(watchNodes, reachable);
|
||||
watchNodes.clear();
|
||||
watchNodes.addAll(reachable);
|
||||
name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
|
||||
}
|
||||
if (changed) {
|
||||
Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name);
|
||||
boolean reachableNow = !reachable.isEmpty();
|
||||
FaceclawWearListener target;
|
||||
Handler handler;
|
||||
synchronized (lock) {
|
||||
target = listener;
|
||||
handler = listenerHandler;
|
||||
}
|
||||
if (target != null && handler != null) {
|
||||
handler.post(() -> {
|
||||
try {
|
||||
target.onWatchConnection(reachableNow, name);
|
||||
} catch (Exception error) {
|
||||
Log.w(TAG, "watch connection listener failed", error);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean sameNodeIds(List<Node> a, List<Node> b) {
|
||||
if (a.size() != b.size()) return false;
|
||||
for (int i = 0; i < a.size(); i++) {
|
||||
if (!a.get(i).getId().equals(b.get(i).getId())) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Post to the JS listener; false when there is none to post to. */
|
||||
private boolean dispatchMessage(String path, String json, String nodeId) {
|
||||
FaceclawWearListener target;
|
||||
Handler handler;
|
||||
synchronized (lock) {
|
||||
target = listener;
|
||||
handler = listenerHandler;
|
||||
}
|
||||
if (target == null || handler == null) return false;
|
||||
handler.post(() -> {
|
||||
try {
|
||||
target.onMessage(path, json, nodeId);
|
||||
} catch (Exception error) {
|
||||
Log.w(TAG, "wear message listener failed for " + path, error);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
private static long readSeq(String json) {
|
||||
try {
|
||||
return new JSONObject(json).optLong("seq", 0);
|
||||
} catch (JSONException error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
/**
|
||||
* Callbacks from the Android Wear (Wear OS) bridge; implemented in JS. Every
|
||||
* call is delivered on the thread that registered the listener (see
|
||||
* FaceclawWearBridge.setListener).
|
||||
*/
|
||||
public interface FaceclawWearListener {
|
||||
/**
|
||||
* A message from the watch app. `path` is the Data Layer message path
|
||||
* (e.g. "/faceclaw/input"); `json` is the UTF-8 payload (always a JSON
|
||||
* object, "{}" when the watch sent none); `nodeId` identifies the watch
|
||||
* so a reply can be addressed to it.
|
||||
*/
|
||||
void onMessage(String path, String json, String nodeId);
|
||||
|
||||
/** The set of reachable watches running the Faceclaw watch app changed. */
|
||||
void onWatchConnection(boolean reachable, String watchName);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import com.google.android.gms.wearable.CapabilityInfo;
|
||||
import com.google.android.gms.wearable.MessageEvent;
|
||||
import com.google.android.gms.wearable.WearableListenerService;
|
||||
|
||||
/**
|
||||
* Play services starts this service (in the app process) for every Data
|
||||
* Layer message under /faceclaw and for capability changes, whether or not
|
||||
* the app is running. It only forwards to FaceclawWearBridge, which owns the
|
||||
* hand-off to the JS side.
|
||||
*/
|
||||
public class FaceclawWearListenerService extends WearableListenerService {
|
||||
@Override
|
||||
public void onMessageReceived(MessageEvent event) {
|
||||
Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId());
|
||||
FaceclawWearBridge.getInstance(this).handleMessage(event);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCapabilityChanged(CapabilityInfo info) {
|
||||
FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info);
|
||||
}
|
||||
}
|
||||
@@ -15,13 +15,18 @@ public final class PreviewBitmapUtil {
|
||||
private PreviewBitmapUtil() {}
|
||||
|
||||
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) {
|
||||
return fromGray(gray, width, height, brightenGamma, false);
|
||||
}
|
||||
|
||||
/** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */
|
||||
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma, boolean green) {
|
||||
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
|
||||
throw new IllegalArgumentException("invalid gray preview buffer");
|
||||
}
|
||||
int[] lut = new int[256];
|
||||
for (int g = 0; g < 256; g++) {
|
||||
int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma))));
|
||||
lut[g] = 0xff000000 | (v << 16) | (v << 8) | v;
|
||||
lut[g] = green ? (0xff000000 | (v << 8)) : (0xff000000 | (v << 16) | (v << 8) | v);
|
||||
}
|
||||
byte[] bytes = new byte[width * height];
|
||||
gray.get(bytes);
|
||||
|
||||
@@ -49,6 +49,14 @@ public class BleProtocol {
|
||||
// protobuf fields to stock implementations, so they are safely ignored.
|
||||
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
|
||||
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
|
||||
/**
|
||||
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
|
||||
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
|
||||
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
|
||||
* to settings read responses and as standalone commandId=3 pushes.
|
||||
*/
|
||||
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
|
||||
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
|
||||
public static final int FACECLAW_WAKE_OP_ACQUIRE = 1;
|
||||
public static final int FACECLAW_WAKE_OP_RELEASE = 2;
|
||||
public static final int FACECLAW_WAKE_OP_CLAIM = 3;
|
||||
@@ -405,6 +413,40 @@ public class BleProtocol {
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a
|
||||
* settings write carrying CFW field 103. Magic zero makes it
|
||||
* fire-and-forget like the wake lease: the firmware consumes the field
|
||||
* before its stock nanopb decoder discards the unknown tag, and the
|
||||
* "ack" is the field-104 status notify that follows.
|
||||
*/
|
||||
public static byte[] buildFaceclawMicControl(byte[] record) {
|
||||
return concat(CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1),
|
||||
encodeVarintField(2, 0),
|
||||
encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the CFW mic status record (field 104) from a settings frame, or
|
||||
* null when absent. The 21-byte body layout is decoded on the TS side;
|
||||
* here we only validate the 'M','C',version prelude.
|
||||
*/
|
||||
public static byte[] parseFaceclawMicStatus(byte[] pb) {
|
||||
if (pb == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD);
|
||||
if (body == null
|
||||
|| body.length < 3
|
||||
|| body[0] != 'M'
|
||||
|| body[1] != 'C') {
|
||||
return null;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
|
||||
* when this is an ordinary settings frame.
|
||||
|
||||
@@ -267,6 +267,21 @@ public class MessageBuilder {
|
||||
);
|
||||
}
|
||||
|
||||
/** CFW mic_control write (settings field 103) targeted at one temple. */
|
||||
public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) {
|
||||
return new OutboundMessage(
|
||||
"mic-control",
|
||||
"mic control " + label + (leftArm ? " L" : " R"),
|
||||
BleProtocol.SID_UI_SETTING,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
0,
|
||||
BleProtocol.buildFaceclawMicControl(record),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
leftArm
|
||||
);
|
||||
}
|
||||
|
||||
public OutboundMessage faceclawWearQuery(boolean leftArm) {
|
||||
return new OutboundMessage(
|
||||
"wear-query-control",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class FastClusteringConfig {
|
||||
private final int numClusters;
|
||||
private final float threshold;
|
||||
|
||||
private FastClusteringConfig(Builder builder) {
|
||||
this.numClusters = builder.numClusters;
|
||||
this.threshold = builder.threshold;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public int getNumClusters() {
|
||||
return numClusters;
|
||||
}
|
||||
|
||||
public float getThreshold() {
|
||||
return threshold;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private int numClusters = -1;
|
||||
private float threshold = 0.5f;
|
||||
|
||||
public FastClusteringConfig build() {
|
||||
return new FastClusteringConfig(this);
|
||||
}
|
||||
|
||||
public Builder setNumClusters(int numClusters) {
|
||||
this.numClusters = numClusters;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setThreshold(float threshold) {
|
||||
this.threshold = threshold;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarization {
|
||||
private long ptr = 0;
|
||||
|
||||
public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization");
|
||||
}
|
||||
}
|
||||
|
||||
public int getSampleRate() {
|
||||
return getSampleRate(ptr);
|
||||
}
|
||||
|
||||
// Only config.clustering is used. All other fields are ignored
|
||||
public void setConfig(OfflineSpeakerDiarizationConfig config) {
|
||||
setConfig(ptr, config);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] process(float[] samples) {
|
||||
return process(ptr, samples);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) {
|
||||
return processWithCallback(ptr, samples, callback, 0);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) {
|
||||
return processWithCallback(ptr, samples, callback, arg);
|
||||
}
|
||||
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
// You'd better call it manually if it is not used anymore
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
private native int getSampleRate(long ptr);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long newFromFile(OfflineSpeakerDiarizationConfig config);
|
||||
|
||||
private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config);
|
||||
|
||||
private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples);
|
||||
|
||||
private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg);
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface OfflineSpeakerDiarizationCallback {
|
||||
Integer invoke(int numProcessedChunks, int numTotalCunks, long arg);
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarizationConfig {
|
||||
private final OfflineSpeakerSegmentationModelConfig segmentation;
|
||||
private final SpeakerEmbeddingExtractorConfig embedding;
|
||||
private final FastClusteringConfig clustering;
|
||||
private final float minDurationOn;
|
||||
private final float minDurationOff;
|
||||
|
||||
private OfflineSpeakerDiarizationConfig(Builder builder) {
|
||||
this.segmentation = builder.segmentation;
|
||||
this.embedding = builder.embedding;
|
||||
this.clustering = builder.clustering;
|
||||
this.minDurationOff = builder.minDurationOff;
|
||||
this.minDurationOn = builder.minDurationOn;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public OfflineSpeakerSegmentationModelConfig getSegmentation() {
|
||||
return segmentation;
|
||||
}
|
||||
|
||||
public SpeakerEmbeddingExtractorConfig getEmbedding() {
|
||||
return embedding;
|
||||
}
|
||||
|
||||
public FastClusteringConfig getClustering() {
|
||||
return clustering;
|
||||
}
|
||||
|
||||
public float getMinDurationOff() {
|
||||
return minDurationOff;
|
||||
}
|
||||
|
||||
public float getMinDurationOn() {
|
||||
return minDurationOn;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build();
|
||||
private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build();
|
||||
private FastClusteringConfig clustering = FastClusteringConfig.builder().build();
|
||||
private float minDurationOn = 0.2f;
|
||||
private float minDurationOff = 0.5f;
|
||||
|
||||
public OfflineSpeakerDiarizationConfig build() {
|
||||
return new OfflineSpeakerDiarizationConfig(this);
|
||||
}
|
||||
|
||||
public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) {
|
||||
this.segmentation = segmentation;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) {
|
||||
this.embedding = embedding;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setClustering(FastClusteringConfig clustering) {
|
||||
this.clustering = clustering;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setMinDurationOff(float minDurationOff) {
|
||||
this.minDurationOff = minDurationOff;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setMinDurationOn(float minDurationOn) {
|
||||
this.minDurationOn = minDurationOn;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarizationSegment {
|
||||
private final float start;
|
||||
private final float end;
|
||||
private final int speaker;
|
||||
|
||||
public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.speaker = speaker;
|
||||
}
|
||||
|
||||
public float getStart() {
|
||||
return start;
|
||||
}
|
||||
|
||||
public float getEnd() {
|
||||
return end;
|
||||
}
|
||||
|
||||
public int getSpeaker() {
|
||||
return speaker;
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerSegmentationModelConfig {
|
||||
private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
|
||||
private OfflineSpeakerSegmentationModelConfig(Builder builder) {
|
||||
this.pyannote = builder.pyannote;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build();
|
||||
private int numThreads = 1;
|
||||
private boolean debug = true;
|
||||
private String provider = "cpu";
|
||||
|
||||
public OfflineSpeakerSegmentationModelConfig build() {
|
||||
return new OfflineSpeakerSegmentationModelConfig(this);
|
||||
}
|
||||
|
||||
public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) {
|
||||
this.pyannote = pyannote;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDebug(boolean debug) {
|
||||
this.debug = debug;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerSegmentationPyannoteModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public OfflineSpeakerSegmentationPyannoteModelConfig build() {
|
||||
return new OfflineSpeakerSegmentationPyannoteModelConfig(this);
|
||||
}
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright 2022-2023 by zhaoming
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineStream {
|
||||
private long ptr = 0;
|
||||
|
||||
public OnlineStream() {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public OnlineStream(long ptr) {
|
||||
this.ptr = ptr;
|
||||
}
|
||||
|
||||
public long getPtr() {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
public void setPtr(long ptr) {
|
||||
this.ptr = ptr;
|
||||
}
|
||||
|
||||
public void acceptWaveform(float[] samples, int sampleRate) {
|
||||
acceptWaveform(this.ptr, samples, sampleRate);
|
||||
}
|
||||
|
||||
public void inputFinished() {
|
||||
inputFinished(this.ptr);
|
||||
}
|
||||
|
||||
public void setOption(String key, String value) {
|
||||
setOption(this.ptr, key, value);
|
||||
}
|
||||
|
||||
public String getOption(String key) {
|
||||
return getOption(this.ptr, key);
|
||||
}
|
||||
|
||||
public boolean hasOption(String key) {
|
||||
return hasOption(this.ptr, key);
|
||||
}
|
||||
|
||||
public void release() {
|
||||
close();
|
||||
}
|
||||
|
||||
public void close() {
|
||||
// stream object must be release after used
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
close();
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
|
||||
|
||||
private native void inputFinished(long ptr);
|
||||
|
||||
private native void setOption(long ptr, String key, String value);
|
||||
|
||||
private native String getOption(long ptr, String key);
|
||||
|
||||
private native boolean hasOption(long ptr, String key);
|
||||
|
||||
private native void delete(long ptr);
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingExtractor {
|
||||
private long ptr = 0;
|
||||
|
||||
public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public OnlineStream createStream() {
|
||||
long p = createStream(ptr);
|
||||
return new OnlineStream(p);
|
||||
}
|
||||
|
||||
public boolean isReady(OnlineStream s) {
|
||||
return isReady(ptr, s.getPtr());
|
||||
}
|
||||
|
||||
public float[] compute(OnlineStream s) {
|
||||
return compute(ptr, s.getPtr());
|
||||
}
|
||||
|
||||
public int getDim() {
|
||||
return dim(ptr);
|
||||
}
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long newFromFile(SpeakerEmbeddingExtractorConfig config);
|
||||
|
||||
private native long createStream(long ptr);
|
||||
|
||||
private native boolean isReady(long ptr, long streamPtr);
|
||||
|
||||
private native float[] compute(long ptr, long streamPtr);
|
||||
|
||||
private native int dim(long ptr);
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingExtractorConfig {
|
||||
private final String model;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
|
||||
private SpeakerEmbeddingExtractorConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
private int numThreads = 1;
|
||||
private boolean debug = true;
|
||||
private String provider = "cpu";
|
||||
|
||||
public SpeakerEmbeddingExtractorConfig build() {
|
||||
return new SpeakerEmbeddingExtractorConfig(this);
|
||||
}
|
||||
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDebug(boolean debug) {
|
||||
this.debug = debug;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingManager {
|
||||
private long ptr = 0;
|
||||
|
||||
public SpeakerEmbeddingManager(int dim) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = create(dim);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public boolean add(String name, float[] embedding) {
|
||||
return add(ptr, name, embedding);
|
||||
}
|
||||
|
||||
public boolean add(String name, float[][] embedding) {
|
||||
return addList(ptr, name, embedding);
|
||||
}
|
||||
|
||||
public boolean remove(String name) {
|
||||
return remove(ptr, name);
|
||||
}
|
||||
|
||||
public String search(float[] embedding, float threshold) {
|
||||
return search(ptr, embedding, threshold);
|
||||
}
|
||||
|
||||
public boolean verify(String name, float[] embedding, float threshold) {
|
||||
return verify(ptr, name, embedding, threshold);
|
||||
}
|
||||
|
||||
public boolean contains(String name) {
|
||||
return contains(ptr, name);
|
||||
}
|
||||
|
||||
public int getNumSpeakers() {
|
||||
return numSpeakers(ptr);
|
||||
}
|
||||
|
||||
public String[] getAllSpeakerNames() {
|
||||
return allSpeakerNames(ptr);
|
||||
}
|
||||
|
||||
private native long create(int dim);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native boolean add(long ptr, String name, float[] embedding);
|
||||
|
||||
private native boolean addList(long ptr, String name, float[][] embedding);
|
||||
|
||||
private native boolean remove(long ptr, String name);
|
||||
|
||||
private native String search(long ptr, float[] embedding, float threshold);
|
||||
|
||||
private native boolean verify(long ptr, String name, float[] embedding, float threshold);
|
||||
|
||||
private native boolean contains(long ptr, String name);
|
||||
|
||||
private native int numSpeakers(long ptr);
|
||||
|
||||
private native String[] allSpeakerNames(long ptr);
|
||||
}
|
||||
Reference in new issue
Block a user