Merge branch 'main' into bluetooth-pairing-improvements

This commit is contained in:
James Babcock committed 2026-08-23 13:39:50 -07:00
commit ee022cdc84
41 files changed
+482 -1329

No files matched your search

@@ -1,30 +1,20 @@
package com.faceclaw.app;
import android.content.Context;
import android.content.res.AssetManager;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotter;
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
import com.k2fsa.sherpa.onnx.OfflineStream;
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.Arrays;
@@ -57,17 +47,15 @@ public class FaceclawVoiceController {
private static final float TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f;
private static final float TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f;
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr";
// Model directory shared with the TS-side download flow (asr-model.ts),
// which fetches the Moonshine files here on demand; they are no longer
// bundled in the APK.
private static final String ASR_ROOT = "faceclaw-voice-asr";
private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27";
private static final String[] MODEL_FILES = {
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
"tokens.txt",
"screen-on-keywords.txt"
};
// Model files for the retired on-phone wake-word spotter, copied to
// filesDir by earlier releases; deleted on sight to reclaim the space.
// (The wakeword is now detected by the glasses firmware itself.)
private static final String LEGACY_KWS_ROOT = "faceclaw-voice";
private static final String[] ASR_MODEL_FILES = {
"encoder_model.ort",
"decoder_model_merged.ort",
@@ -75,7 +63,6 @@ public class FaceclawVoiceController {
};
private enum VoiceInputMode {
WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired)
ONBOARD, // on-phone Moonshine transcription
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
}
@@ -89,10 +76,8 @@ public class FaceclawVoiceController {
private volatile FaceclawBleCommunicator communicator;
private Thread workerThread;
private volatile boolean started;
private VoiceInputMode mode = VoiceInputMode.WAKEWORD;
private KeywordSpotter keywordSpotter;
private VoiceInputMode mode = VoiceInputMode.CLOUD;
private OfflineRecognizer recognizer;
private OnlineStream stream;
private FaceclawLc3Decoder lc3Decoder;
private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
private int transcriptSampleCount;
@@ -140,10 +125,6 @@ public class FaceclawVoiceController {
this.endpointing = endpointing;
}
public void start() {
start("wakeword");
}
public void start(String requestedMode) {
synchronized (lock) {
if (started) {
@@ -194,26 +175,23 @@ public class FaceclawVoiceController {
if ("cloud".equals(requestedMode)) {
return VoiceInputMode.CLOUD;
}
if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) {
return VoiceInputMode.ONBOARD;
}
return VoiceInputMode.WAKEWORD;
return VoiceInputMode.ONBOARD;
}
private void runLoop() {
try {
deleteLegacyKwsFiles();
VoiceInputMode currentMode = mode;
if (currentMode == VoiceInputMode.ONBOARD) {
File modelDir = findAsrModelDir();
if (modelDir == null) {
emitStatus("Voice model not downloaded (see Settings > Voice).");
return;
}
emitStatus("Loading transcription model...");
File modelDir = installAsrModelFiles();
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
resetTranscriptState();
lastTranscript = "";
} else if (currentMode == VoiceInputMode.WAKEWORD) {
emitStatus("Loading wake-word model...");
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
}
lc3Decoder = new FaceclawLc3Decoder();
endpointDetector.reset();
@@ -225,9 +203,7 @@ public class FaceclawVoiceController {
emitStatus(currentMode == VoiceInputMode.CLOUD
? "Listening (cloud)..."
: currentMode == VoiceInputMode.ONBOARD
? "Listening..."
: "Listening for \"screen on\"...");
: "Listening...");
processG2Audio();
// Button released / stop requested: emit one final full-utterance
// transcript so the UI can freeze it.
@@ -312,29 +288,6 @@ public class FaceclawVoiceController {
return b.array();
}
private KeywordSpotterConfig buildConfig(File modelDir) {
return KeywordSpotterConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setOnlineModelConfig(OnlineModelConfig.builder()
.setTransducer(OnlineTransducerModelConfig.builder()
.setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setModelType("zipformer2")
.setModelingUnit("")
.setNumThreads(1)
.build())
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
.setKeywordsScore(1.5f)
.setKeywordsThreshold(0.35f)
.build();
}
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) {
return OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
@@ -352,50 +305,42 @@ public class FaceclawVoiceController {
.build();
}
private File installModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
throw new IOException("Could not create " + modelDir.getAbsolutePath());
}
AssetManager assets = appContext.getAssets();
for (String fileName : MODEL_FILES) {
copyAssetIfNeeded(
assets,
ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName,
new File(modelDir, fileName)
);
}
return modelDir;
}
private File installAsrModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
throw new IOException("Could not create " + modelDir.getAbsolutePath());
}
AssetManager assets = appContext.getAssets();
/**
* The Moonshine model directory, populated by the download flow in
* asr-model.ts (releases before 0.5.0 copied the same files out of the
* APK, so upgraded installs are already complete). Null when any file is
* missing, i.e. the model still needs to be downloaded.
*/
private File findAsrModelDir() {
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR);
for (String fileName : ASR_MODEL_FILES) {
copyAssetIfNeeded(
assets,
ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName,
new File(modelDir, fileName)
);
}
return modelDir;
}
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
if (destination.exists() && destination.length() > 0) {
return;
}
try (InputStream input = assets.open(assetPath);
FileOutputStream output = new FileOutputStream(destination)) {
byte[] buffer = new byte[64 * 1024];
int read;
while ((read = input.read(buffer)) >= 0) {
output.write(buffer, 0, read);
File file = new File(modelDir, fileName);
if (!file.exists() || file.length() == 0) {
return null;
}
}
return modelDir;
}
private void deleteLegacyKwsFiles() {
try {
deleteRecursively(new File(appContext.getFilesDir(), LEGACY_KWS_ROOT));
} catch (Throwable t) {
Log.w(TAG, "failed to delete legacy wake-word files", t);
}
}
private static void deleteRecursively(File file) {
if (!file.exists()) {
return;
}
File[] children = file.listFiles();
if (children != null) {
for (File child : children) {
deleteRecursively(child);
}
}
file.delete();
}
private boolean startG2Audio() {
@@ -413,9 +358,8 @@ public class FaceclawVoiceController {
private void processG2Audio() {
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
while (started && !Thread.currentThread().isInterrupted()) {
OnlineStream currentStream = stream;
FaceclawLc3Decoder currentDecoder = lc3Decoder;
if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) {
if (currentDecoder == null) {
return;
}
@@ -438,40 +382,17 @@ public class FaceclawVoiceController {
}
if (mode == VoiceInputMode.CLOUD) {
emitPcm(pcm, count);
} else if (mode == VoiceInputMode.ONBOARD) {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
processRecognizer(samples);
} else {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
currentStream.acceptWaveform(samples, SAMPLE_RATE);
processKeywordSpotter(currentStream);
processRecognizer(samples);
}
maybeEmitAudioStats(false);
}
}
private void processKeywordSpotter(OnlineStream currentStream) {
KeywordSpotter currentSpotter = keywordSpotter;
if (currentSpotter == null) {
return;
}
while (currentSpotter.isReady(currentStream)) {
currentSpotter.decode(currentStream);
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
String keyword = result == null ? "" : result.getKeyword();
if (keyword != null && keyword.trim().length() > 0) {
currentSpotter.reset(currentStream);
emitWakeWord(keyword);
}
}
}
private void processRecognizer(float[] samples) {
appendTranscriptSamples(samples);
long now = SystemClock.elapsedRealtime();
@@ -783,14 +704,6 @@ public class FaceclawVoiceController {
}
private void releaseSherpa() {
if (stream != null) {
stream.release();
stream = null;
}
if (keywordSpotter != null) {
keywordSpotter.release();
keywordSpotter = null;
}
if (recognizer != null) {
recognizer.release();
recognizer = null;
@@ -893,14 +806,6 @@ public class FaceclawVoiceController {
mainHandler.post(() -> currentListener.onStatus(status));
}
private void emitWakeWord(String keyword) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onWakeWord(keyword));
}
private void emitTranscript(String text, boolean isFinal) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
@@ -3,8 +3,6 @@ package com.faceclaw.app;
public interface FaceclawVoiceControllerListener {
void onStatus(String status);
void onWakeWord(String keyword);
/**
* Best transcript of the current utterance so far. REPLACE semantics: text
* is the complete transcript, not a delta — display it verbatim, replacing
@@ -1,66 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotter {
private long ptr;
public KeywordSpotter(KeywordSpotterConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid KeywordSpotterConfig");
}
}
public OnlineStream createStream() {
return createStream("");
}
public OnlineStream createStream(String keywords) {
return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords));
}
public boolean isReady(OnlineStream stream) {
return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr());
}
public void decode(OnlineStream stream) {
decode(ptr, stream.getPtr());
}
public void reset(OnlineStream stream) {
reset(ptr, stream.getPtr());
}
public KeywordSpotterResult getResult(OnlineStream stream) {
return getResult(ptr, stream.getPtr());
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
super.finalize();
}
private native long newFromFile(KeywordSpotterConfig config);
private native void delete(long ptr);
private native long createStream(long ptr, String keywords);
private native boolean isReady(long ptr, long streamPtr);
private native void decode(long ptr, long streamPtr);
private native void reset(long ptr, long streamPtr);
private native KeywordSpotterResult getResult(long ptr, long streamPtr);
}
@@ -1,92 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterConfig {
private final FeatureConfig featConfig;
private final OnlineModelConfig modelConfig;
private final int maxActivePaths;
private final String keywordsFile;
private final float keywordsScore;
private final float keywordsThreshold;
private final int numTrailingBlanks;
private KeywordSpotterConfig(Builder builder) {
this.featConfig = builder.featConfig;
this.modelConfig = builder.modelConfig;
this.maxActivePaths = builder.maxActivePaths;
this.keywordsFile = builder.keywordsFile;
this.keywordsScore = builder.keywordsScore;
this.keywordsThreshold = builder.keywordsThreshold;
this.numTrailingBlanks = builder.numTrailingBlanks;
}
public static Builder builder() {
return new Builder();
}
public FeatureConfig getFeatConfig() {
return featConfig;
}
public OnlineModelConfig getModelConfig() {
return modelConfig;
}
public int getMaxActivePaths() {
return maxActivePaths;
}
public String getKeywordsFile() {
return keywordsFile;
}
public float getKeywordsScore() {
return keywordsScore;
}
public float getKeywordsThreshold() {
return keywordsThreshold;
}
public int getNumTrailingBlanks() {
return numTrailingBlanks;
}
public static class Builder {
private FeatureConfig featConfig = FeatureConfig.builder().build();
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
private int maxActivePaths = 4;
private String keywordsFile = "keywords.txt";
private float keywordsScore = 1.5f;
private float keywordsThreshold = 0.25f;
private int numTrailingBlanks = 2;
public Builder setFeatureConfig(FeatureConfig featConfig) {
this.featConfig = featConfig;
return this;
}
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
this.modelConfig = modelConfig;
return this;
}
public Builder setKeywordsFile(String keywordsFile) {
this.keywordsFile = keywordsFile;
return this;
}
public Builder setKeywordsScore(float keywordsScore) {
this.keywordsScore = keywordsScore;
return this;
}
public Builder setKeywordsThreshold(float keywordsThreshold) {
this.keywordsThreshold = keywordsThreshold;
return this;
}
public KeywordSpotterConfig build() {
return new KeywordSpotterConfig(this);
}
}
}
@@ -1,25 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterResult {
private final String keyword;
private final String[] tokens;
private final float[] timestamps;
public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) {
this.keyword = keyword == null ? "" : keyword;
this.tokens = tokens == null ? new String[0] : tokens;
this.timestamps = timestamps == null ? new float[0] : timestamps;
}
public String getKeyword() {
return keyword;
}
public String[] getTokens() {
return tokens;
}
public float[] getTimestamps() {
return timestamps;
}
}
@@ -1,137 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineModelConfig {
private final OnlineTransducerModelConfig transducer;
private final OnlineParaformerModelConfig paraformer;
private final OnlineZipformer2CtcModelConfig zipformer2Ctc;
private final OnlineNeMoCtcModelConfig neMoCtc;
private final OnlineToneCtcModelConfig toneCtc;
private final String tokens;
private final int numThreads;
private final boolean debug;
private final String provider;
private final String modelType;
private final String modelingUnit;
private final String bpeVocab;
private OnlineModelConfig(Builder builder) {
this.transducer = builder.transducer;
this.paraformer = builder.paraformer;
this.zipformer2Ctc = builder.zipformer2Ctc;
this.neMoCtc = builder.neMoCtc;
this.toneCtc = builder.toneCtc;
this.tokens = builder.tokens;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
this.modelType = builder.modelType;
this.modelingUnit = builder.modelingUnit;
this.bpeVocab = builder.bpeVocab;
}
public static Builder builder() {
return new Builder();
}
public OnlineTransducerModelConfig getTransducer() {
return transducer;
}
public OnlineParaformerModelConfig getParaformer() {
return paraformer;
}
public OnlineZipformer2CtcModelConfig getZipformer2Ctc() {
return zipformer2Ctc;
}
public OnlineNeMoCtcModelConfig getNeMoCtc() {
return neMoCtc;
}
public OnlineToneCtcModelConfig getToneCtc() {
return toneCtc;
}
public String getTokens() {
return tokens;
}
public int getNumThreads() {
return numThreads;
}
public boolean getDebug() {
return debug;
}
public String getProvider() {
return provider;
}
public String getModelType() {
return modelType;
}
public String getModelingUnit() {
return modelingUnit;
}
public String getBpeVocab() {
return bpeVocab;
}
public static class Builder {
private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build();
private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build();
private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build();
private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build();
private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build();
private String tokens = "";
private int numThreads = 1;
private boolean debug = false;
private String provider = "cpu";
private String modelType = "";
private String modelingUnit = "cjkchar";
private String bpeVocab = "";
public Builder setTransducer(OnlineTransducerModelConfig transducer) {
this.transducer = transducer;
return this;
}
public Builder setTokens(String tokens) {
this.tokens = tokens;
return this;
}
public Builder setModelType(String modelType) {
this.modelType = modelType;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
public Builder setModelingUnit(String modelingUnit) {
this.modelingUnit = modelingUnit;
return this;
}
public Builder setBpeVocab(String bpeVocab) {
this.bpeVocab = bpeVocab;
return this;
}
public OnlineModelConfig build() {
return new OnlineModelConfig(this);
}
}
}
@@ -1,30 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineNeMoCtcModelConfig {
private final String model;
private OnlineNeMoCtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineNeMoCtcModelConfig build() {
return new OnlineNeMoCtcModelConfig(this);
}
}
}
@@ -1,42 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineParaformerModelConfig {
private final String encoder;
private final String decoder;
private OnlineParaformerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public OnlineParaformerModelConfig build() {
return new OnlineParaformerModelConfig(this);
}
}
}
@@ -1,35 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineStream {
private long ptr;
public OnlineStream(long ptr) {
this.ptr = ptr;
}
public long getPtr() {
return ptr;
}
public void acceptWaveform(float[] samples, int sampleRate) {
acceptWaveform(ptr, samples, sampleRate);
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
super.finalize();
}
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
private native void delete(long ptr);
}
@@ -1,30 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineToneCtcModelConfig {
private final String model;
private OnlineToneCtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineToneCtcModelConfig build() {
return new OnlineToneCtcModelConfig(this);
}
}
}
@@ -1,54 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineTransducerModelConfig {
private final String encoder;
private final String decoder;
private final String joiner;
private OnlineTransducerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
this.joiner = builder.joiner;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public String getJoiner() {
return joiner;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
private String joiner = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public Builder setJoiner(String joiner) {
this.joiner = joiner;
return this;
}
public OnlineTransducerModelConfig build() {
return new OnlineTransducerModelConfig(this);
}
}
}
@@ -1,30 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineZipformer2CtcModelConfig {
private final String model;
private OnlineZipformer2CtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineZipformer2CtcModelConfig build() {
return new OnlineZipformer2CtcModelConfig(this);
}
}
}