Add Transcribe screen

This commit is contained in:
James Babcock committed 2026-05-01 15:00:53 -07:00
1 parent 811477811e
commit bcc30545ad
16 files changed
+904 -20

No files matched your search

@@ -12,6 +12,9 @@ import com.k2fsa.sherpa.onnx.KeywordSpotter;
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
import com.k2fsa.sherpa.onnx.OnlineRecognizer;
import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OnlineRecognizerResult;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
@@ -31,6 +34,8 @@ public class FaceclawVoiceController {
private static final int STATS_INTERVAL_MS = 5_000;
private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr";
private static final String ASR_MODEL_DIR = "sherpa-onnx-streaming-zipformer-en-2023-06-26";
private static final String[] MODEL_FILES = {
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
@@ -38,6 +43,17 @@ public class FaceclawVoiceController {
"tokens.txt",
"screen-on-keywords.txt"
};
private static final String[] ASR_MODEL_FILES = {
"encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
"decoder-epoch-99-avg-1-chunk-16-left-128.onnx",
"joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
"tokens.txt"
};
private enum VoiceInputMode {
WAKEWORD,
FULL
}
private final Context appContext;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
@@ -48,9 +64,12 @@ public class FaceclawVoiceController {
private volatile FaceclawBleCommunicator communicator;
private Thread workerThread;
private volatile boolean started;
private VoiceInputMode mode = VoiceInputMode.WAKEWORD;
private KeywordSpotter keywordSpotter;
private OnlineRecognizer recognizer;
private OnlineStream stream;
private FaceclawLc3Decoder lc3Decoder;
private String lastTranscript = "";
private long queuedPackets;
private long queueDroppedPackets;
private long decodedSamples;
@@ -73,6 +92,10 @@ public class FaceclawVoiceController {
}
public void start() {
start("wakeword");
}
public void start(String requestedMode) {
synchronized (lock) {
if (started) {
emitStatus("Voice control is already listening.");
@@ -82,6 +105,7 @@ public class FaceclawVoiceController {
emitStatus("Voice control needs an active G2 connection.");
return;
}
mode = parseMode(requestedMode);
started = true;
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
workerThread.start();
@@ -103,6 +127,13 @@ public class FaceclawVoiceController {
}
if (threadToJoin != null) {
threadToJoin.interrupt();
if (Thread.currentThread() != threadToJoin) {
try {
threadToJoin.join(1500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
@@ -110,19 +141,35 @@ public class FaceclawVoiceController {
stop();
}
private VoiceInputMode parseMode(String requestedMode) {
return "full".equals(requestedMode) ? VoiceInputMode.FULL : VoiceInputMode.WAKEWORD;
}
private void runLoop() {
try {
emitStatus("Voice control loading wake-word model...");
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
VoiceInputMode currentMode = mode;
emitStatus(currentMode == VoiceInputMode.FULL
? "Voice control loading transcription model..."
: "Voice control loading wake-word model...");
if (currentMode == VoiceInputMode.FULL) {
File modelDir = installAsrModelFiles();
recognizer = new OnlineRecognizer(buildRecognizerConfig(modelDir));
stream = recognizer.createStream();
lastTranscript = "";
} else {
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
}
lc3Decoder = new FaceclawLc3Decoder();
if (!startG2Audio()) {
emitStatus("Voice control could not start G2 microphone input.");
return;
}
emitStatus("Voice control listening for \"screen on\" from the G2 mic.");
emitStatus(currentMode == VoiceInputMode.FULL
? "Voice control transcribing from the G2 mic."
: "Voice control listening for \"screen on\" from the G2 mic.");
processG2Audio();
} catch (Throwable error) {
Log.e(TAG, "Voice control failed", error);
@@ -152,6 +199,7 @@ public class FaceclawVoiceController {
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setModelType("zipformer2")
.setModelingUnit("")
.setNumThreads(1)
.build())
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
@@ -160,6 +208,26 @@ public class FaceclawVoiceController {
.build();
}
private OnlineRecognizerConfig buildRecognizerConfig(File modelDir) {
return OnlineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setOnlineModelConfig(OnlineModelConfig.builder()
.setTransducer(OnlineTransducerModelConfig.builder()
.setEncoder(new File(modelDir, "encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath())
.setDecoder(new File(modelDir, "decoder-epoch-99-avg-1-chunk-16-left-128.onnx").getAbsolutePath())
.setJoiner(new File(modelDir, "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setModelType("zipformer2")
.setNumThreads(1)
.build())
.setEnableEndpoint(true)
.build();
}
private File installModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
@@ -176,6 +244,22 @@ public class FaceclawVoiceController {
return modelDir;
}
private File installAsrModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
throw new IOException("Could not create " + modelDir.getAbsolutePath());
}
AssetManager assets = appContext.getAssets();
for (String fileName : ASR_MODEL_FILES) {
copyAssetIfNeeded(
assets,
ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName,
new File(modelDir, fileName)
);
}
return modelDir;
}
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
if (destination.exists() && destination.length() > 0) {
return;
@@ -206,9 +290,8 @@ public class FaceclawVoiceController {
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
while (started && !Thread.currentThread().isInterrupted()) {
OnlineStream currentStream = stream;
KeywordSpotter currentSpotter = keywordSpotter;
FaceclawLc3Decoder currentDecoder = lc3Decoder;
if (currentStream == null || currentSpotter == null || currentDecoder == null) {
if (currentStream == null || currentDecoder == null) {
return;
}
@@ -229,19 +312,73 @@ public class FaceclawVoiceController {
}
currentStream.acceptWaveform(samples, SAMPLE_RATE);
while (currentSpotter.isReady(currentStream)) {
currentSpotter.decode(currentStream);
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
String keyword = result == null ? "" : result.getKeyword();
if (keyword != null && keyword.trim().length() > 0) {
currentSpotter.reset(currentStream);
emitWakeWord(keyword);
}
if (mode == VoiceInputMode.FULL) {
processRecognizer(currentStream);
} else {
processKeywordSpotter(currentStream);
}
maybeEmitAudioStats(false);
}
}
private void processKeywordSpotter(OnlineStream currentStream) {
KeywordSpotter currentSpotter = keywordSpotter;
if (currentSpotter == null) {
return;
}
while (currentSpotter.isReady(currentStream)) {
currentSpotter.decode(currentStream);
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
String keyword = result == null ? "" : result.getKeyword();
if (keyword != null && keyword.trim().length() > 0) {
currentSpotter.reset(currentStream);
emitWakeWord(keyword);
}
}
}
private void processRecognizer(OnlineStream currentStream) {
OnlineRecognizer currentRecognizer = recognizer;
if (currentRecognizer == null) {
return;
}
while (currentRecognizer.isReady(currentStream)) {
currentRecognizer.decode(currentStream);
}
emitRecognizerResult(currentRecognizer, currentStream, false);
if (currentRecognizer.isEndpoint(currentStream)) {
emitRecognizerResult(currentRecognizer, currentStream, true);
currentRecognizer.reset(currentStream);
lastTranscript = "";
}
}
private void emitRecognizerResult(OnlineRecognizer currentRecognizer, OnlineStream currentStream, boolean isFinal) {
OnlineRecognizerResult result = currentRecognizer.getResult(currentStream);
String text = result == null ? "" : result.getText();
if (text == null) {
text = "";
}
if (text.equals(lastTranscript)) {
if (isFinal && lastTranscript.length() > 0) {
emitTranscript("", true);
lastTranscript = "";
}
return;
}
String delta = text;
if (lastTranscript.length() > 0 && text.startsWith(lastTranscript)) {
delta = text.substring(lastTranscript.length());
}
if (delta.trim().length() > 0) {
emitTranscript(delta, false);
}
lastTranscript = isFinal ? "" : text;
if (isFinal) {
emitTranscript("", true);
}
}
private void stopG2Audio() {
FaceclawBleCommunicator currentCommunicator = communicator;
if (currentCommunicator != null) {
@@ -259,6 +396,10 @@ public class FaceclawVoiceController {
keywordSpotter.release();
keywordSpotter = null;
}
if (recognizer != null) {
recognizer.release();
recognizer = null;
}
}
private void releaseLc3() {
@@ -339,6 +480,7 @@ public class FaceclawVoiceController {
+ " duplicate=" + duplicate
+ " late=" + latePackets
+ " maxGapMs=" + maxInterPacketMs
+ "\n"
+ " queueDrop=" + queueDroppedPackets
+ " decodeErrors=" + decodeErrors
+ " wrongArm=" + wrongArmPackets
@@ -363,6 +505,14 @@ public class FaceclawVoiceController {
mainHandler.post(() -> currentListener.onWakeWord(keyword));
}
private void emitTranscript(String text, boolean isFinal) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onTranscript(text, isFinal));
}
private static final class AudioPacket {
final byte[] data;
final String arm;
@@ -4,4 +4,6 @@ public interface FaceclawVoiceControllerListener {
void onStatus(String status);
void onWakeWord(String keyword);
void onTranscript(String text, boolean isFinal);
}
@@ -0,0 +1,66 @@
package com.k2fsa.sherpa.onnx;
public class EndpointConfig {
private final EndpointRule rule1;
private final EndpointRule rule2;
private final EndpointRule rule3;
private EndpointConfig(Builder builder) {
this.rule1 = builder.rule1;
this.rule2 = builder.rule2;
this.rule3 = builder.rule3;
}
public static Builder builder() {
return new Builder();
}
public EndpointRule getRule1() {
return rule1;
}
public EndpointRule getRule2() {
return rule2;
}
public EndpointRule getRule3() {
return rule3;
}
public static class Builder {
private EndpointRule rule1 = EndpointRule.builder()
.setMustContainNonSilence(false)
.setMinTrailingSilence(2.4f)
.setMinUtteranceLength(0.0f)
.build();
private EndpointRule rule2 = EndpointRule.builder()
.setMustContainNonSilence(true)
.setMinTrailingSilence(1.2f)
.setMinUtteranceLength(0.0f)
.build();
private EndpointRule rule3 = EndpointRule.builder()
.setMustContainNonSilence(false)
.setMinTrailingSilence(0.0f)
.setMinUtteranceLength(20.0f)
.build();
public Builder setRule1(EndpointRule rule1) {
this.rule1 = rule1;
return this;
}
public Builder setRule2(EndpointRule rule2) {
this.rule2 = rule2;
return this;
}
public Builder setRule3(EndpointRule rule3) {
this.rule3 = rule3;
return this;
}
public EndpointConfig build() {
return new EndpointConfig(this);
}
}
}
@@ -0,0 +1,54 @@
package com.k2fsa.sherpa.onnx;
public class EndpointRule {
private final boolean mustContainNonSilence;
private final float minTrailingSilence;
private final float minUtteranceLength;
private EndpointRule(Builder builder) {
this.mustContainNonSilence = builder.mustContainNonSilence;
this.minTrailingSilence = builder.minTrailingSilence;
this.minUtteranceLength = builder.minUtteranceLength;
}
public static Builder builder() {
return new Builder();
}
public boolean getMustContainNonSilence() {
return mustContainNonSilence;
}
public float getMinTrailingSilence() {
return minTrailingSilence;
}
public float getMinUtteranceLength() {
return minUtteranceLength;
}
public static class Builder {
private boolean mustContainNonSilence = false;
private float minTrailingSilence = 0.0f;
private float minUtteranceLength = 0.0f;
public Builder setMustContainNonSilence(boolean mustContainNonSilence) {
this.mustContainNonSilence = mustContainNonSilence;
return this;
}
public Builder setMinTrailingSilence(float minTrailingSilence) {
this.minTrailingSilence = minTrailingSilence;
return this;
}
public Builder setMinUtteranceLength(float minUtteranceLength) {
this.minUtteranceLength = minUtteranceLength;
return this;
}
public EndpointRule build() {
return new EndpointRule(this);
}
}
}
@@ -0,0 +1,54 @@
package com.k2fsa.sherpa.onnx;
public class HomophoneReplacerConfig {
private final String dictDir;
private final String lexicon;
private final String ruleFsts;
private HomophoneReplacerConfig(Builder builder) {
this.dictDir = builder.dictDir;
this.lexicon = builder.lexicon;
this.ruleFsts = builder.ruleFsts;
}
public static Builder builder() {
return new Builder();
}
public String getDictDir() {
return dictDir;
}
public String getLexicon() {
return lexicon;
}
public String getRuleFsts() {
return ruleFsts;
}
public static class Builder {
private String dictDir = "";
private String lexicon = "";
private String ruleFsts = "";
public Builder setDictDir(String dictDir) {
this.dictDir = dictDir;
return this;
}
public Builder setLexicon(String lexicon) {
this.lexicon = lexicon;
return this;
}
public Builder setRuleFsts(String ruleFsts) {
this.ruleFsts = ruleFsts;
return this;
}
public HomophoneReplacerConfig build() {
return new HomophoneReplacerConfig(this);
}
}
}
@@ -0,0 +1,42 @@
package com.k2fsa.sherpa.onnx;
public class OnlineCtcFstDecoderConfig {
private final String graph;
private final int maxActive;
private OnlineCtcFstDecoderConfig(Builder builder) {
this.graph = builder.graph;
this.maxActive = builder.maxActive;
}
public static Builder builder() {
return new Builder();
}
public String getGraph() {
return graph;
}
public int getMaxActive() {
return maxActive;
}
public static class Builder {
private String graph = "";
private int maxActive = 3000;
public Builder setGraph(String graph) {
this.graph = graph;
return this;
}
public Builder setMaxActive(int maxActive) {
this.maxActive = maxActive;
return this;
}
public OnlineCtcFstDecoderConfig build() {
return new OnlineCtcFstDecoderConfig(this);
}
}
}
@@ -0,0 +1,42 @@
package com.k2fsa.sherpa.onnx;
public class OnlineLMConfig {
private final String model;
private final float scale;
private OnlineLMConfig(Builder builder) {
this.model = builder.model;
this.scale = builder.scale;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public float getScale() {
return scale;
}
public static class Builder {
private String model = "";
private float scale = 0.5f;
public Builder setModel(String model) {
this.model = model;
return this;
}
public Builder setScale(float scale) {
this.scale = scale;
return this;
}
public OnlineLMConfig build() {
return new OnlineLMConfig(this);
}
}
}
@@ -0,0 +1,66 @@
package com.k2fsa.sherpa.onnx;
public class OnlineRecognizer {
private long ptr;
public OnlineRecognizer(OnlineRecognizerConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid OnlineRecognizerConfig");
}
}
public OnlineStream createStream() {
return createStream("");
}
public OnlineStream createStream(String hotwords) {
long streamPtr = createStream(ptr, hotwords == null ? "" : hotwords);
return new OnlineStream(streamPtr);
}
public boolean isReady(OnlineStream stream) {
return stream != null && isReady(ptr, stream.getPtr());
}
public void decode(OnlineStream stream) {
if (stream != null) {
decode(ptr, stream.getPtr());
}
}
public void reset(OnlineStream stream) {
if (stream != null) {
reset(ptr, stream.getPtr());
}
}
public boolean isEndpoint(OnlineStream stream) {
return stream != null && isEndpoint(ptr, stream.getPtr());
}
public OnlineRecognizerResult getResult(OnlineStream stream) {
if (stream == null) {
return new OnlineRecognizerResult("", new String[0], new float[0], new float[0]);
}
return getResult(ptr, stream.getPtr());
}
public void release() {
if (ptr != 0) {
delete(ptr);
ptr = 0;
}
}
private native long newFromFile(OnlineRecognizerConfig config);
private native void delete(long ptr);
private native long createStream(long ptr, String hotwords);
private native void reset(long ptr, long streamPtr);
private native void decode(long ptr, long streamPtr);
private native boolean isEndpoint(long ptr, long streamPtr);
private native boolean isReady(long ptr, long streamPtr);
private native OnlineRecognizerResult getResult(long ptr, long streamPtr);
}
@@ -0,0 +1,146 @@
package com.k2fsa.sherpa.onnx;
public class OnlineRecognizerConfig {
private final FeatureConfig featConfig;
private final OnlineModelConfig modelConfig;
private final OnlineLMConfig lmConfig;
private final OnlineCtcFstDecoderConfig ctcFstDecoderConfig;
private final HomophoneReplacerConfig hr;
private final EndpointConfig endpointConfig;
private final boolean enableEndpoint;
private final String decodingMethod;
private final int maxActivePaths;
private final String hotwordsFile;
private final float hotwordsScore;
private final String ruleFsts;
private final String ruleFars;
private final float blankPenalty;
private OnlineRecognizerConfig(Builder builder) {
this.featConfig = builder.featConfig;
this.modelConfig = builder.modelConfig;
this.lmConfig = builder.lmConfig;
this.ctcFstDecoderConfig = builder.ctcFstDecoderConfig;
this.hr = builder.hr;
this.endpointConfig = builder.endpointConfig;
this.enableEndpoint = builder.enableEndpoint;
this.decodingMethod = builder.decodingMethod;
this.maxActivePaths = builder.maxActivePaths;
this.hotwordsFile = builder.hotwordsFile;
this.hotwordsScore = builder.hotwordsScore;
this.ruleFsts = builder.ruleFsts;
this.ruleFars = builder.ruleFars;
this.blankPenalty = builder.blankPenalty;
}
public static Builder builder() {
return new Builder();
}
public FeatureConfig getFeatConfig() {
return featConfig;
}
public OnlineModelConfig getModelConfig() {
return modelConfig;
}
public OnlineLMConfig getLmConfig() {
return lmConfig;
}
public OnlineCtcFstDecoderConfig getCtcFstDecoderConfig() {
return ctcFstDecoderConfig;
}
public HomophoneReplacerConfig getHr() {
return hr;
}
public EndpointConfig getEndpointConfig() {
return endpointConfig;
}
public boolean getEnableEndpoint() {
return enableEndpoint;
}
public String getDecodingMethod() {
return decodingMethod;
}
public int getMaxActivePaths() {
return maxActivePaths;
}
public String getHotwordsFile() {
return hotwordsFile;
}
public float getHotwordsScore() {
return hotwordsScore;
}
public String getRuleFsts() {
return ruleFsts;
}
public String getRuleFars() {
return ruleFars;
}
public float getBlankPenalty() {
return blankPenalty;
}
public static class Builder {
private FeatureConfig featConfig = FeatureConfig.builder().build();
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
private OnlineLMConfig lmConfig = OnlineLMConfig.builder().build();
private OnlineCtcFstDecoderConfig ctcFstDecoderConfig = OnlineCtcFstDecoderConfig.builder().build();
private HomophoneReplacerConfig hr = HomophoneReplacerConfig.builder().build();
private EndpointConfig endpointConfig = EndpointConfig.builder().build();
private boolean enableEndpoint = true;
private String decodingMethod = "greedy_search";
private int maxActivePaths = 4;
private String hotwordsFile = "";
private float hotwordsScore = 1.5f;
private String ruleFsts = "";
private String ruleFars = "";
private float blankPenalty = 0.0f;
public Builder setFeatureConfig(FeatureConfig featConfig) {
this.featConfig = featConfig;
return this;
}
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
this.modelConfig = modelConfig;
return this;
}
public Builder setEndpointConfig(EndpointConfig endpointConfig) {
this.endpointConfig = endpointConfig;
return this;
}
public Builder setEnableEndpoint(boolean enableEndpoint) {
this.enableEndpoint = enableEndpoint;
return this;
}
public Builder setDecodingMethod(String decodingMethod) {
this.decodingMethod = decodingMethod;
return this;
}
public Builder setMaxActivePaths(int maxActivePaths) {
this.maxActivePaths = maxActivePaths;
return this;
}
public OnlineRecognizerConfig build() {
return new OnlineRecognizerConfig(this);
}
}
}
@@ -0,0 +1,31 @@
package com.k2fsa.sherpa.onnx;
public class OnlineRecognizerResult {
private final String text;
private final String[] tokens;
private final float[] timestamps;
private final float[] ysProbs;
public OnlineRecognizerResult(String text, String[] tokens, float[] timestamps, float[] ysProbs) {
this.text = text == null ? "" : text;
this.tokens = tokens == null ? new String[0] : tokens;
this.timestamps = timestamps == null ? new float[0] : timestamps;
this.ysProbs = ysProbs == null ? new float[0] : ysProbs;
}
public String getText() {
return text;
}
public String[] getTokens() {
return tokens;
}
public float[] getTimestamps() {
return timestamps;
}
public float[] getYsProbs() {
return ysProbs;
}
}