forked from Blendman974/faceclaw
Merge branch 'main' into ring-health-fixes
This commit is contained in:
commit
edef7138fd
29 files changed
+1089
-234
No files matched your search
+120
-22
@@ -13,6 +13,7 @@ import com.k2fsa.sherpa.onnx.OfflineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineStream;
|
||||
import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -61,12 +62,43 @@ public class FaceclawVoiceController {
|
||||
"decoder_model_merged.ort",
|
||||
"tokens.txt"
|
||||
};
|
||||
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
|
||||
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
|
||||
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
|
||||
//
|
||||
// On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0)
|
||||
// runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of
|
||||
// zero tail padding, capped at 30s. Re-decoding on Moonshine's
|
||||
// TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that
|
||||
// encode, growing with the utterance, roughly 1.4x/second while the user is
|
||||
// still speaking, so onboardModelKind gates that loop off for Whisper --
|
||||
// see processRecognizer(). Whisper is also known to hallucinate text on
|
||||
// near-silent input; recognizeTranscriptSegment() gates that too.
|
||||
private static final String ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8";
|
||||
private static final String[] ASR_WHISPER_MODEL_FILES = {
|
||||
"base.en-encoder.int8.onnx",
|
||||
"base.en-decoder.int8.onnx",
|
||||
"base.en-tokens.txt"
|
||||
};
|
||||
// Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f
|
||||
// buffer) a segment is treated as silence and never reaches the Whisper
|
||||
// recognizer at all, rather than risking a hallucinated non-answer. Picked
|
||||
// conservatively low (well under typical mic noise floor already seen in
|
||||
// this pipeline's normalization target) -- UNTESTED on real hardware, tune
|
||||
// against real glasses captures rather than trusting this number.
|
||||
private static final float WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f;
|
||||
|
||||
private enum VoiceInputMode {
|
||||
ONBOARD, // on-phone Moonshine transcription
|
||||
ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind)
|
||||
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
|
||||
}
|
||||
|
||||
/** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */
|
||||
private enum OnboardModelKind {
|
||||
MOONSHINE,
|
||||
WHISPER
|
||||
}
|
||||
|
||||
private final Context appContext;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private final Object lock = new Object();
|
||||
@@ -86,6 +118,7 @@ public class FaceclawVoiceController {
|
||||
private volatile boolean usePhoneMic;
|
||||
private boolean activePhoneMic;
|
||||
private VoiceInputMode mode = VoiceInputMode.CLOUD;
|
||||
private volatile OnboardModelKind onboardModelKind = OnboardModelKind.MOONSHINE;
|
||||
private OfflineRecognizer recognizer;
|
||||
private FaceclawLc3Decoder lc3Decoder;
|
||||
private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
|
||||
@@ -150,6 +183,17 @@ public class FaceclawVoiceController {
|
||||
this.saveRecordings = saveRecordings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which on-device model {@link #start}("onboard") should load: "whisper"
|
||||
* selects the second on-device model (sherpa-onnx offline Whisper); any
|
||||
* other value (including null/absent) keeps the existing Moonshine model,
|
||||
* so callers that never call this see unchanged behavior. Must be set
|
||||
* before {@link #start}; has no effect in CLOUD mode.
|
||||
*/
|
||||
public void setOnboardModelKind(String kind) {
|
||||
this.onboardModelKind = "whisper".equals(kind) ? OnboardModelKind.WHISPER : OnboardModelKind.MOONSHINE;
|
||||
}
|
||||
|
||||
/**
|
||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||
@@ -222,20 +266,26 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
|
||||
public void start(String requestedMode) {
|
||||
start(requestedMode, 0);
|
||||
}
|
||||
|
||||
public void start(String requestedMode, int captureId) {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
emitStatus("Voice control is already listening.");
|
||||
emitStopped(captureId);
|
||||
return;
|
||||
}
|
||||
if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) {
|
||||
emitStatus("Voice control needs an active G2 connection.");
|
||||
emitStopped(captureId);
|
||||
return;
|
||||
}
|
||||
mode = parseMode(requestedMode);
|
||||
activePhoneMic = usePhoneMic;
|
||||
started = true;
|
||||
audioStarted = false;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
||||
workerThread = new Thread(() -> runLoop(captureId), "FaceclawVoiceController");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
@@ -306,18 +356,19 @@ public class FaceclawVoiceController {
|
||||
return VoiceInputMode.ONBOARD;
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
private void runLoop(int captureId) {
|
||||
try {
|
||||
deleteLegacyKwsFiles();
|
||||
VoiceInputMode currentMode = mode;
|
||||
OnboardModelKind currentOnboardKind = onboardModelKind;
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
File modelDir = findAsrModelDir();
|
||||
File modelDir = findAsrModelDir(currentOnboardKind);
|
||||
if (modelDir == null) {
|
||||
emitStatus("Voice model not downloaded (see Settings > Voice).");
|
||||
return;
|
||||
}
|
||||
emitStatus("Loading transcription model...");
|
||||
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
|
||||
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind));
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
}
|
||||
@@ -387,6 +438,7 @@ public class FaceclawVoiceController {
|
||||
audioStarted = false;
|
||||
workerThread = null;
|
||||
}
|
||||
emitStopped(captureId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,32 +505,48 @@ public class FaceclawVoiceController {
|
||||
return b.array();
|
||||
}
|
||||
|
||||
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) {
|
||||
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir, OnboardModelKind kind) {
|
||||
OfflineModelConfig.Builder modelConfig = OfflineModelConfig.builder()
|
||||
.setNumThreads(1);
|
||||
if (kind == OnboardModelKind.WHISPER) {
|
||||
modelConfig
|
||||
.setWhisper(OfflineWhisperModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "base.en-encoder.int8.onnx").getAbsolutePath())
|
||||
.setDecoder(new File(modelDir, "base.en-decoder.int8.onnx").getAbsolutePath())
|
||||
.setLanguage("en")
|
||||
.setTask("transcribe")
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "base.en-tokens.txt").getAbsolutePath());
|
||||
} else {
|
||||
modelConfig
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath());
|
||||
}
|
||||
return OfflineRecognizerConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setModelConfig(OfflineModelConfig.builder()
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setModelConfig(modelConfig.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* The Moonshine model directory, populated by the download flow in
|
||||
* asr-model.ts (releases before 0.5.0 copied the same files out of the
|
||||
* APK, so upgraded installs are already complete). Null when any file is
|
||||
* The on-device model directory for the given kind, populated by the
|
||||
* download flow in asr-model.ts (releases before 0.5.0 copied the
|
||||
* Moonshine files out of the APK directly, so upgraded installs are
|
||||
* already complete for that model). Null when any expected file is
|
||||
* missing, i.e. the model still needs to be downloaded.
|
||||
*/
|
||||
private File findAsrModelDir() {
|
||||
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR);
|
||||
for (String fileName : ASR_MODEL_FILES) {
|
||||
private File findAsrModelDir(OnboardModelKind kind) {
|
||||
boolean whisper = kind == OnboardModelKind.WHISPER;
|
||||
String dirName = whisper ? ASR_WHISPER_MODEL_DIR : ASR_MODEL_DIR;
|
||||
String[] fileNames = whisper ? ASR_WHISPER_MODEL_FILES : ASR_MODEL_FILES;
|
||||
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + dirName);
|
||||
for (String fileName : fileNames) {
|
||||
File file = new File(modelDir, fileName);
|
||||
if (!file.exists() || file.length() == 0) {
|
||||
return null;
|
||||
@@ -658,6 +726,16 @@ public class FaceclawVoiceController {
|
||||
|
||||
private void processRecognizer(float[] samples) {
|
||||
appendTranscriptSamples(samples);
|
||||
// Each Whisper call re-encodes the whole buffer plus ~10s of tail
|
||||
// padding (see the ASR_WHISPER_* comment above),
|
||||
// so it skips the live-partial redecode Moonshine does on this
|
||||
// interval and only decodes when a segment commits (8s buffer fill)
|
||||
// or the utterance ends (decodeTranscript(true) in runLoop()). This
|
||||
// means no live preview text while speaking in Whisper mode -- status
|
||||
// stays "Listening..." until release. This is a deliberate tradeoff.
|
||||
if (onboardModelKind == OnboardModelKind.WHISPER) {
|
||||
return;
|
||||
}
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES
|
||||
&& now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
|
||||
@@ -767,6 +845,14 @@ public class FaceclawVoiceController {
|
||||
return "";
|
||||
}
|
||||
float[] segment = Arrays.copyOf(transcriptSamples, sampleCount);
|
||||
// Whisper hallucinates text on near-silent input (a known quirk of the
|
||||
// model, not this pipeline); gate it on the PRE-normalization peak, since
|
||||
// normalizePeak() below would otherwise amplify true silence right up to
|
||||
// the target level and hide the very thing being checked for. Moonshine
|
||||
// does not share this failure mode in practice, so it is left unchanged.
|
||||
if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) {
|
||||
return "";
|
||||
}
|
||||
normalizePeak(segment);
|
||||
OfflineStream offlineStream = currentRecognizer.createStream();
|
||||
try {
|
||||
@@ -780,7 +866,7 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
}
|
||||
|
||||
private static void normalizePeak(float[] samples) {
|
||||
private static float peakAmplitude(float[] samples) {
|
||||
float peak = 0f;
|
||||
for (float s : samples) {
|
||||
float a = Math.abs(s);
|
||||
@@ -788,6 +874,11 @@ public class FaceclawVoiceController {
|
||||
peak = a;
|
||||
}
|
||||
}
|
||||
return peak;
|
||||
}
|
||||
|
||||
private static void normalizePeak(float[] samples) {
|
||||
float peak = peakAmplitude(samples);
|
||||
if (peak <= 0f) {
|
||||
return;
|
||||
}
|
||||
@@ -805,7 +896,7 @@ public class FaceclawVoiceController {
|
||||
(committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE;
|
||||
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
|
||||
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
|
||||
Log.i(TAG, "Moonshine decode final=" + isFinal
|
||||
Log.i(TAG, (onboardModelKind == OnboardModelKind.WHISPER ? "Whisper" : "Moonshine") + " decode final=" + isFinal
|
||||
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec)
|
||||
+ " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE)
|
||||
+ " textLen=" + text.length() + " text=\"" + preview + "\"");
|
||||
@@ -930,6 +1021,13 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(currentListener::onSpeechEnd);
|
||||
}
|
||||
|
||||
private void emitStopped(int captureId) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener != null) {
|
||||
mainHandler.post(() -> currentListener.onStopped(captureId));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides when a hands-free utterance is over, so "Hey Even" capture can
|
||||
* stop without a button release.
|
||||
|
||||
+3
@@ -10,6 +10,9 @@ public interface FaceclawVoiceControllerListener {
|
||||
*/
|
||||
void onTranscript(String text, boolean isFinal);
|
||||
|
||||
/** Capture has finished, after the final transcript or a startup/recognition error. */
|
||||
void onStopped(int captureId);
|
||||
|
||||
/**
|
||||
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
|
||||
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
|
||||
|
||||
@@ -162,7 +162,7 @@ public class OfflineModelConfig {
|
||||
public static class Builder {
|
||||
private OfflineTransducerModelConfig transducer = new OfflineTransducerModelConfig();
|
||||
private OfflineParaformerModelConfig paraformer = new OfflineParaformerModelConfig();
|
||||
private OfflineWhisperModelConfig whisper = new OfflineWhisperModelConfig();
|
||||
private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build();
|
||||
private OfflineFireRedAsrModelConfig fireRedAsr = new OfflineFireRedAsrModelConfig();
|
||||
private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build();
|
||||
private OfflineNemoEncDecCtcModelConfig nemo = new OfflineNemoEncDecCtcModelConfig();
|
||||
@@ -191,6 +191,11 @@ public class OfflineModelConfig {
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setWhisper(OfflineWhisperModelConfig whisper) {
|
||||
this.whisper = whisper;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTokens(String tokens) {
|
||||
this.tokens = tokens;
|
||||
return this;
|
||||
@@ -228,16 +233,6 @@ class OfflineParaformerModelConfig {
|
||||
private final QnnConfig qnnConfig = new QnnConfig();
|
||||
}
|
||||
|
||||
class OfflineWhisperModelConfig {
|
||||
private final String encoder = "";
|
||||
private final String decoder = "";
|
||||
private final String language = "en";
|
||||
private final String task = "transcribe";
|
||||
private final int tailPaddings = 1000;
|
||||
private final boolean enableTokenTimestamps = false;
|
||||
private final boolean enableSegmentTimestamps = false;
|
||||
}
|
||||
|
||||
class OfflineFireRedAsrModelConfig {
|
||||
private final String encoder = "";
|
||||
private final String decoder = "";
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
/**
|
||||
* Config for sherpa-onnx's offline (non-streaming) Whisper backend.
|
||||
*
|
||||
* Field names and types here are load-bearing: the native JNI glue reads
|
||||
* them by name via reflection (GetFieldID), not through the getters below,
|
||||
* so they must match sherpa-onnx's own OfflineWhisperModelConfig exactly
|
||||
* (verified against sherpa-onnx/jni/offline-recognizer.cc at the pinned
|
||||
* release tag: it unconditionally reads encoder, decoder, language, task,
|
||||
* tailPaddings, enableTokenTimestamps and enableSegmentTimestamps off the
|
||||
* OfflineModelConfig's "whisper" field). This mirrors the
|
||||
* OfflineMoonshineModelConfig.java pattern already used in this package.
|
||||
*/
|
||||
public class OfflineWhisperModelConfig {
|
||||
private final String encoder;
|
||||
private final String decoder;
|
||||
private final String language;
|
||||
private final String task;
|
||||
private final int tailPaddings;
|
||||
private final boolean enableTokenTimestamps;
|
||||
private final boolean enableSegmentTimestamps;
|
||||
|
||||
private OfflineWhisperModelConfig(Builder builder) {
|
||||
this.encoder = builder.encoder;
|
||||
this.decoder = builder.decoder;
|
||||
this.language = builder.language;
|
||||
this.task = builder.task;
|
||||
this.tailPaddings = builder.tailPaddings;
|
||||
this.enableTokenTimestamps = builder.enableTokenTimestamps;
|
||||
this.enableSegmentTimestamps = builder.enableSegmentTimestamps;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getEncoder() {
|
||||
return encoder;
|
||||
}
|
||||
|
||||
public String getDecoder() {
|
||||
return decoder;
|
||||
}
|
||||
|
||||
public String getLanguage() {
|
||||
return language;
|
||||
}
|
||||
|
||||
public String getTask() {
|
||||
return task;
|
||||
}
|
||||
|
||||
public int getTailPaddings() {
|
||||
return tailPaddings;
|
||||
}
|
||||
|
||||
public boolean getEnableTokenTimestamps() {
|
||||
return enableTokenTimestamps;
|
||||
}
|
||||
|
||||
public boolean getEnableSegmentTimestamps() {
|
||||
return enableSegmentTimestamps;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String encoder = "";
|
||||
private String decoder = "";
|
||||
private String language = "en";
|
||||
private String task = "transcribe";
|
||||
private int tailPaddings = 1000;
|
||||
private boolean enableTokenTimestamps = false;
|
||||
private boolean enableSegmentTimestamps = false;
|
||||
|
||||
public Builder setEncoder(String encoder) {
|
||||
this.encoder = encoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDecoder(String decoder) {
|
||||
this.decoder = decoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLanguage(String language) {
|
||||
this.language = language;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTask(String task) {
|
||||
this.task = task;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTailPaddings(int tailPaddings) {
|
||||
this.tailPaddings = tailPaddings;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEnableTokenTimestamps(boolean enableTokenTimestamps) {
|
||||
this.enableTokenTimestamps = enableTokenTimestamps;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEnableSegmentTimestamps(boolean enableSegmentTimestamps) {
|
||||
this.enableSegmentTimestamps = enableSegmentTimestamps;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OfflineWhisperModelConfig build() {
|
||||
return new OfflineWhisperModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user