Fix an issue in the on-device transcription that would cause it to break on longer inputs

Thanks. Let's move the Microphone button to the top-right position in the 3x3 arrangement.
This commit is contained in:
James Babcock committed 2026-07-26 18:43:33 -07:00
1 parent 57d8d2fc79
commit dcf68cb09a
1 file changed
+85 -31
@@ -37,15 +37,14 @@ public class FaceclawVoiceController {
private static final int EXPECTED_PACKET_INTERVAL_MS = 50; private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
private static final int LATE_PACKET_INTERVAL_MS = 90; private static final int LATE_PACKET_INTERVAL_MS = 90;
private static final int STATS_INTERVAL_MS = 5_000; private static final int STATS_INTERVAL_MS = 5_000;
// Push-to-talk utterance boundaries come from the button, so we accumulate // Push-to-talk utterance boundaries come from the button. We re-decode the
// the whole utterance and re-decode it in full for each live partial. The // current audio segment in full for each live partial and emit the complete
// decode is of the complete audio each time, so the emitted text is the // utterance text (REPLACE, never a delta). The sherpa Moonshine v2 decoder
// model's current best transcript of everything spoken so far (REPLACE, // used here fails once a single input grows past roughly 9.1 seconds, so
// never a delta) — re-decoding a growing buffer and diffing prefixes was // longer utterances are committed in model-safe segments.
// the source of the duplicated/garbled output.
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700; private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700;
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3; private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3;
private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 30; private static final int TRANSCRIPT_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80; private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
private static final String ASSET_ROOT = "faceclaw-voice"; private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
@@ -84,8 +83,11 @@ public class FaceclawVoiceController {
private OfflineRecognizer recognizer; private OfflineRecognizer recognizer;
private OnlineStream stream; private OnlineStream stream;
private FaceclawLc3Decoder lc3Decoder; private FaceclawLc3Decoder lc3Decoder;
private float[] transcriptSamples = new float[TRANSCRIPT_MAX_SAMPLES]; private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
private int transcriptSampleCount; private int transcriptSampleCount;
private long committedTranscriptSampleCount;
private String committedTranscript = "";
private String currentSegmentTranscript = "";
private long lastTranscriptDecodeAtMs; private long lastTranscriptDecodeAtMs;
private String lastTranscript = ""; private String lastTranscript = "";
private volatile boolean saveRecordings; private volatile boolean saveRecordings;
@@ -470,58 +472,110 @@ public class FaceclawVoiceController {
} }
private void appendTranscriptSamples(float[] samples) { private void appendTranscriptSamples(float[] samples) {
// The utterance is bounded by the push-to-talk button; the cap is only a int sourceOffset = 0;
// safety limit. If it's hit, keep the most recent audio and let the while (sourceOffset < samples.length) {
// transcript track that window (the beginning is unlikely to still be on int available = TRANSCRIPT_SEGMENT_MAX_SAMPLES - transcriptSampleCount;
// screen after 30s anyway). int count = Math.min(available, samples.length - sourceOffset);
int available = TRANSCRIPT_MAX_SAMPLES - transcriptSampleCount; System.arraycopy(samples, sourceOffset, transcriptSamples, transcriptSampleCount, count);
if (samples.length > available) { transcriptSampleCount += count;
int drop = samples.length - available; sourceOffset += count;
if (drop >= transcriptSampleCount) {
transcriptSampleCount = 0; if (transcriptSampleCount == TRANSCRIPT_SEGMENT_MAX_SAMPLES) {
} else { commitTranscriptSegment();
System.arraycopy(transcriptSamples, drop, transcriptSamples, 0, transcriptSampleCount - drop);
transcriptSampleCount -= drop;
} }
} }
System.arraycopy(samples, 0, transcriptSamples, transcriptSampleCount, samples.length);
transcriptSampleCount += samples.length;
} }
/** /**
* Decode the entire accumulated utterance and emit the model's current best * Decode the current model-safe segment and emit the best transcript of the
* full transcript (REPLACE semantics — the caller displays it as-is). * complete utterance (REPLACE semantics — the caller displays it as-is).
*/ */
private void decodeTranscript(boolean isFinal) { private void decodeTranscript(boolean isFinal) {
OfflineRecognizer currentRecognizer = recognizer; if (recognizer == null || transcriptSampleCount <= 0) {
if (currentRecognizer == null || transcriptSampleCount <= 0) {
if (isFinal) { if (isFinal) {
emitTranscript(lastTranscript, true); emitTranscript(lastTranscript, true);
} }
return; return;
} }
int segmentSampleCount = transcriptSampleCount;
String segmentText = recognizeTranscriptSegment();
if (segmentText.length() > 0) {
currentSegmentTranscript = segmentText;
} else {
segmentText = currentSegmentTranscript;
}
String text = joinTranscript(committedTranscript, segmentText);
lastTranscript = text;
logTranscriptDecode(isFinal, segmentSampleCount, text);
emitTranscript(text, isFinal);
}
/**
* Finalize a full segment before accepting more audio. This keeps every
* Moonshine invocation below its failing sequence length while retaining
* all earlier text in the replace-semantics preview.
*/
private void commitTranscriptSegment() {
int segmentSampleCount = transcriptSampleCount;
String segmentText = recognizeTranscriptSegment();
if (segmentText.length() == 0) {
segmentText = currentSegmentTranscript;
}
committedTranscript = joinTranscript(committedTranscript, segmentText);
currentSegmentTranscript = "";
committedTranscriptSampleCount += segmentSampleCount;
transcriptSampleCount = 0;
lastTranscript = committedTranscript;
lastTranscriptDecodeAtMs = SystemClock.elapsedRealtime();
logTranscriptDecode(false, segmentSampleCount, committedTranscript);
emitTranscript(committedTranscript, false);
}
private String recognizeTranscriptSegment() {
OfflineRecognizer currentRecognizer = recognizer;
if (currentRecognizer == null || transcriptSampleCount <= 0) {
return "";
}
OfflineStream offlineStream = currentRecognizer.createStream(); OfflineStream offlineStream = currentRecognizer.createStream();
String text;
try { try {
offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE); offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE);
currentRecognizer.decode(offlineStream); currentRecognizer.decode(offlineStream);
OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream); OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream);
String raw = result == null ? "" : result.getText(); String raw = result == null ? "" : result.getText();
text = raw == null ? "" : raw.trim(); return raw == null ? "" : raw.trim();
} finally { } finally {
offlineStream.release(); offlineStream.release();
} }
lastTranscript = text; }
private void logTranscriptDecode(boolean isFinal, int segmentSampleCount, String text) {
double totalAudioSec =
(committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE;
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."; ? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
Log.i(TAG, "Moonshine decode final=" + isFinal Log.i(TAG, "Moonshine decode final=" + isFinal
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", transcriptSampleCount / (double) SAMPLE_RATE) + " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec)
+ " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE)
+ " textLen=" + text.length() + " text=\"" + preview + "\""); + " textLen=" + text.length() + " text=\"" + preview + "\"");
emitTranscript(text, isFinal); }
private static String joinTranscript(String prefix, String suffix) {
if (prefix == null || prefix.length() == 0) {
return suffix == null ? "" : suffix;
}
if (suffix == null || suffix.length() == 0) {
return prefix;
}
char first = suffix.charAt(0);
boolean attachesToPrevious = ".,!?;:%)]}".indexOf(first) >= 0;
return prefix + (attachesToPrevious ? "" : " ") + suffix;
} }
private void resetTranscriptState() { private void resetTranscriptState() {
transcriptSampleCount = 0; transcriptSampleCount = 0;
committedTranscriptSampleCount = 0;
committedTranscript = "";
currentSegmentTranscript = "";
lastTranscriptDecodeAtMs = 0; lastTranscriptDecodeAtMs = 0;
} }