forked from Blendman974/faceclaw
Fix an issue in the on-device transcription that would cause it to break on longer inputs
Thanks. Let's move the Microphone button to the top-right position in the 3x3 arrangement.
This commit is contained in:
1 parent
57d8d2fc79
commit
dcf68cb09a
1 file changed
+85
-31
@@ -37,15 +37,14 @@ public class FaceclawVoiceController {
|
|||||||
private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
|
private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
|
||||||
private static final int LATE_PACKET_INTERVAL_MS = 90;
|
private static final int LATE_PACKET_INTERVAL_MS = 90;
|
||||||
private static final int STATS_INTERVAL_MS = 5_000;
|
private static final int STATS_INTERVAL_MS = 5_000;
|
||||||
// Push-to-talk utterance boundaries come from the button, so we accumulate
|
// Push-to-talk utterance boundaries come from the button. We re-decode the
|
||||||
// the whole utterance and re-decode it in full for each live partial. The
|
// current audio segment in full for each live partial and emit the complete
|
||||||
// decode is of the complete audio each time, so the emitted text is the
|
// utterance text (REPLACE, never a delta). The sherpa Moonshine v2 decoder
|
||||||
// model's current best transcript of everything spoken so far (REPLACE,
|
// used here fails once a single input grows past roughly 9.1 seconds, so
|
||||||
// never a delta) — re-decoding a growing buffer and diffing prefixes was
|
// longer utterances are committed in model-safe segments.
|
||||||
// the source of the duplicated/garbled output.
|
|
||||||
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700;
|
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700;
|
||||||
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3;
|
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3;
|
||||||
private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 30;
|
private static final int TRANSCRIPT_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
|
||||||
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
|
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
|
||||||
private static final String ASSET_ROOT = "faceclaw-voice";
|
private static final String ASSET_ROOT = "faceclaw-voice";
|
||||||
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
||||||
@@ -84,8 +83,11 @@ public class FaceclawVoiceController {
|
|||||||
private OfflineRecognizer recognizer;
|
private OfflineRecognizer recognizer;
|
||||||
private OnlineStream stream;
|
private OnlineStream stream;
|
||||||
private FaceclawLc3Decoder lc3Decoder;
|
private FaceclawLc3Decoder lc3Decoder;
|
||||||
private float[] transcriptSamples = new float[TRANSCRIPT_MAX_SAMPLES];
|
private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
|
||||||
private int transcriptSampleCount;
|
private int transcriptSampleCount;
|
||||||
|
private long committedTranscriptSampleCount;
|
||||||
|
private String committedTranscript = "";
|
||||||
|
private String currentSegmentTranscript = "";
|
||||||
private long lastTranscriptDecodeAtMs;
|
private long lastTranscriptDecodeAtMs;
|
||||||
private String lastTranscript = "";
|
private String lastTranscript = "";
|
||||||
private volatile boolean saveRecordings;
|
private volatile boolean saveRecordings;
|
||||||
@@ -470,58 +472,110 @@ public class FaceclawVoiceController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void appendTranscriptSamples(float[] samples) {
|
private void appendTranscriptSamples(float[] samples) {
|
||||||
// The utterance is bounded by the push-to-talk button; the cap is only a
|
int sourceOffset = 0;
|
||||||
// safety limit. If it's hit, keep the most recent audio and let the
|
while (sourceOffset < samples.length) {
|
||||||
// transcript track that window (the beginning is unlikely to still be on
|
int available = TRANSCRIPT_SEGMENT_MAX_SAMPLES - transcriptSampleCount;
|
||||||
// screen after 30s anyway).
|
int count = Math.min(available, samples.length - sourceOffset);
|
||||||
int available = TRANSCRIPT_MAX_SAMPLES - transcriptSampleCount;
|
System.arraycopy(samples, sourceOffset, transcriptSamples, transcriptSampleCount, count);
|
||||||
if (samples.length > available) {
|
transcriptSampleCount += count;
|
||||||
int drop = samples.length - available;
|
sourceOffset += count;
|
||||||
if (drop >= transcriptSampleCount) {
|
|
||||||
transcriptSampleCount = 0;
|
if (transcriptSampleCount == TRANSCRIPT_SEGMENT_MAX_SAMPLES) {
|
||||||
} else {
|
commitTranscriptSegment();
|
||||||
System.arraycopy(transcriptSamples, drop, transcriptSamples, 0, transcriptSampleCount - drop);
|
|
||||||
transcriptSampleCount -= drop;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
System.arraycopy(samples, 0, transcriptSamples, transcriptSampleCount, samples.length);
|
|
||||||
transcriptSampleCount += samples.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Decode the entire accumulated utterance and emit the model's current best
|
* Decode the current model-safe segment and emit the best transcript of the
|
||||||
* full transcript (REPLACE semantics — the caller displays it as-is).
|
* complete utterance (REPLACE semantics — the caller displays it as-is).
|
||||||
*/
|
*/
|
||||||
private void decodeTranscript(boolean isFinal) {
|
private void decodeTranscript(boolean isFinal) {
|
||||||
OfflineRecognizer currentRecognizer = recognizer;
|
if (recognizer == null || transcriptSampleCount <= 0) {
|
||||||
if (currentRecognizer == null || transcriptSampleCount <= 0) {
|
|
||||||
if (isFinal) {
|
if (isFinal) {
|
||||||
emitTranscript(lastTranscript, true);
|
emitTranscript(lastTranscript, true);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
int segmentSampleCount = transcriptSampleCount;
|
||||||
|
String segmentText = recognizeTranscriptSegment();
|
||||||
|
if (segmentText.length() > 0) {
|
||||||
|
currentSegmentTranscript = segmentText;
|
||||||
|
} else {
|
||||||
|
segmentText = currentSegmentTranscript;
|
||||||
|
}
|
||||||
|
String text = joinTranscript(committedTranscript, segmentText);
|
||||||
|
lastTranscript = text;
|
||||||
|
logTranscriptDecode(isFinal, segmentSampleCount, text);
|
||||||
|
emitTranscript(text, isFinal);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Finalize a full segment before accepting more audio. This keeps every
|
||||||
|
* Moonshine invocation below its failing sequence length while retaining
|
||||||
|
* all earlier text in the replace-semantics preview.
|
||||||
|
*/
|
||||||
|
private void commitTranscriptSegment() {
|
||||||
|
int segmentSampleCount = transcriptSampleCount;
|
||||||
|
String segmentText = recognizeTranscriptSegment();
|
||||||
|
if (segmentText.length() == 0) {
|
||||||
|
segmentText = currentSegmentTranscript;
|
||||||
|
}
|
||||||
|
committedTranscript = joinTranscript(committedTranscript, segmentText);
|
||||||
|
currentSegmentTranscript = "";
|
||||||
|
committedTranscriptSampleCount += segmentSampleCount;
|
||||||
|
transcriptSampleCount = 0;
|
||||||
|
lastTranscript = committedTranscript;
|
||||||
|
lastTranscriptDecodeAtMs = SystemClock.elapsedRealtime();
|
||||||
|
logTranscriptDecode(false, segmentSampleCount, committedTranscript);
|
||||||
|
emitTranscript(committedTranscript, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String recognizeTranscriptSegment() {
|
||||||
|
OfflineRecognizer currentRecognizer = recognizer;
|
||||||
|
if (currentRecognizer == null || transcriptSampleCount <= 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
OfflineStream offlineStream = currentRecognizer.createStream();
|
OfflineStream offlineStream = currentRecognizer.createStream();
|
||||||
String text;
|
|
||||||
try {
|
try {
|
||||||
offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE);
|
offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE);
|
||||||
currentRecognizer.decode(offlineStream);
|
currentRecognizer.decode(offlineStream);
|
||||||
OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream);
|
OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream);
|
||||||
String raw = result == null ? "" : result.getText();
|
String raw = result == null ? "" : result.getText();
|
||||||
text = raw == null ? "" : raw.trim();
|
return raw == null ? "" : raw.trim();
|
||||||
} finally {
|
} finally {
|
||||||
offlineStream.release();
|
offlineStream.release();
|
||||||
}
|
}
|
||||||
lastTranscript = text;
|
}
|
||||||
|
|
||||||
|
private void logTranscriptDecode(boolean isFinal, int segmentSampleCount, String text) {
|
||||||
|
double totalAudioSec =
|
||||||
|
(committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE;
|
||||||
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
|
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
|
||||||
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
|
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
|
||||||
Log.i(TAG, "Moonshine decode final=" + isFinal
|
Log.i(TAG, "Moonshine decode final=" + isFinal
|
||||||
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", transcriptSampleCount / (double) SAMPLE_RATE)
|
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec)
|
||||||
|
+ " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE)
|
||||||
+ " textLen=" + text.length() + " text=\"" + preview + "\"");
|
+ " textLen=" + text.length() + " text=\"" + preview + "\"");
|
||||||
emitTranscript(text, isFinal);
|
}
|
||||||
|
|
||||||
|
private static String joinTranscript(String prefix, String suffix) {
|
||||||
|
if (prefix == null || prefix.length() == 0) {
|
||||||
|
return suffix == null ? "" : suffix;
|
||||||
|
}
|
||||||
|
if (suffix == null || suffix.length() == 0) {
|
||||||
|
return prefix;
|
||||||
|
}
|
||||||
|
char first = suffix.charAt(0);
|
||||||
|
boolean attachesToPrevious = ".,!?;:%)]}".indexOf(first) >= 0;
|
||||||
|
return prefix + (attachesToPrevious ? "" : " ") + suffix;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void resetTranscriptState() {
|
private void resetTranscriptState() {
|
||||||
transcriptSampleCount = 0;
|
transcriptSampleCount = 0;
|
||||||
|
committedTranscriptSampleCount = 0;
|
||||||
|
committedTranscript = "";
|
||||||
|
currentSegmentTranscript = "";
|
||||||
lastTranscriptDecodeAtMs = 0;
|
lastTranscriptDecodeAtMs = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user