Work on voice input

This commit is contained in:
James Babcock committed 2026-07-05 20:59:53 -07:00
1 parent 7b8fc8f37f
commit 097d40efb2
12 files changed
+599 -234

No files matched your search

@@ -36,11 +36,15 @@ public class FaceclawVoiceController {
private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
private static final int LATE_PACKET_INTERVAL_MS = 90;
private static final int STATS_INTERVAL_MS = 5_000;
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 900;
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 2;
private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 8;
private static final int TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES = (int) (SAMPLE_RATE * 1.2);
private static final float TRANSCRIPT_SILENCE_PEAK = 0.015f;
// Push-to-talk utterance boundaries come from the button, so we accumulate
// the whole utterance and re-decode it in full for each live partial. The
// decode is of the complete audio each time, so the emitted text is the
// model's current best transcript of everything spoken so far (REPLACE,
// never a delta) — re-decoding a growing buffer and diffing prefixes was
// the source of the duplicated/garbled output.
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700;
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3;
private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 30;
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
@@ -60,8 +64,9 @@ public class FaceclawVoiceController {
};
private enum VoiceInputMode {
WAKEWORD,
FULL
WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired)
ONBOARD, // on-phone Moonshine transcription
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
}
private final Context appContext;
@@ -80,11 +85,8 @@ public class FaceclawVoiceController {
private FaceclawLc3Decoder lc3Decoder;
private float[] transcriptSamples = new float[TRANSCRIPT_MAX_SAMPLES];
private int transcriptSampleCount;
private int transcriptSilentSamples;
private boolean transcriptHadSpeech;
private long lastTranscriptDecodeAtMs;
private String lastTranscript = "";
private float lastTranscriptPeak;
private long queuedPackets;
private long queueDroppedPackets;
private long decodedSamples;
@@ -157,35 +159,47 @@ public class FaceclawVoiceController {
}
private VoiceInputMode parseMode(String requestedMode) {
return "full".equals(requestedMode) ? VoiceInputMode.FULL : VoiceInputMode.WAKEWORD;
if ("cloud".equals(requestedMode)) {
return VoiceInputMode.CLOUD;
}
if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) {
return VoiceInputMode.ONBOARD;
}
return VoiceInputMode.WAKEWORD;
}
private void runLoop() {
try {
VoiceInputMode currentMode = mode;
emitStatus(currentMode == VoiceInputMode.FULL
? "Voice control loading transcription model..."
: "Voice control loading wake-word model...");
if (currentMode == VoiceInputMode.FULL) {
if (currentMode == VoiceInputMode.ONBOARD) {
emitStatus("Loading transcription model...");
File modelDir = installAsrModelFiles();
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
resetTranscriptState();
lastTranscript = "";
} else {
} else if (currentMode == VoiceInputMode.WAKEWORD) {
emitStatus("Loading wake-word model...");
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
}
lc3Decoder = new FaceclawLc3Decoder();
if (!startG2Audio()) {
emitStatus("Voice control could not start G2 microphone input.");
emitStatus("Could not start G2 microphone input.");
return;
}
emitStatus(currentMode == VoiceInputMode.FULL
? "Voice control transcribing from the G2 mic."
: "Voice control listening for \"screen on\" from the G2 mic.");
emitStatus(currentMode == VoiceInputMode.CLOUD
? "Listening (cloud)..."
: currentMode == VoiceInputMode.ONBOARD
? "Listening..."
: "Listening for \"screen on\"...");
processG2Audio();
// Button released / stop requested: emit one final full-utterance
// transcript so the UI can freeze it.
if (currentMode == VoiceInputMode.ONBOARD) {
decodeTranscript(true);
}
} catch (Throwable error) {
Log.e(TAG, "Voice control failed", error);
emitStatus("Voice control failed: " + error.getMessage());
@@ -318,13 +332,19 @@ public class FaceclawVoiceController {
continue;
}
decodedSamples += count;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
if (mode == VoiceInputMode.FULL) {
if (mode == VoiceInputMode.CLOUD) {
emitPcm(pcm, count);
} else if (mode == VoiceInputMode.ONBOARD) {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
processRecognizer(samples);
} else {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
currentStream.acceptWaveform(samples, SAMPLE_RATE);
processKeywordSpotter(currentStream);
}
@@ -350,29 +370,19 @@ public class FaceclawVoiceController {
private void processRecognizer(float[] samples) {
appendTranscriptSamples(samples);
updateTranscriptSilence(samples);
if (transcriptSampleCount >= TRANSCRIPT_MAX_SAMPLES) {
decodeTranscriptSegment(true);
resetTranscriptState();
lastTranscript = "";
return;
}
long now = SystemClock.elapsedRealtime();
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
decodeTranscriptSegment(false);
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES
&& now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
decodeTranscript(false);
lastTranscriptDecodeAtMs = now;
}
if (transcriptHadSpeech && transcriptSilentSamples >= TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES) {
decodeTranscriptSegment(true);
resetTranscriptState();
lastTranscript = "";
}
}
private void appendTranscriptSamples(float[] samples) {
// The utterance is bounded by the push-to-talk button; the cap is only a
// safety limit. If it's hit, keep the most recent audio and let the
// transcript track that window (the beginning is unlikely to still be on
// screen after 30s anyway).
int available = TRANSCRIPT_MAX_SAMPLES - transcriptSampleCount;
if (samples.length > available) {
int drop = samples.length - available;
@@ -387,83 +397,55 @@ public class FaceclawVoiceController {
transcriptSampleCount += samples.length;
}
private void updateTranscriptSilence(float[] samples) {
float peak = 0.0f;
for (float sample : samples) {
float abs = Math.abs(sample);
if (abs > peak) {
peak = abs;
}
}
if (peak > lastTranscriptPeak) {
lastTranscriptPeak = peak;
}
if (peak < TRANSCRIPT_SILENCE_PEAK) {
transcriptSilentSamples += samples.length;
} else {
transcriptSilentSamples = 0;
transcriptHadSpeech = true;
}
}
private void decodeTranscriptSegment(boolean isFinal) {
/**
* Decode the entire accumulated utterance and emit the model's current best
* full transcript (REPLACE semantics — the caller displays it as-is).
*/
private void decodeTranscript(boolean isFinal) {
OfflineRecognizer currentRecognizer = recognizer;
if (currentRecognizer == null || transcriptSampleCount <= 0) {
if (isFinal) {
emitTranscript(lastTranscript, true);
}
return;
}
OfflineStream offlineStream = currentRecognizer.createStream();
String text;
try {
offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE);
currentRecognizer.decode(offlineStream);
emitRecognizerResult(currentRecognizer.getResult(offlineStream), isFinal, transcriptSampleCount, lastTranscriptPeak);
OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream);
String raw = result == null ? "" : result.getText();
text = raw == null ? "" : raw.trim();
} finally {
offlineStream.release();
}
}
private void emitRecognizerResult(OfflineRecognizerResult result, boolean isFinal, int sampleCount, float peak) {
String text = result == null ? "" : result.getText();
if (text == null) {
text = "";
}
String trimmed = text.trim();
String preview = trimmed.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
? trimmed
: trimmed.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
lastTranscript = text;
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
Log.i(TAG, "Moonshine decode final=" + isFinal
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", sampleCount / (double) SAMPLE_RATE)
+ " peak=" + String.format(java.util.Locale.US, "%.4f", peak)
+ " textLen=" + trimmed.length()
+ " text=\"" + preview + "\"");
emitStatus("Moonshine decode "
+ String.format(java.util.Locale.US, "%.1fs", sampleCount / (double) SAMPLE_RATE)
+ ", textLen=" + trimmed.length());
if (text.equals(lastTranscript)) {
if (isFinal && lastTranscript.length() > 0) {
emitTranscript("", true);
lastTranscript = "";
}
return;
}
String delta = text;
if (lastTranscript.length() > 0 && text.startsWith(lastTranscript)) {
delta = text.substring(lastTranscript.length());
}
if (delta.trim().length() > 0) {
emitTranscript(delta, false);
}
lastTranscript = isFinal ? "" : text;
if (isFinal) {
emitTranscript("", true);
}
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", transcriptSampleCount / (double) SAMPLE_RATE)
+ " textLen=" + text.length() + " text=\"" + preview + "\"");
emitTranscript(text, isFinal);
}
private void resetTranscriptState() {
transcriptSampleCount = 0;
transcriptSilentSamples = 0;
transcriptHadSpeech = false;
lastTranscriptDecodeAtMs = 0;
lastTranscriptPeak = 0.0f;
}
private void emitPcm(short[] pcm, int count) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null || count <= 0) {
return;
}
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
short s = pcm[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
mainHandler.post(() -> currentListener.onPcm(le));
}
private void stopG2Audio() {
@@ -5,5 +5,16 @@ public interface FaceclawVoiceControllerListener {
void onWakeWord(String keyword);
/**
* Best transcript of the current utterance so far. REPLACE semantics: text
* is the complete transcript, not a delta — display it verbatim, replacing
* any previous partial. isFinal marks the end of the utterance.
*/
void onTranscript(String text, boolean isFinal);
/**
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
* little-endian PCM. Empty/absent in onboard mode.
*/
void onPcm(byte[] pcm16le);
}
@@ -26,13 +26,21 @@ public class FaceclawWebSocket {
private volatile boolean closeRequested;
public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener) {
this(url, listener, null, null);
}
public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener, String headerName, String headerValue) {
if (url == null || url.trim().isEmpty()) {
throw new IllegalArgumentException("url is required");
}
if (listener == null) {
throw new IllegalArgumentException("listener is required");
}
Request request = new Request.Builder().url(url.trim()).build();
Request.Builder builder = new Request.Builder().url(url.trim());
if (headerName != null && !headerName.isEmpty() && headerValue != null) {
builder.addHeader(headerName, headerValue);
}
Request request = builder.build();
socket = getClient().newWebSocket(request, new WebSocketListener() {
@Override public void onOpen(WebSocket webSocket, Response response) {
mainHandler.post(() -> {