From 097d40efb254a9861f475a1017e80577f4e39ad7 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 5 Jul 2026 20:59:53 -0700 Subject: [PATCH] Work on voice input --- .../faceclaw/app/FaceclawVoiceController.java | 186 ++++++++---------- .../app/FaceclawVoiceControllerListener.java | 11 ++ .../com/faceclaw/app/FaceclawWebSocket.java | 10 +- app/g2/dashboard-controller.ts | 76 +++---- app/native/elevenlabs-stt.ts | 175 ++++++++++++++++ app/native/voice-control.ts | 91 ++++++--- app/ui/apps/transcribe.ts | 78 +++----- app/ui/apps/voice-input.ts | 137 +++++++++++++ app/ui/dashboard-settings.ts | 27 ++- app/ui/dashboard.ts | 30 ++- app/ui/dashboard/settings-menus.ts | 5 +- app/ui/layers.ts | 7 +- 12 files changed, 599 insertions(+), 234 deletions(-) create mode 100644 app/native/elevenlabs-stt.ts create mode 100644 app/ui/apps/voice-input.ts diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 81e6206e..20248c96 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -36,11 +36,15 @@ public class FaceclawVoiceController { private static final int EXPECTED_PACKET_INTERVAL_MS = 50; private static final int LATE_PACKET_INTERVAL_MS = 90; private static final int STATS_INTERVAL_MS = 5_000; - private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 900; - private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 2; - private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 8; - private static final int TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES = (int) (SAMPLE_RATE * 1.2); - private static final float TRANSCRIPT_SILENCE_PEAK = 0.015f; + // Push-to-talk utterance boundaries come from the button, so we accumulate + // the whole utterance and re-decode it in full for each live partial. The + // decode is of the complete audio each time, so the emitted text is the + // model's current best transcript of everything spoken so far (REPLACE, + // never a delta) — re-decoding a growing buffer and diffing prefixes was + // the source of the duplicated/garbled output. + private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700; + private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3; + private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 30; private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80; private static final String ASSET_ROOT = "faceclaw-voice"; private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; @@ -60,8 +64,9 @@ public class FaceclawVoiceController { }; private enum VoiceInputMode { - WAKEWORD, - FULL + WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired) + ONBOARD, // on-phone Moonshine transcription + CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side } private final Context appContext; @@ -80,11 +85,8 @@ public class FaceclawVoiceController { private FaceclawLc3Decoder lc3Decoder; private float[] transcriptSamples = new float[TRANSCRIPT_MAX_SAMPLES]; private int transcriptSampleCount; - private int transcriptSilentSamples; - private boolean transcriptHadSpeech; private long lastTranscriptDecodeAtMs; private String lastTranscript = ""; - private float lastTranscriptPeak; private long queuedPackets; private long queueDroppedPackets; private long decodedSamples; @@ -157,35 +159,47 @@ public class FaceclawVoiceController { } private VoiceInputMode parseMode(String requestedMode) { - return "full".equals(requestedMode) ? VoiceInputMode.FULL : VoiceInputMode.WAKEWORD; + if ("cloud".equals(requestedMode)) { + return VoiceInputMode.CLOUD; + } + if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) { + return VoiceInputMode.ONBOARD; + } + return VoiceInputMode.WAKEWORD; } private void runLoop() { try { VoiceInputMode currentMode = mode; - emitStatus(currentMode == VoiceInputMode.FULL - ? "Voice control loading transcription model..." - : "Voice control loading wake-word model..."); - if (currentMode == VoiceInputMode.FULL) { + if (currentMode == VoiceInputMode.ONBOARD) { + emitStatus("Loading transcription model..."); File modelDir = installAsrModelFiles(); recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir)); resetTranscriptState(); lastTranscript = ""; - } else { + } else if (currentMode == VoiceInputMode.WAKEWORD) { + emitStatus("Loading wake-word model..."); File modelDir = installModelFiles(); keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); stream = keywordSpotter.createStream(); } lc3Decoder = new FaceclawLc3Decoder(); if (!startG2Audio()) { - emitStatus("Voice control could not start G2 microphone input."); + emitStatus("Could not start G2 microphone input."); return; } - emitStatus(currentMode == VoiceInputMode.FULL - ? "Voice control transcribing from the G2 mic." - : "Voice control listening for \"screen on\" from the G2 mic."); + emitStatus(currentMode == VoiceInputMode.CLOUD + ? "Listening (cloud)..." + : currentMode == VoiceInputMode.ONBOARD + ? "Listening..." + : "Listening for \"screen on\"..."); processG2Audio(); + // Button released / stop requested: emit one final full-utterance + // transcript so the UI can freeze it. + if (currentMode == VoiceInputMode.ONBOARD) { + decodeTranscript(true); + } } catch (Throwable error) { Log.e(TAG, "Voice control failed", error); emitStatus("Voice control failed: " + error.getMessage()); @@ -318,13 +332,19 @@ public class FaceclawVoiceController { continue; } decodedSamples += count; - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = pcm[i] / 32768.0f; - } - if (mode == VoiceInputMode.FULL) { + if (mode == VoiceInputMode.CLOUD) { + emitPcm(pcm, count); + } else if (mode == VoiceInputMode.ONBOARD) { + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + samples[i] = pcm[i] / 32768.0f; + } processRecognizer(samples); } else { + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + samples[i] = pcm[i] / 32768.0f; + } currentStream.acceptWaveform(samples, SAMPLE_RATE); processKeywordSpotter(currentStream); } @@ -350,29 +370,19 @@ public class FaceclawVoiceController { private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); - updateTranscriptSilence(samples); - - if (transcriptSampleCount >= TRANSCRIPT_MAX_SAMPLES) { - decodeTranscriptSegment(true); - resetTranscriptState(); - lastTranscript = ""; - return; - } - long now = SystemClock.elapsedRealtime(); - if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { - decodeTranscriptSegment(false); + if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES + && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { + decodeTranscript(false); lastTranscriptDecodeAtMs = now; } - - if (transcriptHadSpeech && transcriptSilentSamples >= TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES) { - decodeTranscriptSegment(true); - resetTranscriptState(); - lastTranscript = ""; - } } private void appendTranscriptSamples(float[] samples) { + // The utterance is bounded by the push-to-talk button; the cap is only a + // safety limit. If it's hit, keep the most recent audio and let the + // transcript track that window (the beginning is unlikely to still be on + // screen after 30s anyway). int available = TRANSCRIPT_MAX_SAMPLES - transcriptSampleCount; if (samples.length > available) { int drop = samples.length - available; @@ -387,83 +397,55 @@ public class FaceclawVoiceController { transcriptSampleCount += samples.length; } - private void updateTranscriptSilence(float[] samples) { - float peak = 0.0f; - for (float sample : samples) { - float abs = Math.abs(sample); - if (abs > peak) { - peak = abs; - } - } - if (peak > lastTranscriptPeak) { - lastTranscriptPeak = peak; - } - if (peak < TRANSCRIPT_SILENCE_PEAK) { - transcriptSilentSamples += samples.length; - } else { - transcriptSilentSamples = 0; - transcriptHadSpeech = true; - } - } - - private void decodeTranscriptSegment(boolean isFinal) { + /** + * Decode the entire accumulated utterance and emit the model's current best + * full transcript (REPLACE semantics — the caller displays it as-is). + */ + private void decodeTranscript(boolean isFinal) { OfflineRecognizer currentRecognizer = recognizer; if (currentRecognizer == null || transcriptSampleCount <= 0) { + if (isFinal) { + emitTranscript(lastTranscript, true); + } return; } OfflineStream offlineStream = currentRecognizer.createStream(); + String text; try { offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE); currentRecognizer.decode(offlineStream); - emitRecognizerResult(currentRecognizer.getResult(offlineStream), isFinal, transcriptSampleCount, lastTranscriptPeak); + OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream); + String raw = result == null ? "" : result.getText(); + text = raw == null ? "" : raw.trim(); } finally { offlineStream.release(); } - } - - private void emitRecognizerResult(OfflineRecognizerResult result, boolean isFinal, int sampleCount, float peak) { - String text = result == null ? "" : result.getText(); - if (text == null) { - text = ""; - } - String trimmed = text.trim(); - String preview = trimmed.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS - ? trimmed - : trimmed.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."; + lastTranscript = text; + String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS + ? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."; Log.i(TAG, "Moonshine decode final=" + isFinal - + " audioSec=" + String.format(java.util.Locale.US, "%.2f", sampleCount / (double) SAMPLE_RATE) - + " peak=" + String.format(java.util.Locale.US, "%.4f", peak) - + " textLen=" + trimmed.length() - + " text=\"" + preview + "\""); - emitStatus("Moonshine decode " - + String.format(java.util.Locale.US, "%.1fs", sampleCount / (double) SAMPLE_RATE) - + ", textLen=" + trimmed.length()); - if (text.equals(lastTranscript)) { - if (isFinal && lastTranscript.length() > 0) { - emitTranscript("", true); - lastTranscript = ""; - } - return; - } - String delta = text; - if (lastTranscript.length() > 0 && text.startsWith(lastTranscript)) { - delta = text.substring(lastTranscript.length()); - } - if (delta.trim().length() > 0) { - emitTranscript(delta, false); - } - lastTranscript = isFinal ? "" : text; - if (isFinal) { - emitTranscript("", true); - } + + " audioSec=" + String.format(java.util.Locale.US, "%.2f", transcriptSampleCount / (double) SAMPLE_RATE) + + " textLen=" + text.length() + " text=\"" + preview + "\""); + emitTranscript(text, isFinal); } private void resetTranscriptState() { transcriptSampleCount = 0; - transcriptSilentSamples = 0; - transcriptHadSpeech = false; lastTranscriptDecodeAtMs = 0; - lastTranscriptPeak = 0.0f; + } + + private void emitPcm(short[] pcm, int count) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null || count <= 0) { + return; + } + byte[] le = new byte[count * 2]; + for (int i = 0; i < count; i++) { + short s = pcm[i]; + le[i * 2] = (byte) (s & 0xff); + le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); + } + mainHandler.post(() -> currentListener.onPcm(le)); } private void stopG2Audio() { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index a41ee229..88f558bf 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -5,5 +5,16 @@ public interface FaceclawVoiceControllerListener { void onWakeWord(String keyword); + /** + * Best transcript of the current utterance so far. REPLACE semantics: text + * is the complete transcript, not a delta — display it verbatim, replacing + * any previous partial. isFinal marks the end of the utterance. + */ void onTranscript(String text, boolean isFinal); + + /** + * Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit + * little-endian PCM. Empty/absent in onboard mode. + */ + void onPcm(byte[] pcm16le); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java index 3ae5da96..d7668957 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocket.java @@ -26,13 +26,21 @@ public class FaceclawWebSocket { private volatile boolean closeRequested; public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener) { + this(url, listener, null, null); + } + + public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener, String headerName, String headerValue) { if (url == null || url.trim().isEmpty()) { throw new IllegalArgumentException("url is required"); } if (listener == null) { throw new IllegalArgumentException("listener is required"); } - Request request = new Request.Builder().url(url.trim()).build(); + Request.Builder builder = new Request.Builder().url(url.trim()); + if (headerName != null && !headerName.isEmpty() && headerValue != null) { + builder.addHeader(headerName, headerValue); + } + Request request = builder.build(); socket = getClient().newWebSocket(request, new WebSocketListener() { @Override public void onOpen(WebSocket webSocket, Response response) { mainHandler.post(() -> { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index d3349254..29360f3c 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -28,12 +28,13 @@ import { setDashboardActions, } from "../ui/dashboard"; import { + elevenLabsApiKeySetting, nightscoutApiTokenSetting, nightscoutSiteUrlSetting, onAnySettingChanged, rawScreenshotsEnabledSetting, systemCardNameSetting, - voiceControlEnabledSetting, + voiceProviderSetting, type ConfigSettingString, } from "../ui/dashboard-settings"; import { saveRawScreenshot } from "../native/raw-screenshot"; @@ -173,11 +174,10 @@ class DashboardController { disconnect: () => this.disconnect(), startTextSettingEdit: (setting) => this.startTextSettingEdit(setting), endTextSettingEdit: () => this.endTextSettingEdit(), - setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled), setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active), setTranscribeRenderActive: (active) => this.setTranscribeRenderActive(active), - startDedicatedVoiceInput: (mode) => this.startDedicatedVoiceInput(mode), - stopDedicatedVoiceInput: () => this.stopDedicatedVoiceInput(), + startVoiceCapture: () => this.startVoiceCapture(), + stopVoiceCapture: () => this.stopVoiceCapture(), playBuzzerNote: (note, oct, beat) => this.playBuzzerNote(note, oct, beat), }); this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => { @@ -417,7 +417,6 @@ class DashboardController { await nightscoutBridge.start(); await communicator.start(); await this.requestRender("initial"); - this.startVoiceControlIfEnabled(); this.dashboardTimer = setInterval(() => { void this.requestRender("interval").catch((error) => { const message = this.formatError(error); @@ -558,30 +557,32 @@ class DashboardController { this.emit(); } - private async setVoiceControlEnabled(enabled: boolean): Promise { - if (enabled) { - try { - // Treat Android mic permission as the consent gate for voice control, - // even when the current source is the G2 mic over BLE. - await ensureVoicePermissions(); - } catch (error) { - this.appendLog(`voice control permission failed: ${this.formatError(error)}`); - return; - } + /** + * Begin voice capture with the provider chosen in settings. Used by both + * push-to-talk and the Transcribe app. Android mic permission is the consent + * gate even though the audio source is the G2 mic over BLE. + */ + private startVoiceCapture(): void { + if (this.phase !== "connected" || !this.communicator) { + return; } - - voiceControlEnabledSetting.set(enabled); - this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`); - if (enabled) { - this.startVoiceControlIfEnabled(); - } else { - voiceControlBridge.stop(); - } - if (this.phase === "connected" && this.communicator) { - await this.requestRender("interval").catch((error) => { - this.appendLog(`voice setting render failed: ${this.formatError(error)}`); + const communicator = this.communicator; + void ensureVoicePermissions() + .then(() => { + if (this.phase !== "connected" || this.communicator !== communicator) return; + voiceControlBridge.startPushToTalk({ + communicator: communicator.getNativeCommunicator(), + provider: voiceProviderSetting.get(), + elevenLabsApiKey: elevenLabsApiKeySetting.get(), + }); + }) + .catch((error) => { + this.appendLog(`voice permission failed: ${this.formatError(error)}`); }); - } + } + + private stopVoiceCapture(): void { + voiceControlBridge.stopPushToTalk(); } private endTextSettingEdit(): void { @@ -839,27 +840,6 @@ class DashboardController { this.updateDisplayPreviewFromImage(image); } - private startVoiceControlIfEnabled(): void { - if (!this.communicator) return; - if (!voiceControlEnabledSetting.get()) return; - this.updateConnectedForegroundNotification(); - voiceControlBridge.start(this.communicator.getNativeCommunicator(), "wakeword"); - } - - private async startDedicatedVoiceInput(mode: "wakeword" | "full"): Promise { - if (!this.communicator) { - throw new Error("Voice input needs an active G2 connection."); - } - await ensureVoicePermissions(); - this.updateConnectedForegroundNotification(); - voiceControlBridge.start(this.communicator.getNativeCommunicator(), mode); - } - - private stopDedicatedVoiceInput(): void { - voiceControlBridge.stop(); - this.startVoiceControlIfEnabled(); - } - private async playBuzzerNote(note: number, oct: number, beat: number): Promise { if (this.phase !== "connected" || !this.communicator) { return; diff --git a/app/native/elevenlabs-stt.ts b/app/native/elevenlabs-stt.ts new file mode 100644 index 00000000..28eebf6f --- /dev/null +++ b/app/native/elevenlabs-stt.ts @@ -0,0 +1,175 @@ +declare const com: any; +declare const android: any; + +/** + * ElevenLabs realtime speech-to-text over WebSocket. + * https://elevenlabs.io/docs/api-reference/speech-to-text/v-1-speech-to-text-realtime + * + * We push 16 kHz signed-16-bit-LE PCM (base64) as it arrives from the G2 mic + * and, because push-to-talk gives us the utterance boundary, use manual commit: + * commit=true is sent when the button is released, which finalizes the + * transcript. Partial transcripts stream in the meantime. + */ + +const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime"; +// The only model the realtime endpoint accepts (verified against the live API; +// it rejects "scribe_v1" with an explicit message naming this one). +const MODEL_ID = "scribe_v2_realtime"; +const SAMPLE_RATE = 16000; + +export type ElevenLabsTranscriptEvent = { + text: string; + isFinal: boolean; +}; + +export type ElevenLabsSttOptions = { + apiKey: string; + onTranscript: (event: ElevenLabsTranscriptEvent) => void; + onStatus: (status: string) => void; + onError: (message: string) => void; +}; + +export class ElevenLabsSttClient { + private ws: any = null; + private listenerProxy: any = null; + private open = false; + private closed = false; + // PCM that arrived before the socket finished opening; flushed on open. + private readonly pendingChunks: string[] = []; + private latestText = ""; + + constructor(private readonly options: ElevenLabsSttOptions) {} + + start(): void { + if (this.ws) return; + const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`; + this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + onOpen: () => { + this.open = true; + for (const chunk of this.pendingChunks.splice(0)) { + this.sendChunk(chunk); + } + }, + onTextMessage: (message: string) => this.handleMessage(String(message)), + onClosed: () => { + this.open = false; + }, + onFailure: (message: string) => { + if (this.closed) return; + this.options.onError(`ElevenLabs connection failed: ${String(message)}`); + }, + }); + try { + // The API key rides an xi-api-key header, added by FaceclawWebSocket. + this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", this.options.apiKey); + this.options.onStatus("Connecting to ElevenLabs..."); + } catch (error) { + this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`); + } + } + + /** Feed PCM (16 kHz signed-16-bit LE). */ + acceptPcm(pcm: Uint8Array): void { + if (this.closed || pcm.length === 0) return; + const base64 = encodeBase64(pcm); + if (this.open) { + this.sendChunk(base64); + } else { + this.pendingChunks.push(base64); + } + } + + /** End of utterance: flush and commit for a final transcript. */ + finish(): void { + if (this.closed) return; + const commitMessage = JSON.stringify({ + message_type: "input_audio_chunk", + audio_base_64: "", + commit: true, + sample_rate: SAMPLE_RATE, + }); + if (this.open) { + this.trySend(commitMessage); + } else { + this.pendingChunks.push("__commit__"); + } + } + + stop(): void { + this.closed = true; + if (this.ws) { + try { + this.ws.close(1000, "bye"); + } catch { + // ignore + } + this.ws = null; + } + this.listenerProxy = null; + } + + private sendChunk(base64OrCommit: string): void { + if (base64OrCommit === "__commit__") { + this.finish(); + return; + } + this.trySend( + JSON.stringify({ + message_type: "input_audio_chunk", + audio_base_64: base64OrCommit, + commit: false, + sample_rate: SAMPLE_RATE, + }), + ); + } + + private trySend(message: string): void { + try { + this.ws?.sendText(message); + } catch (error) { + console.warn("elevenlabs send failed", error); + } + } + + private handleMessage(text: string): void { + let message: any; + try { + message = JSON.parse(text); + } catch { + return; + } + switch (message?.message_type) { + case "session_started": + this.options.onStatus("Listening (ElevenLabs)..."); + return; + case "partial_transcript": + this.latestText = String(message.text ?? ""); + this.options.onTranscript({ text: this.latestText, isFinal: false }); + return; + case "committed_transcript": + case "committed_transcript_with_timestamps": + this.latestText = String(message.text ?? this.latestText); + this.options.onTranscript({ text: this.latestText, isFinal: true }); + return; + case "error": + case "auth_error": + case "quota_exceeded": + case "rate_limited": + case "input_error": + this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`); + return; + default: + return; + } + } +} + +function encodeBase64(bytes: Uint8Array): string { + if (!global.isAndroid) return ""; + const javaBytes = Array.create("byte", bytes.length); + for (let i = 0; i < bytes.length; i++) { + const value = bytes[i]!; + javaBytes[i] = value > 127 ? value - 256 : value; + } + return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP)); +} diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index abf077d6..9e4b98f5 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,17 +1,31 @@ import { Utils } from "@nativescript/core"; +import { ElevenLabsSttClient } from "./elevenlabs-stt"; +import { toUint8Array } from "../util/array-util"; + declare const com: any; export type VoiceControlState = { status: string; }; -export type VoiceInputMode = "off" | "wakeword" | "full"; +export type VoiceProviderKind = "onboard" | "elevenlabs"; + export type VoiceTranscriptEvent = { + /** + * Complete best transcript of the current utterance. REPLACE semantics — + * render as-is, replacing any previous partial. Not a delta. + */ text: string; isFinal: boolean; }; +export type PushToTalkOptions = { + communicator: any; + provider: VoiceProviderKind; + elevenLabsApiKey: string; +}; + export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); private readonly wakeWordListeners = new Set<(keyword: string) => void>(); @@ -20,7 +34,8 @@ export class FaceclawVoiceControlBridge { private listenerProxy: any | null = null; private status = "Voice control stopped."; private started = false; - private mode: VoiceInputMode = "off"; + // Non-null while a cloud provider owns the transcript; Java only decodes PCM. + private cloudClient: ElevenLabsSttClient | null = null; onStatus(listener: (state: VoiceControlState) => void): () => void { this.statusListeners.add(listener); @@ -38,30 +53,55 @@ export class FaceclawVoiceControlBridge { return () => this.transcriptListeners.delete(listener); } - start(communicator?: any, mode: VoiceInputMode = "wakeword"): void { - if (mode === "off") { - this.stop(); + /** Begin push-to-talk capture with the selected provider. */ + startPushToTalk(options: PushToTalkOptions): void { + if (!global.isAndroid) return; + if (this.started) this.stop(); + this.ensureController(); + this.controller?.setCommunicator(options.communicator); + + if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) { + this.cloudClient = new ElevenLabsSttClient({ + apiKey: options.elevenLabsApiKey.trim(), + onTranscript: (event) => this.emitTranscript(event.text, event.isFinal), + onStatus: (status) => this.setStatus(status), + onError: (message) => this.setStatus(message), + }); + this.cloudClient.start(); + this.started = true; + this.controller?.start("cloud"); return; } - if (!global.isAndroid) return; - this.ensureController(); - if (communicator) { - this.controller?.setCommunicator(communicator); - } - if (this.started && this.mode !== mode) { - this.controller?.stop(); - this.started = false; + + if (options.provider === "elevenlabs") { + this.setStatus("No ElevenLabs key set; using on-device voice."); } + this.cloudClient = null; this.started = true; - this.mode = mode; - this.controller?.start(mode); + this.controller?.start("onboard"); + } + + /** End push-to-talk: stop the mic and, for cloud, commit for a final result. */ + stopPushToTalk(): void { + if (!this.started) { + this.cloudClient?.finish(); + return; + } + // Order matters for cloud: stopping the Java controller flushes any final + // decode/PCM; then commit so ElevenLabs finalizes the transcript. The + // socket is closed once the committed transcript arrives, or on stop(). + this.controller?.stop(); + this.started = false; + this.cloudClient?.finish(); } stop(): void { - if (!this.started || !global.isAndroid) return; + if (global.isAndroid) { + this.controller?.stop(); + } this.started = false; - this.mode = "off"; - this.controller?.stop(); + this.cloudClient?.stop(); + this.cloudClient = null; this.setStatus("Voice control stopped."); } @@ -82,15 +122,22 @@ export class FaceclawVoiceControlBridge { } }, onTranscript: (text: string, isFinal: boolean) => { - const event = { text: String(text), isFinal: Boolean(isFinal) }; - for (const listener of this.transcriptListeners) { - listener(event); - } + this.emitTranscript(String(text), Boolean(isFinal)); + }, + onPcm: (pcm: any) => { + this.cloudClient?.acceptPcm(toUint8Array(pcm)); }, }); this.controller.setListener(this.listenerProxy); } + private emitTranscript(text: string, isFinal: boolean): void { + const event = { text, isFinal }; + for (const listener of this.transcriptListeners) { + listener(event); + } + } + private setStatus(status: string): void { this.status = status; for (const listener of this.statusListeners) { diff --git a/app/ui/apps/transcribe.ts b/app/ui/apps/transcribe.ts index 2f47df7a..efe10c55 100644 --- a/app/ui/apps/transcribe.ts +++ b/app/ui/apps/transcribe.ts @@ -1,37 +1,37 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; -import { Layer, type DashboardInputEvent, type LayerContext } from "../layers"; +import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers"; export class TranscribeLayer implements Layer { - private mode: "off" | "wakeword" | "full" = "full"; private status = "Starting transcription..."; - private transcript = ""; + // Finalized utterances, plus the live (replace-semantics) partial appended + // when painting. + private finalizedText = ""; + private liveText = ""; private unsubscribeTranscript: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; + private actions: LayerActions | null = null; start(ctx: LayerContext): void { + this.actions = ctx.actions; this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { this.status = state.status; + ctx.actions.requestRender(); }); void ctx.actions.setTranscribeRenderActive(true); - void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => { - this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`; - }); + void Promise.resolve(ctx.actions.startVoiceCapture()); } - paint(ctx: LayerContext): GrayImage { + paint(): GrayImage { const font = getDefaultSmallFont(); const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); - const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off"; - const text = this.transcript || "Listening..."; + const text = this.displayText() || "Listening..."; const wrapped = wrapTranscribeText(font, text, G2_LENS_WIDTH - 64); image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52); image.drawText(font, 24, 24, "Transcribe", 200); - image.drawText(font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130); - //image.drawText(font, 24, 46, truncateTranscribeLine(this.status, 74), 110); image.drawText(font, 24, 46, this.status, 110); const firstLine = Math.max(0, wrapped.length - 10); @@ -40,63 +40,46 @@ export class TranscribeLayer implements Layer { image.drawText(font, 32, y, wrapped[index]!, 230); } - image.drawText(font, 24, 252, "Click: mode / Double-click: back", 110); + image.drawText(font, 24, 252, "Double-click: back", 110); return image; } handleInput(event: DashboardInputEvent, ctx: LayerContext): void { if (event.type === "double-click") { - this.stop(ctx); ctx.stack.pop(); - return; - } - if (event.type === "click") { - this.cycleMode(ctx); } } - private cycleMode(ctx: LayerContext): void { - if (this.mode === "full") { - this.mode = "wakeword"; - void ctx.actions.startDedicatedVoiceInput("wakeword"); - } else if (this.mode === "wakeword") { - this.mode = "off"; - void ctx.actions.stopDedicatedVoiceInput(); - } else { - this.mode = "full"; - void ctx.actions.startDedicatedVoiceInput("full"); - } - } - - private stop(ctx: LayerContext): void { + onRemoved(): void { this.unsubscribeTranscript?.(); this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); this.unsubscribeStatus = null; - void ctx.actions.setTranscribeRenderActive(false); - void ctx.actions.stopDedicatedVoiceInput(); + void this.actions?.setTranscribeRenderActive(false); + void this.actions?.stopVoiceCapture(); + this.actions = null; + } + + private displayText(): string { + if (!this.finalizedText) return this.liveText; + if (!this.liveText) return this.finalizedText; + return `${this.finalizedText} ${this.liveText}`; } private onTranscript(event: VoiceTranscriptEvent): void { if (event.isFinal) { - if (this.transcript && !this.transcript.endsWith("\n")) { - this.transcript = `${this.transcript}\n`; + const finalText = event.text.trim() || this.liveText.trim(); + if (finalText) { + this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText; } - return; + this.liveText = ""; + } else { + // Replace semantics: the live partial is the whole current best transcript. + this.liveText = event.text.trim(); } - this.transcript = appendTranscriptChunk(this.transcript, event.text); } } -function appendTranscriptChunk(transcript: string, chunk: string): string { - const trimmed = chunk.replace(/\s+/g, " ").trim(); - if (!trimmed) return transcript; - if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) { - return `${transcript}${trimmed}`; - } - return `${transcript} ${trimmed}`; -} - function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): string[] { const words = text.split(/\s+/).filter(Boolean); const lines: string[] = []; @@ -116,6 +99,3 @@ function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): stri return lines.length ? lines : [""]; } -function truncateTranscribeLine(text: string, maxChars: number): string { - return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`; -} diff --git a/app/ui/apps/voice-input.ts b/app/ui/apps/voice-input.ts new file mode 100644 index 00000000..db6e15de --- /dev/null +++ b/app/ui/apps/voice-input.ts @@ -0,0 +1,137 @@ +import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; +import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont"; +import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; +import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers"; + +const DIALOG_X = 40; +const DIALOG_Y = 40; +const DIALOG_W = G2_LENS_WIDTH - 80; +const DIALOG_H = G2_LENS_HEIGHT - 80; +const TEXT_MAX_WIDTH = DIALOG_W - 32; + +/** + * Push-to-talk voice dialog, drawn on top of whatever is already on screen. + * The mic runs while the button is held (long-press); releasing it stops the + * mic and finalizes. The dialog stays up until dismissed with a double-click. + */ +export class VoiceInputLayer implements Layer { + private status = "Listening..."; + private finalizedText = ""; + private liveText = ""; + private capturing = false; + private unsubscribeTranscript: (() => void) | null = null; + private unsubscribeStatus: (() => void) | null = null; + + constructor( + private readonly actions: LayerActions, + private readonly onClosed: () => void, + ) {} + + startCapture(): void { + if (this.capturing) return; + this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); + this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { + this.status = state.status; + this.actions.requestRender(); + }); + this.capturing = true; + void this.actions.startVoiceCapture(); + this.actions.requestRender(); + } + + /** Button released: stop the mic and finalize, but keep the dialog open. */ + endCapture(): void { + if (!this.capturing) return; + this.capturing = false; + void this.actions.stopVoiceCapture(); + if (this.status.startsWith("Listening")) { + this.status = "Double-click to close."; + } + this.actions.requestRender(); + } + + paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage { + const font = getDefaultSmallFont(); + const image = paintBelow(); + + // Solid dialog box over the underlying UI. + image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 0, 10); + image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10); + + const left = DIALOG_X + 16; + image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220); + image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130); + + const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)"); + const wrapped = wrapText(font, text, TEXT_MAX_WIDTH); + const maxLines = 8; + const firstLine = Math.max(0, wrapped.length - maxLines); + for (let index = firstLine; index < wrapped.length; index++) { + image.drawText(font, left, DIALOG_Y + 56 + (index - firstLine) * 16, wrapped[index]!, 235); + } + return image; + } + + handleInput(event: DashboardInputEvent, ctx: LayerContext): void { + if (event.type === "double-click") { + ctx.stack.pop(); + } + } + + onRemoved(): void { + this.unsubscribeTranscript?.(); + this.unsubscribeTranscript = null; + this.unsubscribeStatus?.(); + this.unsubscribeStatus = null; + if (this.capturing) { + this.capturing = false; + void this.actions.stopVoiceCapture(); + } + this.onClosed(); + } + + private displayText(): string { + if (!this.finalizedText) return this.liveText; + if (!this.liveText) return this.finalizedText; + return `${this.finalizedText} ${this.liveText}`; + } + + private onTranscript(event: VoiceTranscriptEvent): void { + if (event.isFinal) { + const finalText = event.text.trim() || this.liveText.trim(); + if (finalText) { + this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText; + } + this.liveText = ""; + } else { + this.liveText = event.text.trim(); + } + this.actions.requestRender(); + } +} + +function wrapText(font: BdfFont, text: string, maxWidth: number): string[] { + const words = text.split(/\s+/).filter(Boolean); + const lines: string[] = []; + let line = ""; + for (const word of words) { + const candidate = line ? `${line} ${word}` : word; + if (line && font.measureText(candidate) > maxWidth) { + lines.push(line); + line = word; + } else { + line = candidate; + } + } + if (line) lines.push(line); + return lines.length ? lines : [""]; +} + +function truncate(font: BdfFont, text: string, maxWidth: number): string { + if (font.measureText(text) <= maxWidth) return text; + let out = text; + while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) { + out = out.slice(0, -1); + } + return `${out}...`; +} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 9b134a4b..0f9c3c56 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -267,6 +267,27 @@ export const rawScreenshotsEnabledSetting = new ConfigSettingBoolean({ defaultValue: false, }); +export type VoiceProvider = "onboard" | "elevenlabs"; + +export const voiceProviderSetting = new ConfigSettingEnum({ + id: "voice-provider", + label: "Provider", + storageKey: "voice.provider", + defaultValue: "onboard", + values: ["onboard", "elevenlabs"], + formatValue: (value) => (value === "elevenlabs" ? "ElevenLabs" : "On-device"), +}); + +export const elevenLabsApiKeySetting = new ConfigSettingString({ + id: "elevenlabs-api-key", + label: "ElevenLabs key", + storageKey: "voice.elevenLabsApiKey", + defaultValue: "", + editorTitle: "ElevenLabs API key", + glassesEditTitle: "Edit ElevenLabs key", + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), +}); + export const terminalHostSetting = new ConfigSettingString({ id: "terminal-host", label: "Host", @@ -448,8 +469,10 @@ export function textSettingMenuItem( void ctx.actions.startTextSettingEdit(setting); ctx.stack.push(new EditTextSettingLayer(setting)); }, - render: ({ image, x, y, width, selected }) => { - image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.label}: ${truncateSetting(setting.get())}`, 200); + render: ({ image, x, y }) => { + // displayValue honors the setting's formatValue, so secrets (API keys, + // tokens) can mask themselves instead of rendering in the clear. + image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.label}: ${truncateSetting(setting.displayValue())}`, 200); } }; } diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts index 3c2bed57..3cba75ed 100644 --- a/app/ui/dashboard.ts +++ b/app/ui/dashboard.ts @@ -1,5 +1,4 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; -import { getDefaultSmallFont } from "../graphics/bdffont"; import { type RawInputEvent } from "../native/faceclaw-communicator"; import { mediaControllerBridge } from "../native/media-controller"; import { nightscoutBridge } from "../native/nightscout-bridge"; @@ -9,6 +8,7 @@ import { EditTextSettingLayer, isNightscoutSettingsConfigured, screenTimeoutSett import { Layer, LayerActions, LayerStack, type DashboardInputEvent, type LayerContext } from "./layers"; import { SingleNotificationLayer } from "./notifications"; import { TelepromptLayer } from "./apps/teleprompt"; +import { VoiceInputLayer } from "./apps/voice-input"; import { drawSystemCard } from "./dashboard/system-card"; import { createRootMenuLayer } from "./dashboard/root-menu"; @@ -44,16 +44,19 @@ const dashboardActions: LayerActions = { disconnect: () => {}, startTextSettingEdit: () => {}, endTextSettingEdit: () => {}, - setVoiceControlEnabled: () => {}, setStopwatchRenderActive: () => {}, setTranscribeRenderActive: () => {}, - startDedicatedVoiceInput: () => {}, - stopDedicatedVoiceInput: () => {}, + startVoiceCapture: () => {}, + stopVoiceCapture: () => {}, playBuzzerNote: () => {}, }; -const dashboardFont = getDefaultSmallFont(); export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 }; +// The push-to-talk dialog is a singleton on the layer stack; tracked here so +// long-press-release (a global event) can reach it and so a second long-press +// doesn't stack duplicates. +let activeVoiceInputLayer: VoiceInputLayer | null = null; + function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent { if (event.kind === "sys-event") { if (event.eventType === OsEventTypeList.CLICK_EVENT) { @@ -117,6 +120,23 @@ export async function receiveInput(event: RawInputEvent): Promise { } return; } + // Push-to-talk is intercepted globally so it works over any UI: long-press + // opens the voice dialog and starts the mic; releasing it stops the mic. + if (inputEvent.type === "long-press") { + if (!activeVoiceInputLayer) { + const layer = new VoiceInputLayer(dashboardActions, () => { + if (activeVoiceInputLayer === layer) activeVoiceInputLayer = null; + }); + activeVoiceInputLayer = layer; + dashboardLayers.push(layer); + layer.startCapture(); + } + return; + } + if (inputEvent.type === "long-press-release") { + activeVoiceInputLayer?.endCapture(); + return; + } await dashboardLayers.handleInput(inputEvent); } diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 3b862ac9..d97def58 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -1,5 +1,5 @@ import { dashboardState, TOP_LEFT_MENU_LAYOUT } from "../dashboard"; -import { batteryDisplayModeSetting, dashboardSlotIds, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceControlEnabledSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings"; +import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings"; import { MenuLayer } from "../menu"; export function createSettingsMenuLayer(): MenuLayer { @@ -69,7 +69,8 @@ function createVoiceSettingsMenuLayer(): MenuLayer { return new MenuLayer( "Settings > Voice", [ - toggleSettingMenuItem(voiceControlEnabledSetting), + enumSettingMenuItem(voiceProviderSetting), + textSettingMenuItem(elevenLabsApiKeySetting), ], TOP_LEFT_MENU_LAYOUT, ); diff --git a/app/ui/layers.ts b/app/ui/layers.ts index b09c8581..d431700c 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -18,11 +18,12 @@ export type LayerActions = { disconnect: () => Promise | void; startTextSettingEdit: (setting: ConfigSettingString) => Promise | void; endTextSettingEdit: () => Promise | void; - setVoiceControlEnabled: (enabled: boolean) => Promise | void; setStopwatchRenderActive: (active: boolean) => Promise | void; setTranscribeRenderActive: (active: boolean) => Promise | void; - startDedicatedVoiceInput: (mode: "wakeword" | "full") => Promise | void; - stopDedicatedVoiceInput: () => Promise | void; + /** Start voice capture with the provider chosen in settings. */ + startVoiceCapture: () => Promise | void; + /** Stop voice capture; for a cloud provider this also commits for a final result. */ + stopVoiceCapture: () => Promise | void; playBuzzerNote: (note: number, oct: number, beat: number) => Promise | void; };