forked from Blendman974/faceclaw
Work on voice input
This commit is contained in:
1 parent
7b8fc8f37f
commit
097d40efb2
12 files changed
+599
-234
No files matched your search
+84
-102
@@ -36,11 +36,15 @@ public class FaceclawVoiceController {
|
||||
private static final int EXPECTED_PACKET_INTERVAL_MS = 50;
|
||||
private static final int LATE_PACKET_INTERVAL_MS = 90;
|
||||
private static final int STATS_INTERVAL_MS = 5_000;
|
||||
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 900;
|
||||
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 2;
|
||||
private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 8;
|
||||
private static final int TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES = (int) (SAMPLE_RATE * 1.2);
|
||||
private static final float TRANSCRIPT_SILENCE_PEAK = 0.015f;
|
||||
// Push-to-talk utterance boundaries come from the button, so we accumulate
|
||||
// the whole utterance and re-decode it in full for each live partial. The
|
||||
// decode is of the complete audio each time, so the emitted text is the
|
||||
// model's current best transcript of everything spoken so far (REPLACE,
|
||||
// never a delta) — re-decoding a growing buffer and diffing prefixes was
|
||||
// the source of the duplicated/garbled output.
|
||||
private static final int TRANSCRIPT_DECODE_INTERVAL_MS = 700;
|
||||
private static final int TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3;
|
||||
private static final int TRANSCRIPT_MAX_SAMPLES = SAMPLE_RATE * 30;
|
||||
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
|
||||
private static final String ASSET_ROOT = "faceclaw-voice";
|
||||
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
||||
@@ -60,8 +64,9 @@ public class FaceclawVoiceController {
|
||||
};
|
||||
|
||||
private enum VoiceInputMode {
|
||||
WAKEWORD,
|
||||
FULL
|
||||
WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired)
|
||||
ONBOARD, // on-phone Moonshine transcription
|
||||
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
|
||||
}
|
||||
|
||||
private final Context appContext;
|
||||
@@ -80,11 +85,8 @@ public class FaceclawVoiceController {
|
||||
private FaceclawLc3Decoder lc3Decoder;
|
||||
private float[] transcriptSamples = new float[TRANSCRIPT_MAX_SAMPLES];
|
||||
private int transcriptSampleCount;
|
||||
private int transcriptSilentSamples;
|
||||
private boolean transcriptHadSpeech;
|
||||
private long lastTranscriptDecodeAtMs;
|
||||
private String lastTranscript = "";
|
||||
private float lastTranscriptPeak;
|
||||
private long queuedPackets;
|
||||
private long queueDroppedPackets;
|
||||
private long decodedSamples;
|
||||
@@ -157,35 +159,47 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
|
||||
private VoiceInputMode parseMode(String requestedMode) {
|
||||
return "full".equals(requestedMode) ? VoiceInputMode.FULL : VoiceInputMode.WAKEWORD;
|
||||
if ("cloud".equals(requestedMode)) {
|
||||
return VoiceInputMode.CLOUD;
|
||||
}
|
||||
if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) {
|
||||
return VoiceInputMode.ONBOARD;
|
||||
}
|
||||
return VoiceInputMode.WAKEWORD;
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
try {
|
||||
VoiceInputMode currentMode = mode;
|
||||
emitStatus(currentMode == VoiceInputMode.FULL
|
||||
? "Voice control loading transcription model..."
|
||||
: "Voice control loading wake-word model...");
|
||||
if (currentMode == VoiceInputMode.FULL) {
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
emitStatus("Loading transcription model...");
|
||||
File modelDir = installAsrModelFiles();
|
||||
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
} else {
|
||||
} else if (currentMode == VoiceInputMode.WAKEWORD) {
|
||||
emitStatus("Loading wake-word model...");
|
||||
File modelDir = installModelFiles();
|
||||
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
|
||||
stream = keywordSpotter.createStream();
|
||||
}
|
||||
lc3Decoder = new FaceclawLc3Decoder();
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Voice control could not start G2 microphone input.");
|
||||
emitStatus("Could not start G2 microphone input.");
|
||||
return;
|
||||
}
|
||||
|
||||
emitStatus(currentMode == VoiceInputMode.FULL
|
||||
? "Voice control transcribing from the G2 mic."
|
||||
: "Voice control listening for \"screen on\" from the G2 mic.");
|
||||
emitStatus(currentMode == VoiceInputMode.CLOUD
|
||||
? "Listening (cloud)..."
|
||||
: currentMode == VoiceInputMode.ONBOARD
|
||||
? "Listening..."
|
||||
: "Listening for \"screen on\"...");
|
||||
processG2Audio();
|
||||
// Button released / stop requested: emit one final full-utterance
|
||||
// transcript so the UI can freeze it.
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
decodeTranscript(true);
|
||||
}
|
||||
} catch (Throwable error) {
|
||||
Log.e(TAG, "Voice control failed", error);
|
||||
emitStatus("Voice control failed: " + error.getMessage());
|
||||
@@ -318,13 +332,19 @@ public class FaceclawVoiceController {
|
||||
continue;
|
||||
}
|
||||
decodedSamples += count;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
if (mode == VoiceInputMode.FULL) {
|
||||
if (mode == VoiceInputMode.CLOUD) {
|
||||
emitPcm(pcm, count);
|
||||
} else if (mode == VoiceInputMode.ONBOARD) {
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
processRecognizer(samples);
|
||||
} else {
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
processKeywordSpotter(currentStream);
|
||||
}
|
||||
@@ -350,29 +370,19 @@ public class FaceclawVoiceController {
|
||||
|
||||
private void processRecognizer(float[] samples) {
|
||||
appendTranscriptSamples(samples);
|
||||
updateTranscriptSilence(samples);
|
||||
|
||||
if (transcriptSampleCount >= TRANSCRIPT_MAX_SAMPLES) {
|
||||
decodeTranscriptSegment(true);
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
return;
|
||||
}
|
||||
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
|
||||
decodeTranscriptSegment(false);
|
||||
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES
|
||||
&& now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
|
||||
decodeTranscript(false);
|
||||
lastTranscriptDecodeAtMs = now;
|
||||
}
|
||||
|
||||
if (transcriptHadSpeech && transcriptSilentSamples >= TRANSCRIPT_SILENCE_ENDPOINT_SAMPLES) {
|
||||
decodeTranscriptSegment(true);
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
}
|
||||
}
|
||||
|
||||
private void appendTranscriptSamples(float[] samples) {
|
||||
// The utterance is bounded by the push-to-talk button; the cap is only a
|
||||
// safety limit. If it's hit, keep the most recent audio and let the
|
||||
// transcript track that window (the beginning is unlikely to still be on
|
||||
// screen after 30s anyway).
|
||||
int available = TRANSCRIPT_MAX_SAMPLES - transcriptSampleCount;
|
||||
if (samples.length > available) {
|
||||
int drop = samples.length - available;
|
||||
@@ -387,83 +397,55 @@ public class FaceclawVoiceController {
|
||||
transcriptSampleCount += samples.length;
|
||||
}
|
||||
|
||||
private void updateTranscriptSilence(float[] samples) {
|
||||
float peak = 0.0f;
|
||||
for (float sample : samples) {
|
||||
float abs = Math.abs(sample);
|
||||
if (abs > peak) {
|
||||
peak = abs;
|
||||
}
|
||||
}
|
||||
if (peak > lastTranscriptPeak) {
|
||||
lastTranscriptPeak = peak;
|
||||
}
|
||||
if (peak < TRANSCRIPT_SILENCE_PEAK) {
|
||||
transcriptSilentSamples += samples.length;
|
||||
} else {
|
||||
transcriptSilentSamples = 0;
|
||||
transcriptHadSpeech = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void decodeTranscriptSegment(boolean isFinal) {
|
||||
/**
|
||||
* Decode the entire accumulated utterance and emit the model's current best
|
||||
* full transcript (REPLACE semantics — the caller displays it as-is).
|
||||
*/
|
||||
private void decodeTranscript(boolean isFinal) {
|
||||
OfflineRecognizer currentRecognizer = recognizer;
|
||||
if (currentRecognizer == null || transcriptSampleCount <= 0) {
|
||||
if (isFinal) {
|
||||
emitTranscript(lastTranscript, true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
OfflineStream offlineStream = currentRecognizer.createStream();
|
||||
String text;
|
||||
try {
|
||||
offlineStream.acceptWaveform(Arrays.copyOf(transcriptSamples, transcriptSampleCount), SAMPLE_RATE);
|
||||
currentRecognizer.decode(offlineStream);
|
||||
emitRecognizerResult(currentRecognizer.getResult(offlineStream), isFinal, transcriptSampleCount, lastTranscriptPeak);
|
||||
OfflineRecognizerResult result = currentRecognizer.getResult(offlineStream);
|
||||
String raw = result == null ? "" : result.getText();
|
||||
text = raw == null ? "" : raw.trim();
|
||||
} finally {
|
||||
offlineStream.release();
|
||||
}
|
||||
}
|
||||
|
||||
private void emitRecognizerResult(OfflineRecognizerResult result, boolean isFinal, int sampleCount, float peak) {
|
||||
String text = result == null ? "" : result.getText();
|
||||
if (text == null) {
|
||||
text = "";
|
||||
}
|
||||
String trimmed = text.trim();
|
||||
String preview = trimmed.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
|
||||
? trimmed
|
||||
: trimmed.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
|
||||
lastTranscript = text;
|
||||
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
|
||||
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
|
||||
Log.i(TAG, "Moonshine decode final=" + isFinal
|
||||
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", sampleCount / (double) SAMPLE_RATE)
|
||||
+ " peak=" + String.format(java.util.Locale.US, "%.4f", peak)
|
||||
+ " textLen=" + trimmed.length()
|
||||
+ " text=\"" + preview + "\"");
|
||||
emitStatus("Moonshine decode "
|
||||
+ String.format(java.util.Locale.US, "%.1fs", sampleCount / (double) SAMPLE_RATE)
|
||||
+ ", textLen=" + trimmed.length());
|
||||
if (text.equals(lastTranscript)) {
|
||||
if (isFinal && lastTranscript.length() > 0) {
|
||||
emitTranscript("", true);
|
||||
lastTranscript = "";
|
||||
}
|
||||
return;
|
||||
}
|
||||
String delta = text;
|
||||
if (lastTranscript.length() > 0 && text.startsWith(lastTranscript)) {
|
||||
delta = text.substring(lastTranscript.length());
|
||||
}
|
||||
if (delta.trim().length() > 0) {
|
||||
emitTranscript(delta, false);
|
||||
}
|
||||
lastTranscript = isFinal ? "" : text;
|
||||
if (isFinal) {
|
||||
emitTranscript("", true);
|
||||
}
|
||||
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", transcriptSampleCount / (double) SAMPLE_RATE)
|
||||
+ " textLen=" + text.length() + " text=\"" + preview + "\"");
|
||||
emitTranscript(text, isFinal);
|
||||
}
|
||||
|
||||
private void resetTranscriptState() {
|
||||
transcriptSampleCount = 0;
|
||||
transcriptSilentSamples = 0;
|
||||
transcriptHadSpeech = false;
|
||||
lastTranscriptDecodeAtMs = 0;
|
||||
lastTranscriptPeak = 0.0f;
|
||||
}
|
||||
|
||||
private void emitPcm(short[] pcm, int count) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null || count <= 0) {
|
||||
return;
|
||||
}
|
||||
byte[] le = new byte[count * 2];
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = pcm[i];
|
||||
le[i * 2] = (byte) (s & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onPcm(le));
|
||||
}
|
||||
|
||||
private void stopG2Audio() {
|
||||
|
||||
+11
@@ -5,5 +5,16 @@ public interface FaceclawVoiceControllerListener {
|
||||
|
||||
void onWakeWord(String keyword);
|
||||
|
||||
/**
|
||||
* Best transcript of the current utterance so far. REPLACE semantics: text
|
||||
* is the complete transcript, not a delta — display it verbatim, replacing
|
||||
* any previous partial. isFinal marks the end of the utterance.
|
||||
*/
|
||||
void onTranscript(String text, boolean isFinal);
|
||||
|
||||
/**
|
||||
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
|
||||
* little-endian PCM. Empty/absent in onboard mode.
|
||||
*/
|
||||
void onPcm(byte[] pcm16le);
|
||||
}
|
||||
@@ -26,13 +26,21 @@ public class FaceclawWebSocket {
|
||||
private volatile boolean closeRequested;
|
||||
|
||||
public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener) {
|
||||
this(url, listener, null, null);
|
||||
}
|
||||
|
||||
public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener, String headerName, String headerValue) {
|
||||
if (url == null || url.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("url is required");
|
||||
}
|
||||
if (listener == null) {
|
||||
throw new IllegalArgumentException("listener is required");
|
||||
}
|
||||
Request request = new Request.Builder().url(url.trim()).build();
|
||||
Request.Builder builder = new Request.Builder().url(url.trim());
|
||||
if (headerName != null && !headerName.isEmpty() && headerValue != null) {
|
||||
builder.addHeader(headerName, headerValue);
|
||||
}
|
||||
Request request = builder.build();
|
||||
socket = getClient().newWebSocket(request, new WebSocketListener() {
|
||||
@Override public void onOpen(WebSocket webSocket, Response response) {
|
||||
mainHandler.post(() -> {
|
||||
|
||||
@@ -28,12 +28,13 @@ import {
|
||||
setDashboardActions,
|
||||
} from "../ui/dashboard";
|
||||
import {
|
||||
elevenLabsApiKeySetting,
|
||||
nightscoutApiTokenSetting,
|
||||
nightscoutSiteUrlSetting,
|
||||
onAnySettingChanged,
|
||||
rawScreenshotsEnabledSetting,
|
||||
systemCardNameSetting,
|
||||
voiceControlEnabledSetting,
|
||||
voiceProviderSetting,
|
||||
type ConfigSettingString,
|
||||
} from "../ui/dashboard-settings";
|
||||
import { saveRawScreenshot } from "../native/raw-screenshot";
|
||||
@@ -173,11 +174,10 @@ class DashboardController {
|
||||
disconnect: () => this.disconnect(),
|
||||
startTextSettingEdit: (setting) => this.startTextSettingEdit(setting),
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
|
||||
setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active),
|
||||
setTranscribeRenderActive: (active) => this.setTranscribeRenderActive(active),
|
||||
startDedicatedVoiceInput: (mode) => this.startDedicatedVoiceInput(mode),
|
||||
stopDedicatedVoiceInput: () => this.stopDedicatedVoiceInput(),
|
||||
startVoiceCapture: () => this.startVoiceCapture(),
|
||||
stopVoiceCapture: () => this.stopVoiceCapture(),
|
||||
playBuzzerNote: (note, oct, beat) => this.playBuzzerNote(note, oct, beat),
|
||||
});
|
||||
this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => {
|
||||
@@ -417,7 +417,6 @@ class DashboardController {
|
||||
await nightscoutBridge.start();
|
||||
await communicator.start();
|
||||
await this.requestRender("initial");
|
||||
this.startVoiceControlIfEnabled();
|
||||
this.dashboardTimer = setInterval(() => {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
const message = this.formatError(error);
|
||||
@@ -558,30 +557,32 @@ class DashboardController {
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private async setVoiceControlEnabled(enabled: boolean): Promise<void> {
|
||||
if (enabled) {
|
||||
try {
|
||||
// Treat Android mic permission as the consent gate for voice control,
|
||||
// even when the current source is the G2 mic over BLE.
|
||||
await ensureVoicePermissions();
|
||||
} catch (error) {
|
||||
this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
|
||||
return;
|
||||
}
|
||||
/**
|
||||
* Begin voice capture with the provider chosen in settings. Used by both
|
||||
* push-to-talk and the Transcribe app. Android mic permission is the consent
|
||||
* gate even though the audio source is the G2 mic over BLE.
|
||||
*/
|
||||
private startVoiceCapture(): void {
|
||||
if (this.phase !== "connected" || !this.communicator) {
|
||||
return;
|
||||
}
|
||||
|
||||
voiceControlEnabledSetting.set(enabled);
|
||||
this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`);
|
||||
if (enabled) {
|
||||
this.startVoiceControlIfEnabled();
|
||||
} else {
|
||||
voiceControlBridge.stop();
|
||||
}
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
await this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`voice setting render failed: ${this.formatError(error)}`);
|
||||
const communicator = this.communicator;
|
||||
void ensureVoicePermissions()
|
||||
.then(() => {
|
||||
if (this.phase !== "connected" || this.communicator !== communicator) return;
|
||||
voiceControlBridge.startPushToTalk({
|
||||
communicator: communicator.getNativeCommunicator(),
|
||||
provider: voiceProviderSetting.get(),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
this.appendLog(`voice permission failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private stopVoiceCapture(): void {
|
||||
voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
private endTextSettingEdit(): void {
|
||||
@@ -839,27 +840,6 @@ class DashboardController {
|
||||
this.updateDisplayPreviewFromImage(image);
|
||||
}
|
||||
|
||||
private startVoiceControlIfEnabled(): void {
|
||||
if (!this.communicator) return;
|
||||
if (!voiceControlEnabledSetting.get()) return;
|
||||
this.updateConnectedForegroundNotification();
|
||||
voiceControlBridge.start(this.communicator.getNativeCommunicator(), "wakeword");
|
||||
}
|
||||
|
||||
private async startDedicatedVoiceInput(mode: "wakeword" | "full"): Promise<void> {
|
||||
if (!this.communicator) {
|
||||
throw new Error("Voice input needs an active G2 connection.");
|
||||
}
|
||||
await ensureVoicePermissions();
|
||||
this.updateConnectedForegroundNotification();
|
||||
voiceControlBridge.start(this.communicator.getNativeCommunicator(), mode);
|
||||
}
|
||||
|
||||
private stopDedicatedVoiceInput(): void {
|
||||
voiceControlBridge.stop();
|
||||
this.startVoiceControlIfEnabled();
|
||||
}
|
||||
|
||||
private async playBuzzerNote(note: number, oct: number, beat: number): Promise<void> {
|
||||
if (this.phase !== "connected" || !this.communicator) {
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
declare const com: any;
|
||||
declare const android: any;
|
||||
|
||||
/**
|
||||
* ElevenLabs realtime speech-to-text over WebSocket.
|
||||
* https://elevenlabs.io/docs/api-reference/speech-to-text/v-1-speech-to-text-realtime
|
||||
*
|
||||
* We push 16 kHz signed-16-bit-LE PCM (base64) as it arrives from the G2 mic
|
||||
* and, because push-to-talk gives us the utterance boundary, use manual commit:
|
||||
* commit=true is sent when the button is released, which finalizes the
|
||||
* transcript. Partial transcripts stream in the meantime.
|
||||
*/
|
||||
|
||||
const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
|
||||
// The only model the realtime endpoint accepts (verified against the live API;
|
||||
// it rejects "scribe_v1" with an explicit message naming this one).
|
||||
const MODEL_ID = "scribe_v2_realtime";
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
export type ElevenLabsTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type ElevenLabsSttOptions = {
|
||||
apiKey: string;
|
||||
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
export class ElevenLabsSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// PCM that arrived before the socket finished opening; flushed on open.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private latestText = "";
|
||||
|
||||
constructor(private readonly options: ElevenLabsSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
// The API key rides an xi-api-key header, added by FaceclawWebSocket.
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", this.options.apiKey);
|
||||
this.options.onStatus("Connecting to ElevenLabs...");
|
||||
} catch (error) {
|
||||
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Feed PCM (16 kHz signed-16-bit LE). */
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.closed || pcm.length === 0) return;
|
||||
const base64 = encodeBase64(pcm);
|
||||
if (this.open) {
|
||||
this.sendChunk(base64);
|
||||
} else {
|
||||
this.pendingChunks.push(base64);
|
||||
}
|
||||
}
|
||||
|
||||
/** End of utterance: flush and commit for a final transcript. */
|
||||
finish(): void {
|
||||
if (this.closed) return;
|
||||
const commitMessage = JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: "",
|
||||
commit: true,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
});
|
||||
if (this.open) {
|
||||
this.trySend(commitMessage);
|
||||
} else {
|
||||
this.pendingChunks.push("__commit__");
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.finish();
|
||||
return;
|
||||
}
|
||||
this.trySend(
|
||||
JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: base64OrCommit,
|
||||
commit: false,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
} catch (error) {
|
||||
console.warn("elevenlabs send failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
private handleMessage(text: string): void {
|
||||
let message: any;
|
||||
try {
|
||||
message = JSON.parse(text);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
switch (message?.message_type) {
|
||||
case "session_started":
|
||||
this.options.onStatus("Listening (ElevenLabs)...");
|
||||
return;
|
||||
case "partial_transcript":
|
||||
this.latestText = String(message.text ?? "");
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
return;
|
||||
case "committed_transcript":
|
||||
case "committed_transcript_with_timestamps":
|
||||
this.latestText = String(message.text ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
return;
|
||||
case "error":
|
||||
case "auth_error":
|
||||
case "quota_exceeded":
|
||||
case "rate_limited":
|
||||
case "input_error":
|
||||
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function encodeBase64(bytes: Uint8Array): string {
|
||||
if (!global.isAndroid) return "";
|
||||
const javaBytes = Array.create("byte", bytes.length);
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const value = bytes[i]!;
|
||||
javaBytes[i] = value > 127 ? value - 256 : value;
|
||||
}
|
||||
return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP));
|
||||
}
|
||||
+69
-22
@@ -1,17 +1,31 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type VoiceControlState = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type VoiceInputMode = "off" | "wakeword" | "full";
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs";
|
||||
|
||||
export type VoiceTranscriptEvent = {
|
||||
/**
|
||||
* Complete best transcript of the current utterance. REPLACE semantics —
|
||||
* render as-is, replacing any previous partial. Not a delta.
|
||||
*/
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type PushToTalkOptions = {
|
||||
communicator: any;
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
};
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
@@ -20,7 +34,8 @@ export class FaceclawVoiceControlBridge {
|
||||
private listenerProxy: any | null = null;
|
||||
private status = "Voice control stopped.";
|
||||
private started = false;
|
||||
private mode: VoiceInputMode = "off";
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: ElevenLabsSttClient | null = null;
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
@@ -38,30 +53,55 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.transcriptListeners.delete(listener);
|
||||
}
|
||||
|
||||
start(communicator?: any, mode: VoiceInputMode = "wakeword"): void {
|
||||
if (mode === "off") {
|
||||
this.stop();
|
||||
/** Begin push-to-talk capture with the selected provider. */
|
||||
startPushToTalk(options: PushToTalkOptions): void {
|
||||
if (!global.isAndroid) return;
|
||||
if (this.started) this.stop();
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
|
||||
if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) {
|
||||
this.cloudClient = new ElevenLabsSttClient({
|
||||
apiKey: options.elevenLabsApiKey.trim(),
|
||||
onTranscript: (event) => this.emitTranscript(event.text, event.isFinal),
|
||||
onStatus: (status) => this.setStatus(status),
|
||||
onError: (message) => this.setStatus(message),
|
||||
});
|
||||
this.cloudClient.start();
|
||||
this.started = true;
|
||||
this.controller?.start("cloud");
|
||||
return;
|
||||
}
|
||||
if (!global.isAndroid) return;
|
||||
this.ensureController();
|
||||
if (communicator) {
|
||||
this.controller?.setCommunicator(communicator);
|
||||
}
|
||||
if (this.started && this.mode !== mode) {
|
||||
this.controller?.stop();
|
||||
this.started = false;
|
||||
|
||||
if (options.provider === "elevenlabs") {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
}
|
||||
this.cloudClient = null;
|
||||
this.started = true;
|
||||
this.mode = mode;
|
||||
this.controller?.start(mode);
|
||||
this.controller?.start("onboard");
|
||||
}
|
||||
|
||||
/** End push-to-talk: stop the mic and, for cloud, commit for a final result. */
|
||||
stopPushToTalk(): void {
|
||||
if (!this.started) {
|
||||
this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
// Order matters for cloud: stopping the Java controller flushes any final
|
||||
// decode/PCM; then commit so ElevenLabs finalizes the transcript. The
|
||||
// socket is closed once the committed transcript arrives, or on stop().
|
||||
this.controller?.stop();
|
||||
this.started = false;
|
||||
this.cloudClient?.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.started || !global.isAndroid) return;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
this.started = false;
|
||||
this.mode = "off";
|
||||
this.controller?.stop();
|
||||
this.cloudClient?.stop();
|
||||
this.cloudClient = null;
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
|
||||
@@ -82,15 +122,22 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
},
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
const event = { text: String(text), isFinal: Boolean(isFinal) };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
onPcm: (pcm: any) => {
|
||||
this.cloudClient?.acceptPcm(toUint8Array(pcm));
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
private emitTranscript(text: string, isFinal: boolean): void {
|
||||
const event = { text, isFinal };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
}
|
||||
|
||||
private setStatus(status: string): void {
|
||||
this.status = status;
|
||||
for (const listener of this.statusListeners) {
|
||||
|
||||
+29
-49
@@ -1,37 +1,37 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { Layer, type DashboardInputEvent, type LayerContext } from "../layers";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
|
||||
export class TranscribeLayer implements Layer {
|
||||
private mode: "off" | "wakeword" | "full" = "full";
|
||||
private status = "Starting transcription...";
|
||||
private transcript = "";
|
||||
// Finalized utterances, plus the live (replace-semantics) partial appended
|
||||
// when painting.
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private actions: LayerActions | null = null;
|
||||
|
||||
start(ctx: LayerContext): void {
|
||||
this.actions = ctx.actions;
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
ctx.actions.requestRender();
|
||||
});
|
||||
void ctx.actions.setTranscribeRenderActive(true);
|
||||
void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => {
|
||||
this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`;
|
||||
});
|
||||
void Promise.resolve(ctx.actions.startVoiceCapture());
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
paint(): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off";
|
||||
const text = this.transcript || "Listening...";
|
||||
const text = this.displayText() || "Listening...";
|
||||
const wrapped = wrapTranscribeText(font, text, G2_LENS_WIDTH - 64);
|
||||
|
||||
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
|
||||
image.drawText(font, 24, 24, "Transcribe", 200);
|
||||
image.drawText(font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130);
|
||||
//image.drawText(font, 24, 46, truncateTranscribeLine(this.status, 74), 110);
|
||||
image.drawText(font, 24, 46, this.status, 110);
|
||||
|
||||
const firstLine = Math.max(0, wrapped.length - 10);
|
||||
@@ -40,63 +40,46 @@ export class TranscribeLayer implements Layer {
|
||||
image.drawText(font, 32, y, wrapped[index]!, 230);
|
||||
}
|
||||
|
||||
image.drawText(font, 24, 252, "Click: mode / Double-click: back", 110);
|
||||
image.drawText(font, 24, 252, "Double-click: back", 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "double-click") {
|
||||
this.stop(ctx);
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click") {
|
||||
this.cycleMode(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
private cycleMode(ctx: LayerContext): void {
|
||||
if (this.mode === "full") {
|
||||
this.mode = "wakeword";
|
||||
void ctx.actions.startDedicatedVoiceInput("wakeword");
|
||||
} else if (this.mode === "wakeword") {
|
||||
this.mode = "off";
|
||||
void ctx.actions.stopDedicatedVoiceInput();
|
||||
} else {
|
||||
this.mode = "full";
|
||||
void ctx.actions.startDedicatedVoiceInput("full");
|
||||
}
|
||||
}
|
||||
|
||||
private stop(ctx: LayerContext): void {
|
||||
onRemoved(): void {
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
void ctx.actions.setTranscribeRenderActive(false);
|
||||
void ctx.actions.stopDedicatedVoiceInput();
|
||||
void this.actions?.setTranscribeRenderActive(false);
|
||||
void this.actions?.stopVoiceCapture();
|
||||
this.actions = null;
|
||||
}
|
||||
|
||||
private displayText(): string {
|
||||
if (!this.finalizedText) return this.liveText;
|
||||
if (!this.liveText) return this.finalizedText;
|
||||
return `${this.finalizedText} ${this.liveText}`;
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (event.isFinal) {
|
||||
if (this.transcript && !this.transcript.endsWith("\n")) {
|
||||
this.transcript = `${this.transcript}\n`;
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
return;
|
||||
this.liveText = "";
|
||||
} else {
|
||||
// Replace semantics: the live partial is the whole current best transcript.
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
this.transcript = appendTranscriptChunk(this.transcript, event.text);
|
||||
}
|
||||
}
|
||||
|
||||
function appendTranscriptChunk(transcript: string, chunk: string): string {
|
||||
const trimmed = chunk.replace(/\s+/g, " ").trim();
|
||||
if (!trimmed) return transcript;
|
||||
if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) {
|
||||
return `${transcript}${trimmed}`;
|
||||
}
|
||||
return `${transcript} ${trimmed}`;
|
||||
}
|
||||
|
||||
function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
@@ -116,6 +99,3 @@ function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): stri
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
function truncateTranscribeLine(text: string, maxChars: number): string {
|
||||
return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
|
||||
const DIALOG_X = 40;
|
||||
const DIALOG_Y = 40;
|
||||
const DIALOG_W = G2_LENS_WIDTH - 80;
|
||||
const DIALOG_H = G2_LENS_HEIGHT - 80;
|
||||
const TEXT_MAX_WIDTH = DIALOG_W - 32;
|
||||
|
||||
/**
|
||||
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
|
||||
* The mic runs while the button is held (long-press); releasing it stops the
|
||||
* mic and finalizes. The dialog stays up until dismissed with a double-click.
|
||||
*/
|
||||
export class VoiceInputLayer implements Layer {
|
||||
private status = "Listening...";
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private capturing = false;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly onClosed: () => void,
|
||||
) {}
|
||||
|
||||
startCapture(): void {
|
||||
if (this.capturing) return;
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.actions.requestRender();
|
||||
});
|
||||
this.capturing = true;
|
||||
void this.actions.startVoiceCapture();
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
/** Button released: stop the mic and finalize, but keep the dialog open. */
|
||||
endCapture(): void {
|
||||
if (!this.capturing) return;
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.status.startsWith("Listening")) {
|
||||
this.status = "Double-click to close.";
|
||||
}
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = paintBelow();
|
||||
|
||||
// Solid dialog box over the underlying UI.
|
||||
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 0, 10);
|
||||
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
|
||||
|
||||
const left = DIALOG_X + 16;
|
||||
image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220);
|
||||
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
|
||||
|
||||
const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)");
|
||||
const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
|
||||
const maxLines = 8;
|
||||
const firstLine = Math.max(0, wrapped.length - maxLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
image.drawText(font, left, DIALOG_Y + 56 + (index - firstLine) * 16, wrapped[index]!, 235);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
if (this.capturing) {
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
}
|
||||
this.onClosed();
|
||||
}
|
||||
|
||||
private displayText(): string {
|
||||
if (!this.finalizedText) return this.liveText;
|
||||
if (!this.liveText) return this.finalizedText;
|
||||
return `${this.finalizedText} ${this.liveText}`;
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (event.isFinal) {
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
this.liveText = "";
|
||||
} else {
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
this.actions.requestRender();
|
||||
}
|
||||
}
|
||||
|
||||
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && font.measureText(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = candidate;
|
||||
}
|
||||
}
|
||||
if (line) lines.push(line);
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
function truncate(font: BdfFont, text: string, maxWidth: number): string {
|
||||
if (font.measureText(text) <= maxWidth) return text;
|
||||
let out = text;
|
||||
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
|
||||
out = out.slice(0, -1);
|
||||
}
|
||||
return `${out}...`;
|
||||
}
|
||||
@@ -267,6 +267,27 @@ export const rawScreenshotsEnabledSetting = new ConfigSettingBoolean({
|
||||
defaultValue: false,
|
||||
});
|
||||
|
||||
export type VoiceProvider = "onboard" | "elevenlabs";
|
||||
|
||||
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
id: "voice-provider",
|
||||
label: "Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: ["onboard", "elevenlabs"],
|
||||
formatValue: (value) => (value === "elevenlabs" ? "ElevenLabs" : "On-device"),
|
||||
});
|
||||
|
||||
export const elevenLabsApiKeySetting = new ConfigSettingString({
|
||||
id: "elevenlabs-api-key",
|
||||
label: "ElevenLabs key",
|
||||
storageKey: "voice.elevenLabsApiKey",
|
||||
defaultValue: "",
|
||||
editorTitle: "ElevenLabs API key",
|
||||
glassesEditTitle: "Edit ElevenLabs key",
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
|
||||
});
|
||||
|
||||
export const terminalHostSetting = new ConfigSettingString({
|
||||
id: "terminal-host",
|
||||
label: "Host",
|
||||
@@ -448,8 +469,10 @@ export function textSettingMenuItem<TId extends string = string>(
|
||||
void ctx.actions.startTextSettingEdit(setting);
|
||||
ctx.stack.push(new EditTextSettingLayer(setting));
|
||||
},
|
||||
render: ({ image, x, y, width, selected }) => {
|
||||
image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.label}: ${truncateSetting(setting.get())}`, 200);
|
||||
render: ({ image, x, y }) => {
|
||||
// displayValue honors the setting's formatValue, so secrets (API keys,
|
||||
// tokens) can mask themselves instead of rendering in the clear.
|
||||
image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.label}: ${truncateSetting(setting.displayValue())}`, 200);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
+25
-5
@@ -1,5 +1,4 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import { getDefaultSmallFont } from "../graphics/bdffont";
|
||||
import { type RawInputEvent } from "../native/faceclaw-communicator";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { nightscoutBridge } from "../native/nightscout-bridge";
|
||||
@@ -9,6 +8,7 @@ import { EditTextSettingLayer, isNightscoutSettingsConfigured, screenTimeoutSett
|
||||
import { Layer, LayerActions, LayerStack, type DashboardInputEvent, type LayerContext } from "./layers";
|
||||
import { SingleNotificationLayer } from "./notifications";
|
||||
import { TelepromptLayer } from "./apps/teleprompt";
|
||||
import { VoiceInputLayer } from "./apps/voice-input";
|
||||
import { drawSystemCard } from "./dashboard/system-card";
|
||||
import { createRootMenuLayer } from "./dashboard/root-menu";
|
||||
|
||||
@@ -44,16 +44,19 @@ const dashboardActions: LayerActions = {
|
||||
disconnect: () => {},
|
||||
startTextSettingEdit: () => {},
|
||||
endTextSettingEdit: () => {},
|
||||
setVoiceControlEnabled: () => {},
|
||||
setStopwatchRenderActive: () => {},
|
||||
setTranscribeRenderActive: () => {},
|
||||
startDedicatedVoiceInput: () => {},
|
||||
stopDedicatedVoiceInput: () => {},
|
||||
startVoiceCapture: () => {},
|
||||
stopVoiceCapture: () => {},
|
||||
playBuzzerNote: () => {},
|
||||
};
|
||||
const dashboardFont = getDefaultSmallFont();
|
||||
export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 };
|
||||
|
||||
// The push-to-talk dialog is a singleton on the layer stack; tracked here so
|
||||
// long-press-release (a global event) can reach it and so a second long-press
|
||||
// doesn't stack duplicates.
|
||||
let activeVoiceInputLayer: VoiceInputLayer | null = null;
|
||||
|
||||
function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
|
||||
if (event.kind === "sys-event") {
|
||||
if (event.eventType === OsEventTypeList.CLICK_EVENT) {
|
||||
@@ -117,6 +120,23 @@ export async function receiveInput(event: RawInputEvent): Promise<void> {
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Push-to-talk is intercepted globally so it works over any UI: long-press
|
||||
// opens the voice dialog and starts the mic; releasing it stops the mic.
|
||||
if (inputEvent.type === "long-press") {
|
||||
if (!activeVoiceInputLayer) {
|
||||
const layer = new VoiceInputLayer(dashboardActions, () => {
|
||||
if (activeVoiceInputLayer === layer) activeVoiceInputLayer = null;
|
||||
});
|
||||
activeVoiceInputLayer = layer;
|
||||
dashboardLayers.push(layer);
|
||||
layer.startCapture();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (inputEvent.type === "long-press-release") {
|
||||
activeVoiceInputLayer?.endCapture();
|
||||
return;
|
||||
}
|
||||
await dashboardLayers.handleInput(inputEvent);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { dashboardState, TOP_LEFT_MENU_LAYOUT } from "../dashboard";
|
||||
import { batteryDisplayModeSetting, dashboardSlotIds, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceControlEnabledSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
|
||||
import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
|
||||
import { MenuLayer } from "../menu";
|
||||
|
||||
export function createSettingsMenuLayer(): MenuLayer {
|
||||
@@ -69,7 +69,8 @@ function createVoiceSettingsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Settings > Voice",
|
||||
[
|
||||
toggleSettingMenuItem(voiceControlEnabledSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
textSettingMenuItem(elevenLabsApiKeySetting),
|
||||
],
|
||||
TOP_LEFT_MENU_LAYOUT,
|
||||
);
|
||||
|
||||
+4
-3
@@ -18,11 +18,12 @@ export type LayerActions = {
|
||||
disconnect: () => Promise<void> | void;
|
||||
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
|
||||
endTextSettingEdit: () => Promise<void> | void;
|
||||
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
|
||||
setStopwatchRenderActive: (active: boolean) => Promise<void> | void;
|
||||
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
|
||||
startDedicatedVoiceInput: (mode: "wakeword" | "full") => Promise<void> | void;
|
||||
stopDedicatedVoiceInput: () => Promise<void> | void;
|
||||
/** Start voice capture with the provider chosen in settings. */
|
||||
startVoiceCapture: () => Promise<void> | void;
|
||||
/** Stop voice capture; for a cloud provider this also commits for a final result. */
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
playBuzzerNote: (note: number, oct: number, beat: number) => Promise<void> | void;
|
||||
};
|
||||
|
||||
|
||||
Reference in new issue
Block a user